Compression: Handle auto encryption when size is unknown (#7600)

When size is unknown and auto encryption is enabled,
and compression is set to true, putobject API is failing.

Moving adding the SSE-S3 header as part of the request to before
checking if compression can be done, otherwise the size is set to -1
and that seems to cause problems.

cmd/admin-handlers.go

@@ -35,7 +35,6 @@ import (
 	"github.com/tidwall/sjson"
 
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/handlers"
@@ -1416,70 +1415,3 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 	// Send success response
 	writeSuccessResponseHeadersOnly(w)
 }
-
-// UpdateAdminCredsHandler - POST /minio/admin/v1/config/credential
-// ----------
-// Update admin credentials in a minio server
-func (a adminAPIHandlers) UpdateAdminCredentialsHandler(w http.ResponseWriter,
-	r *http.Request) {
-
-	ctx := newContext(r, w, "UpdateCredentialsHandler")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	// Avoid setting new credentials when they are already passed
-	// by the environment. Deny if WORM is enabled.
-	if globalIsEnvCreds || globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
-	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
-		// More than maxConfigSize bytes were available
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
-		return
-	}
-
-	password := globalServerConfig.GetCredential().SecretKey
-	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
-	if err != nil {
-		logger.LogIf(ctx, err)
-		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
-		return
-	}
-
-	// Decode request body
-	var req madmin.SetCredsReq
-	if err = json.Unmarshal(configBytes, &req); err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrRequestBodyParse), r.URL)
-		return
-	}
-
-	creds, err := auth.CreateCredentials(req.AccessKey, req.SecretKey)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Acquire lock before updating global configuration.
-	globalServerConfigMu.Lock()
-	defer globalServerConfigMu.Unlock()
-
-	// Update local credentials in memory.
-	globalServerConfig.SetCredential(creds)
-
-	// Set active creds.
-	globalActiveCred = creds
-
-	if err = saveServerConfig(ctx, objectAPI, globalServerConfig); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Reply to the client before restarting minio server.
-	writeSuccessResponseHeadersOnly(w)
-}

cmd/admin-handlers_test.go

@@ -599,86 +599,6 @@ func TestServiceRestartHandler(t *testing.T) {
 	testServicesCmdHandler(restartCmd, t)
 }
 
-// Test for service set creds management REST API.
-func TestServiceSetCreds(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
-	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
-	}
-	defer adminTestBed.TearDown()
-
-	// Initialize admin peers to make admin RPC calls. Note: In a
-	// single node setup, this degenerates to a simple function
-	// call under the hood.
-	globalMinioAddr = "127.0.0.1:9000"
-
-	credentials := globalServerConfig.GetCredential()
-
-	testCases := []struct {
-		AccessKey          string
-		SecretKey          string
-		EnvKeysSet         bool
-		ExpectedStatusCode int
-	}{
-		// Bad secret key
-		{"minio", "minio", false, http.StatusBadRequest},
-		// Bad  secret key set from the env
-		{"minio", "minio", true, http.StatusMethodNotAllowed},
-		// Good keys set from the env
-		{"minio", "minio123", true, http.StatusMethodNotAllowed},
-		// Successful operation should be the last one to
-		// not change server credentials during tests.
-		{"minio", "minio123", false, http.StatusOK},
-	}
-	for i, testCase := range testCases {
-		// Set or unset environement keys
-		globalIsEnvCreds = testCase.EnvKeysSet
-
-		// Construct setCreds request body
-		body, err := json.Marshal(madmin.SetCredsReq{
-			AccessKey: testCase.AccessKey,
-			SecretKey: testCase.SecretKey})
-		if err != nil {
-			t.Fatalf("JSONify err: %v", err)
-		}
-
-		ebody, err := madmin.EncryptData(credentials.SecretKey, body)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// Construct setCreds request
-		req, err := getServiceCmdRequest(setCreds, credentials, ebody)
-		if err != nil {
-			t.Fatalf("Failed to build service status request %v", err)
-		}
-
-		rec := httptest.NewRecorder()
-
-		// Execute request
-		adminTestBed.router.ServeHTTP(rec, req)
-
-		// Check if the http code response is expected
-		if rec.Code != testCase.ExpectedStatusCode {
-			t.Errorf("Test %d: Wrong status code, expected = %d, found = %d", i+1, testCase.ExpectedStatusCode, rec.Code)
-			resp, _ := ioutil.ReadAll(rec.Body)
-			t.Errorf("Expected to receive %d status code but received %d. Body (%s)",
-				http.StatusOK, rec.Code, string(resp))
-		}
-
-		// If we got 200 OK, check if new credentials are really set
-		if rec.Code == http.StatusOK {
-			cred := globalServerConfig.GetCredential()
-			if cred.AccessKey != testCase.AccessKey {
-				t.Errorf("Test %d: Wrong access key, expected = %s, found = %s", i+1, testCase.AccessKey, cred.AccessKey)
-			}
-			if cred.SecretKey != testCase.SecretKey {
-				t.Errorf("Test %d: Wrong secret key, expected = %s, found = %s", i+1, testCase.SecretKey, cred.SecretKey)
-			}
-		}
-	}
-}
-
 // buildAdminRequest - helper function to build an admin API request.
 func buildAdminRequest(queryVal url.Values, method, path string,
 	contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error) {

cmd/admin-router.go

@@ -74,8 +74,6 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 	/// Config operations
 	if enableConfigOps {
-		// Update credentials
-		adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateAdminCredentialsHandler))
 		// Get config
 		adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
 		// Set config

cmd/api-errors.go

@@ -65,6 +65,7 @@ const (
 	ErrBadDigest
 	ErrEntityTooSmall
 	ErrEntityTooLarge
+	ErrPolicyTooLarge
 	ErrIncompleteBody
 	ErrInternalError
 	ErrInvalidAccessKeyID
@@ -398,6 +399,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Your proposed upload exceeds the maximum allowed object size.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrPolicyTooLarge: {
+		Code:           "PolicyTooLarge",
+		Description:    "Policy exceeds the maximum allowed document size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrIncompleteBody: {
 		Code:           "IncompleteBody",
 		Description:    "You did not provide the number of bytes specified by the Content-Length HTTP header.",
@@ -1479,7 +1485,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	if err == nil {
 		return ErrNone
 	}
-
 	// Verify if the underlying error is signature mismatch.
 	switch err {
 	case errInvalidArgument:
@@ -1529,6 +1534,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrKMSAuthFailure
 	case errOperationTimedOut, context.Canceled, context.DeadlineExceeded:
 		apiErr = ErrOperationTimedOut
+	case errNetworkConnReset:
+		apiErr = ErrSlowDown
 	}
 
 	// Compression errors

cmd/api-router.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"net"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -45,7 +46,13 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
 	apiRouter := router.PathPrefix("/").Subrouter()
 	var routers []*mux.Router
 	for _, domainName := range globalDomainNames {
-		routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName).Subrouter())
+		var hostPort string
+		if globalMinioPort != "443" && globalMinioPort != "80" {
+			hostPort = net.JoinHostPort(domainName, globalMinioPort)
+		} else {
+			hostPort = domainName
+		}
+		routers = append(routers, apiRouter.Host("{bucket:.+}."+hostPort).Subrouter())
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 

cmd/bitrot-streaming.go

@@ -32,33 +32,20 @@ type streamingBitrotWriter struct {
 	h         hash.Hash
 	shardSize int64
 	canClose  chan struct{} // Needed to avoid race explained in Close() call.
-
-	// Following two fields are used only to make sure that Write(p) is called such that
-	// len(p) is always the block size except the last block, i.e prevent programmer errors.
-	currentBlockIdx int
-	verifyTillIdx   int
 }
 
 func (b *streamingBitrotWriter) Write(p []byte) (int, error) {
-	if b.currentBlockIdx < b.verifyTillIdx && int64(len(p)) != b.shardSize {
-		// All blocks except last should be of the length b.shardSize
-		logger.LogIf(context.Background(), errUnexpected)
-		return 0, errUnexpected
-	}
 	if len(p) == 0 {
 		return 0, nil
 	}
 	b.h.Reset()
 	b.h.Write(p)
 	hashBytes := b.h.Sum(nil)
-	n, err := b.iow.Write(hashBytes)
-	if n != len(hashBytes) {
-		logger.LogIf(context.Background(), err)
+	_, err := b.iow.Write(hashBytes)
+	if err != nil {
 		return 0, err
 	}
-	n, err = b.iow.Write(p)
-	b.currentBlockIdx++
-	return n, err
+	return b.iow.Write(p)
 }
 
 func (b *streamingBitrotWriter) Close() error {
@@ -78,16 +65,14 @@ func (b *streamingBitrotWriter) Close() error {
 func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.WriteCloser {
 	r, w := io.Pipe()
 	h := algo.New()
-	bw := &streamingBitrotWriter{w, h, shardSize, make(chan struct{}), 0, int(length / shardSize)}
+	bw := &streamingBitrotWriter{w, h, shardSize, make(chan struct{})}
 	go func() {
+		totalFileSize := int64(-1) // For compressed objects length will be unknown (represented by length=-1)
+		if length != -1 {
 			bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
-		totalFileSize := bitrotSumsTotalSize + length
-		err := disk.CreateFile(volume, filePath, totalFileSize, r)
-		if err != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("storageDisk", disk.String())
-			ctx := logger.SetReqInfo(context.Background(), reqInfo)
-			logger.LogIf(ctx, err)
+			totalFileSize = bitrotSumsTotalSize + length
 		}
+		err := disk.CreateFile(volume, filePath, totalFileSize, r)
 		r.CloseWithError(err)
 		close(bw.canClose)
 	}()
@@ -118,7 +103,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	var err error
 	if offset%b.shardSize != 0 {
 		// Offset should always be aligned to b.shardSize
-		logger.LogIf(context.Background(), errUnexpected)
+		// Can never happen unless there are programmer bugs
 		return 0, errUnexpected
 	}
 	if b.rc == nil {
@@ -127,25 +112,20 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
 		b.rc, err = b.disk.ReadFileStream(b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
 		if err != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("storageDisk", b.disk.String())
-			ctx := logger.SetReqInfo(context.Background(), reqInfo)
-			logger.LogIf(ctx, err)
 			return 0, err
 		}
 	}
 	if offset != b.currOffset {
-		logger.LogIf(context.Background(), errUnexpected)
+		// Can never happen unless there are programmer bugs
 		return 0, errUnexpected
 	}
 	b.h.Reset()
 	_, err = io.ReadFull(b.rc, b.hashBytes)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
 		return 0, err
 	}
 	_, err = io.ReadFull(b.rc, buf)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
 		return 0, err
 	}
 	b.h.Write(buf)

cmd/bitrot-whole.go

@@ -31,19 +31,9 @@ type wholeBitrotWriter struct {
 	filePath  string
 	shardSize int64 // This is the shard size of the erasure logic
 	hash.Hash       // For bitrot hash
-
-	// Following two fields are used only to make sure that Write(p) is called such that
-	// len(p) is always the block size except the last block and prevent programmer errors.
-	currentBlockIdx int
-	lastBlockIdx    int
 }
 
 func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
-	if b.currentBlockIdx < b.lastBlockIdx && int64(len(p)) != b.shardSize {
-		// All blocks except last should be of the length b.shardSize
-		logger.LogIf(context.Background(), errUnexpected)
-		return 0, errUnexpected
-	}
 	err := b.disk.AppendFile(b.volume, b.filePath, p)
 	if err != nil {
 		logger.LogIf(context.Background(), err)
@@ -54,7 +44,6 @@ func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
 		logger.LogIf(context.Background(), err)
 		return 0, err
 	}
-	b.currentBlockIdx++
 	return len(p), nil
 }
 
@@ -63,8 +52,8 @@ func (b *wholeBitrotWriter) Close() error {
 }
 
 // Returns whole-file bitrot writer.
-func newWholeBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.WriteCloser {
-	return &wholeBitrotWriter{disk, volume, filePath, shardSize, algo.New(), 0, int(length / shardSize)}
+func newWholeBitrotWriter(disk StorageAPI, volume, filePath string, algo BitrotAlgorithm, shardSize int64) io.WriteCloser {
+	return &wholeBitrotWriter{disk, volume, filePath, shardSize, algo.New()}
 }
 
 // Implementation to verify bitrot for the whole file.

cmd/bitrot.go

@@ -120,7 +120,7 @@ func newBitrotWriter(disk StorageAPI, volume, filePath string, length int64, alg
 	if algo == HighwayHash256S {
 		return newStreamingBitrotWriter(disk, volume, filePath, length, algo, shardSize)
 	}
-	return newWholeBitrotWriter(disk, volume, filePath, length, algo, shardSize)
+	return newWholeBitrotWriter(disk, volume, filePath, algo, shardSize)
 }
 
 func newBitrotReader(disk StorageAPI, bucket string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) io.ReaderAt {

cmd/bucket-handlers.go

@@ -263,16 +263,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	var s3Error APIErrorCode
-	if s3Error = checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, ""); s3Error != ErrNone {
-		// In the event access is denied, a 200 response should still be returned
-		// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
-		if s3Error != ErrAccessDenied {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
-
 	// Content-Length is required and should be non-zero
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if r.ContentLength <= 0 {
@@ -324,37 +314,32 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		deleteObject = api.CacheAPI().DeleteObject
 	}
 
-	var dErrs = make([]error, len(deleteObjects.Objects))
+	var dErrs = make([]APIErrorCode, len(deleteObjects.Objects))
 	for index, object := range deleteObjects.Objects {
-		// If the request is denied access, each item
-		// should be marked as 'AccessDenied'
-		if s3Error == ErrAccessDenied {
-			dErrs[index] = PrefixAccessDenied{
-				Bucket: bucket,
-				Object: object.ObjectName,
+		if dErrs[index] = checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName); dErrs[index] != ErrNone {
+			if dErrs[index] == ErrSignatureDoesNotMatch || dErrs[index] == ErrInvalidAccessKeyID {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(dErrs[index]), r.URL, guessIsBrowserReq(r))
+				return
 			}
 			continue
 		}
-		dErrs[index] = deleteObject(ctx, bucket, object.ObjectName)
+		err := deleteObject(ctx, bucket, object.ObjectName)
+		if err != nil {
+			dErrs[index] = toAPIErrorCode(ctx, err)
+		}
 	}
 
 	// Collect deleted objects and errors if any.
 	var deletedObjects []ObjectIdentifier
 	var deleteErrors []DeleteError
-	for index, err := range dErrs {
+	for index, errCode := range dErrs {
 		object := deleteObjects.Objects[index]
 		// Success deleted objects are collected separately.
-		if err == nil {
+		if errCode == ErrNone || errCode == ErrNoSuchKey {
 			deletedObjects = append(deletedObjects, object)
 			continue
 		}
-		if _, ok := err.(ObjectNotFound); ok {
-			// If the object is not found it should be
-			// accounted as deleted as per S3 spec.
-			deletedObjects = append(deletedObjects, object)
-			continue
-		}
-		apiErr := toAPIError(ctx, err)
+		apiErr := getAPIError(errCode)
 		// Error during delete should be collected separately.
 		deleteErrors = append(deleteErrors, DeleteError{
 			Code:    apiErr.Code,

cmd/bucket-policy-handlers.go

@@ -71,7 +71,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 	// Error out if Content-Length is beyond allowed size.
 	if r.ContentLength > maxBucketPolicySize {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPolicyTooLarge), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/config-common.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 
 	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/minio/minio/cmd/logger"
@@ -105,15 +106,29 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
 
 // watchConfigEtcd - watches for changes on `configFile` on etcd and loads them.
 func watchConfigEtcd(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
-	defer cancel()
-	for watchResp := range globalEtcdClient.Watch(ctx, configFile) {
+	for {
+		watchCh := globalEtcdClient.Watch(context.Background(), iamConfigPrefix)
+		select {
+		case <-GlobalServiceDoneCh:
+			return
+		case watchResp, ok := <-watchCh:
+			if !ok {
+				time.Sleep(1 * time.Second)
+				continue
+			}
+			if err := watchResp.Err(); err != nil {
+				logger.LogIf(context.Background(), err)
+				// log and retry.
+				time.Sleep(1 * time.Second)
+				continue
+			}
 			for _, event := range watchResp.Events {
 				if event.IsModify() || event.IsCreate() {
 					loadCfgFn(objAPI)
 				}
 			}
 		}
+	}
 }
 
 func checkConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {

cmd/disk-cache.go

@@ -375,7 +375,7 @@ func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 // Returns function "listDir" of the type listDirFunc.
 // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
 // disks - list of fsObjects
-func listDirCacheFactory(isLeaf IsLeafFunc, disks []*cacheFSObjects) ListDirFunc {
+func listDirCacheFactory(isLeaf func(string, string) bool, disks []*cacheFSObjects) ListDirFunc {
 	listCacheDirs := func(bucket, prefixDir, prefixEntry string) (dirs []string) {
 		var entries []string
 		for _, disk := range disks {
@@ -391,6 +391,12 @@ func listDirCacheFactory(isLeaf IsLeafFunc, disks []*cacheFSObjects) ListDirFunc
 				continue
 			}
 
+			for i := range entries {
+				if isLeaf(bucket, entries[i]) {
+					entries[i] = strings.TrimSuffix(entries[i], slashSeparator)
+				}
+			}
+
 			// Filter entries that have the prefix prefixEntry.
 			entries = filterMatchingPrefix(entries, prefixEntry)
 			dirs = append(dirs, entries...)
@@ -399,7 +405,7 @@ func listDirCacheFactory(isLeaf IsLeafFunc, disks []*cacheFSObjects) ListDirFunc
 	}
 
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string) {
 		cacheEntries := listCacheDirs(bucket, prefixDir, prefixEntry)
 		for _, entry := range cacheEntries {
 			// Find elements in entries which are not in mergedEntries
@@ -411,7 +417,7 @@ func listDirCacheFactory(isLeaf IsLeafFunc, disks []*cacheFSObjects) ListDirFunc
 			mergedEntries = append(mergedEntries, entry)
 			sort.Strings(mergedEntries)
 		}
-		return mergedEntries, false
+		return mergedEntries
 	}
 	return listDir
 }
@@ -430,25 +436,16 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 	walkResultCh, endWalkCh := c.listPool.Release(listParams{bucket, recursive, marker, prefix})
 	if walkResultCh == nil {
 		endWalkCh = make(chan struct{})
-		isLeaf := func(bucket, object string) bool {
+
+		listDir := listDirCacheFactory(func(bucket, object string) bool {
 			fs, err := c.cache.getCacheFS(ctx, bucket, object)
 			if err != nil {
 				return false
 			}
 			_, err = fs.getObjectInfo(ctx, bucket, object)
 			return err == nil
-		}
-
-		isLeafDir := func(bucket, object string) bool {
-			fs, err := c.cache.getCacheFS(ctx, bucket, object)
-			if err != nil {
-				return false
-			}
-			return fs.isObjectDir(bucket, object)
-		}
-
-		listDir := listDirCacheFactory(isLeaf, c.cache.cfs)
-		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
+		}, c.cache.cfs)
+		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, endWalkCh)
 	}
 
 	for i := 0; i < maxKeys; {
@@ -458,10 +455,6 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 			eof = true
 			break
 		}
-		// For any walk error return right away.
-		if walkResult.err != nil {
-			return result, toObjectErr(walkResult.err, bucket, prefix)
-		}
 
 		entry := walkResult.entry
 		var objInfo ObjectInfo

cmd/endpoint-ellipses_test.go

@@ -44,7 +44,6 @@ func TestCreateServerEndpoints(t *testing.T) {
 		{":9000", []string{"/export1{1...32}", "/export1{1...32}"}, false},
 		// Same host cannot export same disk on two ports - special case localhost.
 		{":9001", []string{"http://localhost:900{1...2}/export{1...64}"}, false},
-
 		// Valid inputs.
 		{":9000", []string{"/export1"}, true},
 		{":9000", []string{"/export1", "/export2", "/export3", "/export4"}, true},

cmd/endpoint.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/url"
@@ -26,7 +27,10 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"time"
 
+	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/cli"
 	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/cpu"
@@ -44,6 +48,8 @@ const (
 
 	// URLEndpointType - URL style endpoint type enum.
 	URLEndpointType
+
+	retryInterval = 5 // In Seconds.
 )
 
 // Endpoint - any type of endpoint.
@@ -51,6 +57,7 @@ type Endpoint struct {
 	*url.URL
 	IsLocal  bool
 	SetIndex int
+	HostName string
 }
 
 func (endpoint Endpoint) String() string {
@@ -75,6 +82,19 @@ func (endpoint Endpoint) IsHTTPS() bool {
 	return endpoint.Scheme == "https"
 }
 
+// UpdateIsLocal - resolves the host and updates if it is local or not.
+func (endpoint *Endpoint) UpdateIsLocal() error {
+	if !endpoint.IsLocal {
+		isLocal, err := isLocalHost(endpoint.HostName)
+		if err != nil {
+			return err
+		}
+		endpoint.IsLocal = isLocal
+
+	}
+	return nil
+}
+
 // NewEndpoint - returns new endpoint based on given arguments.
 func NewEndpoint(arg string) (ep Endpoint, e error) {
 	// isEmptyPath - check whether given path is not empty.
@@ -87,6 +107,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 	}
 
 	var isLocal bool
+	var host string
 	u, err := url.Parse(arg)
 	if err == nil && u.Host != "" {
 		// URL style of endpoint.
@@ -98,7 +119,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 			return ep, fmt.Errorf("invalid URL endpoint format")
 		}
 
-		var host, port string
+		var port string
 		host, port, err = net.SplitHostPort(u.Host)
 		if err != nil {
 			if !strings.Contains(err.Error(), "missing port in address") {
@@ -150,10 +171,6 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 			}
 		}
 
-		isLocal, err = isLocalHost(host)
-		if err != nil {
-			return ep, err
-		}
 	} else {
 		// Only check if the arg is an ip address and ask for scheme since its absent.
 		// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
@@ -168,6 +185,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 	return Endpoint{
 		URL:      u,
 		IsLocal:  isLocal,
+		HostName: host,
 	}, nil
 }
 
@@ -200,6 +218,74 @@ func (endpoints EndpointList) GetString(i int) string {
 	return endpoints[i].String()
 }
 
+// UpdateIsLocal - resolves the host and discovers the local host.
+func (endpoints EndpointList) UpdateIsLocal() error {
+	var epsResolved int
+	var foundLocal bool
+	resolvedList := make([]bool, len(endpoints))
+	// Mark the starting time
+	startTime := time.Now()
+	keepAliveTicker := time.NewTicker(retryInterval * time.Second)
+	defer keepAliveTicker.Stop()
+	for {
+		// Break if the local endpoint is found already. Or all the endpoints are resolved.
+		if foundLocal || (epsResolved == len(endpoints)) {
+			break
+		}
+		// Retry infinitely on Kubernetes and Docker swarm.
+		// This is needed as the remote hosts are sometime
+		// not available immediately.
+		select {
+		case <-globalOSSignalCh:
+			return fmt.Errorf("The endpoint resolution got interrupted")
+		default:
+			for i, resolved := range resolvedList {
+				if resolved {
+					continue
+				}
+
+				// return err if not Docker or Kubernetes
+				// We use IsDocker() method to check for Docker Swarm environment
+				// as there is no reliable way to clearly identify Swarm from
+				// Docker environment.
+				isLocal, err := isLocalHost(endpoints[i].HostName)
+				if err != nil {
+					if !IsDocker() && !IsKubernetes() {
+						return err
+					}
+					// time elapsed
+					timeElapsed := time.Since(startTime)
+					// log error only if more than 1s elapsed
+					if timeElapsed > time.Second {
+						// log the message to console about the host not being
+						// resolveable.
+						reqInfo := (&logger.ReqInfo{}).AppendTags("host", endpoints[i].HostName)
+						reqInfo.AppendTags("elapsedTime", humanize.RelTime(startTime, startTime.Add(timeElapsed), "elapsed", ""))
+						ctx := logger.SetReqInfo(context.Background(), reqInfo)
+						logger.LogIf(ctx, err)
+					}
+				} else {
+					resolvedList[i] = true
+					endpoints[i].IsLocal = isLocal
+					epsResolved++
+					if !foundLocal {
+						foundLocal = isLocal
+					}
+				}
+			}
+
+			// Wait for the tick, if the there exist a local endpoint in discovery.
+			// Non docker/kubernetes environment does not need to wait.
+			if !foundLocal && (IsDocker() && IsKubernetes()) {
+				<-keepAliveTicker.C
+			}
+		}
+	}
+
+	return nil
+
+}
+
 // localEndpointsMemUsage - returns ServerMemUsageInfo for only the
 // local endpoints from given list of endpoints
 func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
@@ -302,9 +388,30 @@ func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
 		uniqueArgs.Add(arg)
 		endpoints = append(endpoints, endpoint)
 	}
+
 	return endpoints, nil
 }
 
+func checkEndpointsSubOptimal(ctx *cli.Context, setupType SetupType, endpoints EndpointList) (err error) {
+	// Validate sub optimal ordering only for distributed setup.
+	if setupType != DistXLSetupType {
+		return nil
+	}
+	var endpointOrder int
+	err = fmt.Errorf("Too many disk args are local, input is in sub-optimal order. Please review input args: %s", ctx.Args())
+	for _, endpoint := range endpoints {
+		if endpoint.IsLocal {
+			endpointOrder++
+		} else {
+			endpointOrder--
+		}
+		if endpointOrder >= 2 {
+			return err
+		}
+	}
+	return nil
+}
+
 // Checks if there are any cross device mounts.
 func checkCrossDeviceMounts(endpoints EndpointList) (err error) {
 	var absPaths []string
@@ -341,6 +448,9 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		if err != nil {
 			return serverAddr, endpoints, setupType, err
 		}
+		if err := endpoint.UpdateIsLocal(); err != nil {
+			return serverAddr, endpoints, setupType, err
+		}
 		if endpoint.Type() != PathEndpointType {
 			return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
 		}
@@ -381,6 +491,10 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		return serverAddr, endpoints, setupType, nil
 	}
 
+	if err := endpoints.UpdateIsLocal(); err != nil {
+		return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
+	}
+
 	// Here all endpoints are URL style.
 	endpointPathSet := set.NewStringSet()
 	localEndpointCount := 0

cmd/endpoint_test.go

@@ -17,13 +17,53 @@
 package cmd
 
 import (
+	"flag"
 	"fmt"
 	"net/url"
 	"reflect"
 	"strings"
 	"testing"
+
+	"github.com/minio/cli"
 )
 
+func TestSubOptimalEndpointInput(t *testing.T) {
+	args1 := []string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}
+	args2 := []string{"http://example.org/d1", "http://example.com/d1", "http://example.net/d1", "http://example.edu/d1"}
+
+	tests := []struct {
+		setupType SetupType
+		ctx       *cli.Context
+		endpoints EndpointList
+		isErr     bool
+	}{
+		{
+			setupType: DistXLSetupType,
+			ctx:       cli.NewContext(cli.NewApp(), flag.NewFlagSet("", flag.ContinueOnError), nil),
+			endpoints: mustGetNewEndpointList(args1...),
+			isErr:     false,
+		},
+		{
+			setupType: DistXLSetupType,
+			ctx:       cli.NewContext(cli.NewApp(), flag.NewFlagSet("", flag.ContinueOnError), nil),
+			endpoints: mustGetNewEndpointList(args2...),
+			isErr:     false,
+		},
+	}
+	for i, test := range tests {
+		test := test
+		t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
+			err := checkEndpointsSubOptimal(test.ctx, test.setupType, test.endpoints)
+			if test.isErr && err == nil {
+				t.Error("expected err but found nil")
+			}
+			if !test.isErr && err != nil {
+				t.Errorf("expected err nil but found an err %s", err)
+			}
+		})
+	}
+}
+
 func TestNewEndpoint(t *testing.T) {
 	u1, _ := url.Parse("http://localhost/path")
 	u2, _ := url.Parse("https://example.org/path")
@@ -49,11 +89,11 @@ func TestNewEndpoint(t *testing.T) {
 		{"http:path", Endpoint{URL: &url.URL{Path: "http:path"}, IsLocal: true}, PathEndpointType, nil},
 		{"http:/path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
 		{"http:///path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
-		{"http://localhost/path", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
-		{"http://localhost/path//", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
-		{"https://example.org/path", Endpoint{URL: u2}, URLEndpointType, nil},
-		{"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true}, URLEndpointType, nil},
-		{"http://192.168.253.200/path", Endpoint{URL: u4}, URLEndpointType, nil},
+		{"http://localhost/path", Endpoint{URL: u1, IsLocal: true, HostName: "localhost"}, URLEndpointType, nil},
+		{"http://localhost/path//", Endpoint{URL: u1, IsLocal: true, HostName: "localhost"}, URLEndpointType, nil},
+		{"https://example.org/path", Endpoint{URL: u2, IsLocal: false, HostName: "example.org"}, URLEndpointType, nil},
+		{"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true, HostName: "127.0.0.1"}, URLEndpointType, nil},
+		{"http://192.168.253.200/path", Endpoint{URL: u4, IsLocal: false, HostName: "192.168.253.200"}, URLEndpointType, nil},
 		{"", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
 		{"/", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
 		{`\`, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
@@ -71,6 +111,10 @@ func TestNewEndpoint(t *testing.T) {
 
 	for _, testCase := range testCases {
 		endpoint, err := NewEndpoint(testCase.arg)
+		if err == nil {
+			err = endpoint.UpdateIsLocal()
+		}
+
 		if testCase.expectedErr == nil {
 			if err != nil {
 				t.Fatalf("error: expected = <nil>, got = %v", err)
@@ -261,46 +305,46 @@ func TestCreateEndpoints(t *testing.T) {
 
 		// DistXL type
 		{"127.0.0.1:10000", [][]string{{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", EndpointList{
-			Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0]},
-			Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1]},
-			Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2]},
-			Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3]},
+			Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0], HostName: nonLoopBackIP},
+			Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1], HostName: nonLoopBackIP},
+			Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2], HostName: "example.org"},
+			Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3], HostName: "example.com"},
 		}, DistXLSetupType, nil},
 
 		{"127.0.0.1:10000", [][]string{{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", EndpointList{
-			Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0]},
-			Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1]},
-			Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2]},
-			Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3]},
+			Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0], HostName: nonLoopBackIP},
+			Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1], HostName: nonLoopBackIP},
+			Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2], HostName: "example.org"},
+			Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3], HostName: "example.com"},
 		}, DistXLSetupType, nil},
 
 		{":80", [][]string{{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}}, ":80", EndpointList{
-			Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0]},
-			Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1]},
-			Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2]},
-			Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3]},
+			Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0], HostName: nonLoopBackIP},
+			Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1], HostName: "example.org"},
+			Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2], HostName: "example.com"},
+			Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3], HostName: "example.net"},
 		}, DistXLSetupType, nil},
 
 		{":9000", [][]string{{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}}, ":9000", EndpointList{
-			Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0]},
-			Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1]},
-			Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2]},
-			Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3]},
+			Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0], HostName: nonLoopBackIP},
+			Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1], HostName: "example.org"},
+			Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2], HostName: "example.com"},
+			Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3], HostName: "example.net"},
 		}, DistXLSetupType, nil},
 
 		{":9000", [][]string{{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}}, ":9000", EndpointList{
-			Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0]},
-			Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1]},
-			Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2]},
-			Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3]},
+			Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0], HostName: nonLoopBackIP},
+			Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1], HostName: nonLoopBackIP},
+			Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2], HostName: nonLoopBackIP},
+			Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3], HostName: nonLoopBackIP},
 		}, DistXLSetupType, nil},
 
 		// DistXL Setup using only local host.
 		{":9003", [][]string{{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}}, ":9003", EndpointList{
-			Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0]},
-			Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1]},
-			Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2]},
-			Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3]},
+			Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0], HostName: "localhost"},
+			Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1], HostName: "localhost"},
+			Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2], HostName: "127.0.0.1"},
+			Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3], HostName: nonLoopBackIP},
 		}, DistXLSetupType, nil},
 	}
 
@@ -357,6 +401,11 @@ func TestGetLocalPeer(t *testing.T) {
 
 	for i, testCase := range testCases {
 		endpoints, _ := NewEndpointList(testCase.endpointArgs...)
+		if !endpoints[0].IsLocal {
+			if err := endpoints.UpdateIsLocal(); err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		}
 		remotePeer := GetLocalPeer(endpoints)
 		if remotePeer != testCase.expectedResult {
 			t.Fatalf("Test %d: expected: %v, got: %v", i+1, testCase.expectedResult, remotePeer)
@@ -384,6 +433,11 @@ func TestGetRemotePeers(t *testing.T) {
 
 	for _, testCase := range testCases {
 		endpoints, _ := NewEndpointList(testCase.endpointArgs...)
+		if !endpoints[0].IsLocal {
+			if err := endpoints.UpdateIsLocal(); err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		}
 		remotePeers := GetRemotePeers(endpoints)
 		if !reflect.DeepEqual(remotePeers, testCase.expectedResult) {
 			t.Fatalf("expected: %v, got: %v", testCase.expectedResult, remotePeers)

cmd/erasure.go

@@ -103,6 +103,9 @@ func (e *Erasure) ShardFileSize(totalLength int64) int64 {
 	if totalLength == 0 {
 		return 0
 	}
+	if totalLength == -1 {
+		return -1
+	}
 	numShards := totalLength / e.blockSize
 	lastBlockSize := totalLength % int64(e.blockSize)
 	lastShardSize := ceilFrac(lastBlockSize, int64(e.dataBlocks))

cmd/fs-v1-helpers.go

@@ -206,6 +206,9 @@ func osErrToFSFileErr(err error) error {
 	if isSysErrPathNotFound(err) {
 		return errFileNotFound
 	}
+	if isSysErrTooManyFiles(err) {
+		return errTooManyOpenFiles
+	}
 	return err
 }
 

cmd/fs-v1.go

@@ -1001,17 +1001,17 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) er
 // Returns function "listDir" of the type listDirFunc.
 // isLeaf - is used by listDir function to check if an entry
 // is a leaf or non-leaf entry.
-func (fs *FSObjects) listDirFactory(isLeaf IsLeafFunc) ListDirFunc {
+func (fs *FSObjects) listDirFactory() ListDirFunc {
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string) {
 		var err error
 		entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 		if err != nil && err != errFileNotFound {
 			logger.LogIf(context.Background(), err)
 			return
 		}
-		entries, delayIsLeaf = filterListEntries(bucket, prefixDir, entries, prefixEntry, isLeaf)
-		return entries, delayIsLeaf
+		sort.Strings(entries)
+		return filterMatchingPrefix(entries, prefixEntry)
 	}
 
 	// Return list factory instance.
@@ -1097,22 +1097,8 @@ func (fs *FSObjects) getObjectETag(ctx context.Context, bucket, entry string, lo
 // ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool
 // state for future re-entrant list requests.
 func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-	isLeaf := func(bucket, object string) bool {
-		// bucket argument is unused as we don't need to StatFile
-		// to figure if it's a file, just need to check that the
-		// object string does not end with "/".
-		return !hasSuffix(object, slashSeparator)
-	}
-	// Return true if the specified object is an empty directory
-	isLeafDir := func(bucket, object string) bool {
-		if !hasSuffix(object, slashSeparator) {
-			return false
-		}
-		return fs.isObjectDir(bucket, object)
-	}
-	listDir := fs.listDirFactory(isLeaf)
-
-	return listObjects(ctx, fs, bucket, prefix, marker, delimiter, maxKeys, fs.listPool, isLeaf, isLeafDir, listDir, fs.getObjectInfo, fs.getObjectInfo)
+	return listObjects(ctx, fs, bucket, prefix, marker, delimiter, maxKeys, fs.listPool,
+		fs.listDirFactory(), fs.getObjectInfo, fs.getObjectInfo)
 }
 
 // ReloadFormat - no-op for fs, Valid only for XL.

cmd/gateway-common.go

@@ -44,8 +44,8 @@ var (
 	// ListObjects function alias.
 	ListObjects = listObjects
 
-	// FilterListEntries function alias.
-	FilterListEntries = filterListEntries
+	// FilterMatchingPrefix function alias.
+	FilterMatchingPrefix = filterMatchingPrefix
 
 	// IsStringEqual is string equal.
 	IsStringEqual = isStringEqual

cmd/gateway/gcs/gateway-gcs.go

@@ -915,15 +915,12 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r
 	}
 
 	// Close the object writer upon success.
-	w.Close()
-
-	attrs, err := object.Attrs(ctx)
-	if err != nil {
+	if err := w.Close(); err != nil {
 		logger.LogIf(ctx, err)
 		return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
 	}
 
-	return fromGCSAttrsToObjectInfo(attrs), nil
+	return fromGCSAttrsToObjectInfo(w.Attrs()), nil
 }
 
 // CopyObject - Copies a blob from source container to destination container.
@@ -1093,7 +1090,10 @@ func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key strin
 		return minio.PartInfo{}, gcsToObjectError(err, bucket, key)
 	}
 	// Make sure to close the object writer upon success.
-	w.Close()
+	if err := w.Close(); err != nil {
+		logger.LogIf(ctx, err)
+		return minio.PartInfo{}, gcsToObjectError(err, bucket, key)
+	}
 	return minio.PartInfo{
 		PartNumber:   partNumber,
 		ETag:         etag,

cmd/gateway/hdfs/gateway-hdfs.go

@@ -29,8 +29,8 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/colinmarc/hdfs/v2"
 	"github.com/minio/cli"
+	"github.com/minio/hdfs/v3"
 	"github.com/minio/minio-go/pkg/s3utils"
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/cmd/logger"
@@ -287,24 +287,9 @@ func (n *hdfsObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketIn
 	return buckets, nil
 }
 
-func (n *hdfsObjects) isObjectDir(bucket, object string) bool {
-	f, err := n.clnt.Open(minio.PathJoin(hdfsSeparator, bucket, object))
-	if err != nil {
-		return false
-	}
-	defer f.Close()
-
-	entries, err := f.Readdir(1)
-	if err != nil {
-		return false
-	}
-
-	return len(entries) == 0
-}
-
-func (n *hdfsObjects) listDirFactory(isLeaf minio.IsLeafFunc) minio.ListDirFunc {
+func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string) {
 		f, err := n.clnt.Open(minio.PathJoin(hdfsSeparator, bucket, prefixDir))
 		if err != nil {
 			if os.IsNotExist(err) {
@@ -327,8 +312,7 @@ func (n *hdfsObjects) listDirFactory(isLeaf minio.IsLeafFunc) minio.ListDirFunc
 			}
 		}
 		fis = nil
-		entries, delayIsLeaf = minio.FilterListEntries(bucket, prefixDir, entries, prefixEntry, isLeaf)
-		return entries, delayIsLeaf
+		return minio.FilterMatchingPrefix(entries, prefixEntry)
 	}
 
 	// Return list factory instance.
@@ -337,27 +321,11 @@ func (n *hdfsObjects) listDirFactory(isLeaf minio.IsLeafFunc) minio.ListDirFunc
 
 // ListObjects lists all blobs in HDFS bucket filtered by prefix.
 func (n *hdfsObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
-	isLeaf := func(bucket, object string) bool {
-		// bucket argument is unused as we don't need to StatFile
-		// to figure if it's a file, just need to check that the
-		// object string does not end with "/".
-		return !strings.HasSuffix(object, hdfsSeparator)
-	}
-
-	// Return true if the specified object is an empty directory
-	isLeafDir := func(bucket, object string) bool {
-		if !strings.HasSuffix(object, hdfsSeparator) {
-			return false
-		}
-		return n.isObjectDir(bucket, object)
-	}
-	listDir := n.listDirFactory(isLeaf)
-
 	getObjectInfo := func(ctx context.Context, bucket, entry string) (minio.ObjectInfo, error) {
 		return n.GetObjectInfo(ctx, bucket, entry, minio.ObjectOptions{})
 	}
 
-	return minio.ListObjects(ctx, n, bucket, prefix, marker, delimiter, maxKeys, n.listPool, isLeaf, isLeafDir, listDir, getObjectInfo, getObjectInfo)
+	return minio.ListObjects(ctx, n, bucket, prefix, marker, delimiter, maxKeys, n.listPool, n.listDirFactory(), getObjectInfo, getObjectInfo)
 }
 
 // Check if the given error corresponds to ENOTEMPTY for unix
@@ -477,13 +445,15 @@ func (n *hdfsObjects) GetObject(ctx context.Context, bucket, key string, startOf
 	if err != nil {
 		return hdfsToObjectErr(ctx, err, bucket, key)
 	}
-	if _, err = rd.Seek(startOffset, io.SeekStart); err != nil {
-		return hdfsToObjectErr(ctx, err, bucket, key)
+	defer rd.Close()
+	_, err = io.Copy(writer, io.NewSectionReader(rd, startOffset, length))
+	if err == io.ErrClosedPipe {
+		// hdfs library doesn't send EOF correctly, so io.Copy attempts
+		// to write which returns io.ErrClosedPipe - just ignore
+		// this for now.
+		err = nil
 	}
-	if _, err = io.Copy(writer, io.LimitReader(rd, length)); err != nil {
 	return hdfsToObjectErr(ctx, err, bucket, key)
-	}
-	return nil
 }
 
 // GetObjectInfo reads object info and replies back ObjectInfo.
@@ -528,17 +498,19 @@ func (n *hdfsObjects) PutObject(ctx context.Context, bucket string, object strin
 			return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
 		}
 		defer n.deleteObject(minio.PathJoin(hdfsSeparator, minioMetaTmpBucket), tmpname)
-		defer w.Close()
 		if _, err = io.Copy(w, r); err != nil {
+			w.Close()
 			return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
 		}
 		dir := path.Dir(name)
 		if dir != "" {
 			if err = n.clnt.MkdirAll(dir, os.FileMode(0755)); err != nil {
+				w.Close()
 				n.deleteObject(minio.PathJoin(hdfsSeparator, bucket), dir)
 				return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
 			}
 		}
+		w.Close()
 		if err = n.clnt.Rename(tmpname, name); err != nil {
 			return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
 		}

cmd/http/conn_bug_21133.go

@@ -1,48 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package http
-
-import (
-	"net"
-	"sync/atomic"
-	"time"
-)
-
-// QuirkConn - similar to golang net.Conn struct, but contains a workaround of the
-// following the go bug reported here https://github.com/golang/go/issues/21133.
-// Once the bug will be fixed, we can remove this structure and replaces it with
-// the standard net.Conn
-type QuirkConn struct {
-	net.Conn
-	hadReadDeadlineInPast int32 // atomic
-}
-
-// SetReadDeadline - implements a workaround of SetReadDeadline go bug
-func (q *QuirkConn) SetReadDeadline(t time.Time) error {
-	inPast := int32(0)
-	if t.Before(time.Now()) {
-		inPast = 1
-	}
-	atomic.StoreInt32(&q.hadReadDeadlineInPast, inPast)
-	return q.Conn.SetReadDeadline(t)
-}
-
-// canSetReadDeadline - returns if it is safe to set a new
-// read deadline without triggering golang/go#21133 issue.
-func (q *QuirkConn) canSetReadDeadline() bool {
-	return atomic.LoadInt32(&q.hadReadDeadlineInPast) != 1
-}

cmd/http/deadlineconn.go

@@ -23,7 +23,7 @@ import (
 
 // DeadlineConn - is a generic stream-oriented network connection supporting buffered reader and read/write timeout.
 type DeadlineConn struct {
-	QuirkConn
+	net.Conn
 	readTimeout            time.Duration // sets the read timeout in the connection.
 	writeTimeout           time.Duration // sets the write timeout in the connection.
 	updateBytesReadFunc    func(int)     // function to be called to update bytes read.
@@ -32,7 +32,7 @@ type DeadlineConn struct {
 
 // Sets read timeout
 func (c *DeadlineConn) setReadTimeout() {
-	if c.readTimeout != 0 && c.canSetReadDeadline() {
+	if c.readTimeout != 0 {
 		c.SetReadDeadline(time.Now().UTC().Add(c.readTimeout))
 	}
 }
@@ -68,7 +68,7 @@ func (c *DeadlineConn) Write(b []byte) (n int, err error) {
 // newDeadlineConn - creates a new connection object wrapping net.Conn with deadlines.
 func newDeadlineConn(c net.Conn, readTimeout, writeTimeout time.Duration, updateBytesReadFunc, updateBytesWrittenFunc func(int)) *DeadlineConn {
 	return &DeadlineConn{
-		QuirkConn:              QuirkConn{Conn: c},
+		Conn:                   c,
 		readTimeout:            readTimeout,
 		writeTimeout:           writeTimeout,
 		updateBytesReadFunc:    updateBytesReadFunc,

cmd/http/timeoutconn.go

@@ -1,62 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package http
-
-import (
-	"net"
-	"time"
-)
-
-// TimeoutConn - is wrapped net.Conn with read/write timeouts.
-type TimeoutConn struct {
-	QuirkConn
-	readTimeout  time.Duration
-	writeTimeout time.Duration
-}
-
-func (c *TimeoutConn) setReadTimeout() {
-	if c.readTimeout != 0 && c.canSetReadDeadline() {
-		c.SetReadDeadline(time.Now().UTC().Add(c.readTimeout))
-	}
-}
-
-func (c *TimeoutConn) setWriteTimeout() {
-	if c.writeTimeout != 0 {
-		c.SetWriteDeadline(time.Now().UTC().Add(c.writeTimeout))
-	}
-}
-
-// Read - reads data from the connection with timeout.
-func (c *TimeoutConn) Read(b []byte) (n int, err error) {
-	c.setReadTimeout()
-	return c.Conn.Read(b)
-}
-
-// Write - writes data to the connection with timeout.
-func (c *TimeoutConn) Write(b []byte) (n int, err error) {
-	c.setWriteTimeout()
-	return c.Conn.Write(b)
-}
-
-// NewTimeoutConn - creates a new timeout connection.
-func NewTimeoutConn(c net.Conn, readTimeout, writeTimeout time.Duration) *TimeoutConn {
-	return &TimeoutConn{
-		QuirkConn:    QuirkConn{Conn: c},
-		readTimeout:  readTimeout,
-		writeTimeout: writeTimeout,
-	}
-}

cmd/iam.go

@@ -62,6 +62,9 @@ type IAMSys struct {
 
 // Load - loads iam subsystem
 func (sys *IAMSys) Load(objAPI ObjectLayer) error {
+	if globalEtcdClient != nil {
+		return sys.refreshEtcd()
+	}
 	return sys.refresh(objAPI)
 }
 
@@ -71,6 +74,36 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
 		return errInvalidArgument
 	}
 
+	if globalEtcdClient != nil {
+		defer func() {
+			go func() {
+				// Refresh IAMSys with etcd watch.
+				for {
+					watchCh := globalEtcdClient.Watch(context.Background(), iamConfigPrefix)
+					select {
+					case <-GlobalServiceDoneCh:
+						return
+					case watchResp, ok := <-watchCh:
+						if !ok {
+							time.Sleep(1 * time.Second)
+							continue
+						}
+						if err := watchResp.Err(); err != nil {
+							logger.LogIf(context.Background(), err)
+							// log and retry.
+							time.Sleep(1 * time.Second)
+							continue
+						}
+						for _, event := range watchResp.Events {
+							if event.IsModify() || event.IsCreate() || event.Type == etcd.EventTypeDelete {
+								sys.refreshEtcd()
+							}
+						}
+					}
+				}
+			}()
+		}()
+	} else {
 		defer func() {
 			// Refresh IAMSys in background.
 			go func() {
@@ -86,6 +119,7 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
 				}
 			}()
 		}()
+	}
 
 	doneCh := make(chan struct{})
 	defer close(doneCh)
@@ -95,6 +129,9 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
 	//  - Read quorum is lost just after the initialization
 	//    of the object layer.
 	for range newRetryTimerSimple(doneCh) {
+		if globalEtcdClient != nil {
+			return sys.refreshEtcd()
+		}
 		// Load IAMSys once during boot.
 		if err := sys.refresh(objAPI); err != nil {
 			if err == errDiskNotFound ||
@@ -697,13 +734,11 @@ func setDefaultCannedPolicies(policies map[string]iampolicy.Policy) {
 	}
 }
 
-// Refresh IAMSys.
-func (sys *IAMSys) refresh(objAPI ObjectLayer) error {
+func (sys *IAMSys) refreshEtcd() error {
 	iamUsersMap := make(map[string]auth.Credentials)
 	iamPolicyMap := make(map[string]string)
 	iamCannedPolicyMap := make(map[string]iampolicy.Policy)
 
-	if globalEtcdClient != nil {
 	if err := reloadEtcdPolicies(iamConfigPoliciesPrefix, iamCannedPolicyMap); err != nil {
 		return err
 	}
@@ -713,7 +748,26 @@ func (sys *IAMSys) refresh(objAPI ObjectLayer) error {
 	if err := reloadEtcdUsers(iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
 		return err
 	}
-	} else {
+
+	// Sets default canned policies, if none are set.
+	setDefaultCannedPolicies(iamCannedPolicyMap)
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	sys.iamUsersMap = iamUsersMap
+	sys.iamPolicyMap = iamPolicyMap
+	sys.iamCannedPolicyMap = iamCannedPolicyMap
+
+	return nil
+}
+
+// Refresh IAMSys.
+func (sys *IAMSys) refresh(objAPI ObjectLayer) error {
+	iamUsersMap := make(map[string]auth.Credentials)
+	iamPolicyMap := make(map[string]string)
+	iamCannedPolicyMap := make(map[string]iampolicy.Policy)
+
 	if err := reloadPolicies(objAPI, iamConfigPoliciesPrefix, iamCannedPolicyMap); err != nil {
 		return err
 	}
@@ -723,9 +777,8 @@ func (sys *IAMSys) refresh(objAPI ObjectLayer) error {
 	if err := reloadUsers(objAPI, iamConfigSTSPrefix, iamUsersMap, iamPolicyMap); err != nil {
 		return err
 	}
-	}
 
-	// Sets default canned policies, if none set.
+	// Sets default canned policies, if none are set.
 	setDefaultCannedPolicies(iamCannedPolicyMap)
 
 	sys.Lock()

cmd/logger/target/console/console.go

@@ -45,12 +45,13 @@ func (c *Target) Send(e interface{}) error {
 		return nil
 	}
 
-	trace := make([]string, len(entry.Trace.Source))
+	traceLength := len(entry.Trace.Source)
+	trace := make([]string, traceLength)
 
 	// Add a sequence number and formatting for each stack trace
 	// No formatting is required for the first entry
 	for i, element := range entry.Trace.Source {
-		trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
+		trace[i] = fmt.Sprintf("%8v: %s", traceLength-i, element)
 	}
 
 	tagString := ""

cmd/naughty-disk_test.go

@@ -111,11 +111,11 @@ func (d *naughtyDisk) DeleteVol(volume string) (err error) {
 	return d.disk.DeleteVol(volume)
 }
 
-func (d *naughtyDisk) ListDir(volume, path string, count int) (entries []string, err error) {
+func (d *naughtyDisk) ListDir(volume, path string, count int, leafFile string) (entries []string, err error) {
 	if err := d.calcError(); err != nil {
 		return []string{}, err
 	}
-	return d.disk.ListDir(volume, path, count)
+	return d.disk.ListDir(volume, path, count, leafFile)
 }
 
 func (d *naughtyDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {

cmd/net.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"net"
@@ -27,9 +26,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
-	"time"
 
-	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 )
@@ -102,42 +99,9 @@ func getHostIP(host string) (ipList set.StringSet, err error) {
 	var ips []net.IP
 
 	if ips, err = net.LookupIP(host); err != nil {
-		// return err if not Docker or Kubernetes
-		// We use IsDocker() method to check for Docker Swarm environment
-		// as there is no reliable way to clearly identify Swarm from
-		// Docker environment.
-		if !IsDocker() && !IsKubernetes() {
 		return ipList, err
 	}
 
-		// channel to indicate completion of host resolution
-		doneCh := make(chan struct{})
-		// Indicate retry routine to exit cleanly, upon this function return.
-		defer close(doneCh)
-		// Mark the starting time
-		startTime := time.Now()
-		// wait for hosts to resolve in exponentialbackoff manner
-		for range newRetryTimerSimple(doneCh) {
-			// Retry infinitely on Kubernetes and Docker swarm.
-			// This is needed as the remote hosts are sometime
-			// not available immediately.
-			if ips, err = net.LookupIP(host); err == nil {
-				break
-			}
-			// time elapsed
-			timeElapsed := time.Since(startTime)
-			// log error only if more than 1s elapsed
-			if timeElapsed > time.Second {
-				// log the message to console about the host not being
-				// resolveable.
-				reqInfo := (&logger.ReqInfo{}).AppendTags("host", host)
-				reqInfo.AppendTags("elapsedTime", humanize.RelTime(startTime, startTime.Add(timeElapsed), "elapsed", ""))
-				ctx := logger.SetReqInfo(context.Background(), reqInfo)
-				logger.LogIf(ctx, err)
-			}
-		}
-	}
-
 	ipList = set.NewStringSet()
 	for _, ip := range ips {
 		ipList.Add(ip.String())

cmd/object-api-common.go

@@ -116,7 +116,7 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string)
 		}
 
 		// If it's a directory, list and call delFunc() for each entry.
-		entries, err := storage.ListDir(volume, entryPath, -1)
+		entries, err := storage.ListDir(volume, entryPath, -1, "")
 		// If entryPath prefix never existed, safe to ignore.
 		if err == errFileNotFound {
 			return nil
@@ -162,7 +162,7 @@ func removeListenerConfig(ctx context.Context, objAPI ObjectLayer, bucket string
 	return objAPI.DeleteObject(ctx, minioMetaBucket, lcPath)
 }
 
-func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, delimiter string, maxKeys int, tpool *TreeWalkPool, isLeaf IsLeafFunc, isLeafDir IsLeafDirFunc, listDir ListDirFunc, getObjInfo func(context.Context, string, string) (ObjectInfo, error), getObjectInfoDirs ...func(context.Context, string, string) (ObjectInfo, error)) (loi ListObjectsInfo, err error) {
+func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, delimiter string, maxKeys int, tpool *TreeWalkPool, listDir ListDirFunc, getObjInfo func(context.Context, string, string) (ObjectInfo, error), getObjectInfoDirs ...func(context.Context, string, string) (ObjectInfo, error)) (loi ListObjectsInfo, err error) {
 	if err := checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, obj); err != nil {
 		return loi, err
 	}
@@ -203,7 +203,7 @@ func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, d
 	walkResultCh, endWalkCh := tpool.Release(listParams{bucket, recursive, marker, prefix})
 	if walkResultCh == nil {
 		endWalkCh = make(chan struct{})
-		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
+		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, endWalkCh)
 	}
 
 	var objInfos []ObjectInfo
@@ -218,14 +218,6 @@ func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, d
 			eof = true
 			break
 		}
-		// For any walk error return right away.
-		if walkResult.err != nil {
-			// File not found is a valid case.
-			if walkResult.err == errFileNotFound {
-				continue
-			}
-			return loi, toObjectErr(walkResult.err, bucket, prefix)
-		}
 
 		var objInfo ObjectInfo
 		var err error
@@ -235,6 +227,14 @@ func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, d
 				if err == nil {
 					break
 				}
+				if err == errFileNotFound {
+					err = nil
+					objInfo = ObjectInfo{
+						Bucket: bucket,
+						Name:   walkResult.entry,
+						IsDir:  true,
+					}
+				}
 			}
 		} else {
 			objInfo, err = getObjInfo(ctx, bucket, walkResult.entry)

cmd/object-api-errors.go

@@ -27,6 +27,22 @@ import (
 // underlying storage layer.
 func toObjectErr(err error, params ...string) error {
 	switch err {
+	case errDiskNotFound:
+		switch len(params) {
+		case 1:
+			err = BucketNotFound{Bucket: params[0]}
+		case 2:
+			err = ObjectNotFound{
+				Bucket: params[0],
+				Object: params[1],
+			}
+		case 3:
+			err = InvalidUploadID{
+				Bucket:   params[0],
+				Object:   params[1],
+				UploadID: params[2],
+			}
+		}
 	case errVolumeNotFound:
 		if len(params) >= 1 {
 			err = BucketNotFound{Bucket: params[0]}
@@ -41,6 +57,8 @@ func toObjectErr(err error, params ...string) error {
 		}
 	case errDiskFull:
 		err = StorageFull{}
+	case errTooManyOpenFiles:
+		err = SlowDown{}
 	case errFileAccessDenied:
 		if len(params) >= 2 {
 			err = PrefixAccessDenied{
@@ -63,11 +81,18 @@ func toObjectErr(err error, params ...string) error {
 			}
 		}
 	case errFileNotFound:
-		if len(params) >= 2 {
+		switch len(params) {
+		case 2:
 			err = ObjectNotFound{
 				Bucket: params[0],
 				Object: params[1],
 			}
+		case 3:
+			err = InvalidUploadID{
+				Bucket:   params[0],
+				Object:   params[1],
+				UploadID: params[2],
+			}
 		}
 	case errFileNameTooLong:
 		if len(params) >= 2 {
@@ -114,6 +139,13 @@ func (e StorageFull) Error() string {
 	return "Storage reached its minimum free disk threshold."
 }
 
+// SlowDown  too many file descriptors open or backend busy .
+type SlowDown struct{}
+
+func (e SlowDown) Error() string {
+	return "Please reduce your request rate"
+}
+
 // InsufficientReadQuorum storage cannot satisfy quorum for read operation.
 type InsufficientReadQuorum struct{}
 
@@ -321,6 +353,8 @@ func (e MalformedUploadID) Error() string {
 
 // InvalidUploadID invalid upload id.
 type InvalidUploadID struct {
+	Bucket   string
+	Object   string
 	UploadID string
 }
 

cmd/object-api-listobjects_test.go

@@ -35,8 +35,8 @@ func TestListObjects(t *testing.T) {
 }
 
 // Unit test for ListObjects in general.
-func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
-
+func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
+	t, _ := t1.(*testing.T)
 	testBuckets := []string{
 		// This bucket is used for testing ListObject operations.
 		"test-bucket-list-object",
@@ -66,7 +66,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 		{"obj0", "obj0", nil},
 		{"obj1", "obj1", nil},
 		{"obj2", "obj2", nil},
-		{"z-empty-dir/", "", nil},
 	}
 	for _, object := range testObjects {
 		md5Bytes := md5.Sum([]byte(object.content))
@@ -96,7 +95,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-1.
@@ -193,7 +191,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-10.
@@ -205,7 +202,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-11.
@@ -215,7 +211,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 			Objects: []ObjectInfo{
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-12.
@@ -224,7 +219,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-13.
@@ -238,7 +232,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-14.
@@ -255,7 +248,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-15.
@@ -270,7 +262,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-16.
@@ -284,7 +275,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
-				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-17.
@@ -535,7 +525,10 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 
 	for i, testCase := range testCases {
-		result, err := obj.ListObjects(context.Background(), testCase.bucketName, testCase.prefix, testCase.marker, testCase.delimeter, int(testCase.maxKeys))
+		testCase := testCase
+		t.Run(fmt.Sprintf("Test%d-%s", i+1, instanceType), func(t *testing.T) {
+			result, err := obj.ListObjects(context.Background(), testCase.bucketName,
+				testCase.prefix, testCase.marker, testCase.delimeter, int(testCase.maxKeys))
 			if err != nil && testCase.shouldPass {
 				t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
 			}
@@ -564,7 +557,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 					if testCase.result.Objects[j].Name != result.Objects[j].Name {
 						t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name)
 					}
-				//FIXME: we should always check for ETag
+					// FIXME: we should always check for ETag
 					if result.Objects[j].ETag == "" && !strings.HasSuffix(result.Objects[j].Name, slashSeparator) {
 						t.Errorf("Test %d: %s: Expected ETag to be not empty, but found empty instead (%v)", i+1, instanceType, result.Objects[j].Name)
 					}
@@ -585,11 +578,13 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 			}
 			// Take ListObject treeWalk go-routine to completion, if available in the treewalk pool.
 			if result.IsTruncated {
-			_, err = obj.ListObjects(context.Background(), testCase.bucketName, testCase.prefix, result.NextMarker, testCase.delimeter, 1000)
+				_, err = obj.ListObjects(context.Background(), testCase.bucketName,
+					testCase.prefix, result.NextMarker, testCase.delimeter, 1000)
 				if err != nil {
 					t.Fatal(err)
 				}
 			}
+		})
 	}
 }
 

cmd/object-api-multipart_test.go

@@ -228,8 +228,7 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [
 	}
 
 	// This causes quorum failure verify.
-	disks = disks[len(disks)-3:]
-	for _, disk := range disks {
+	for _, disk := range disks[len(disks)-3:] {
 		os.RemoveAll(disk)
 	}
 
@@ -240,9 +239,26 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [
 		t.Fatalf("Test %s: expected to fail but passed instead", instanceType)
 	}
 	// as majority of xl.json are not available, we expect uploadID to be not available.
-	expectedErr := InvalidUploadID{UploadID: testCase.uploadID}
-	if err.Error() != expectedErr.Error() {
-		t.Fatalf("Test %s: expected error %s, got %s instead.", instanceType, expectedErr, err)
+	expectedErr1 := InsufficientReadQuorum{}
+	if err.Error() != expectedErr1.Error() {
+		t.Fatalf("Test %s: expected error %s, got %s instead.", instanceType, expectedErr1, err)
+	}
+
+	// This causes invalid upload id.
+	for _, disk := range disks {
+		os.RemoveAll(disk)
+	}
+
+	// Object part upload should fail with bucket not found.
+	_, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), ObjectOptions{})
+	if err == nil {
+		t.Fatalf("Test %s: expected to fail but passed instead", instanceType)
+	}
+
+	// As all disks at not available, bucket not found.
+	expectedErr2 := BucketNotFound{Bucket: testCase.bucketName}
+	if err.Error() != expectedErr2.Error() {
+		t.Fatalf("Test %s: expected error %s, got %s instead.", instanceType, expectedErr2, err)
 	}
 }
 

cmd/object-handlers.go

@@ -1176,6 +1176,11 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 
+	// This request header needs to be set prior to setting ObjectOptions
+	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
+		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
+	}
+
 	actualSize := size
 
 	if objectAPI.IsCompressionSupported() && isCompressible(r.Header, object) && size > 0 {
@@ -1205,10 +1210,6 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	rawReader := hashReader
 	pReader := NewPutObjReader(rawReader, nil, nil)
 
-	// This request header needs to be set prior to setting ObjectOptions
-	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
-		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
-	}
 	// get gateway encryption options
 	var opts ObjectOptions
 	opts, err = putOpts(ctx, r, bucket, object, metadata)

cmd/posix-errors.go

@@ -136,3 +136,12 @@ func isSysErrCrossDevice(err error) bool {
 	e, ok := err.(*os.LinkError)
 	return ok && e.Err == syscall.EXDEV
 }
+
+// Check if given error corresponds to too many open files
+func isSysErrTooManyFiles(err error) bool {
+	if err == syscall.ENFILE || err == syscall.EMFILE {
+		return true
+	}
+	pathErr, ok := err.(*os.PathError)
+	return ok && (pathErr.Err == syscall.ENFILE || pathErr.Err == syscall.EMFILE)
+}

cmd/posix-list-dir_unix.go

@@ -20,7 +20,6 @@ package cmd
 
 import (
 	"os"
-	"path"
 	"runtime"
 	"sync"
 	"syscall"
@@ -79,7 +78,7 @@ func parseDirents(dirPath string, buf []byte) (entries []string, err error) {
 			// On Linux XFS does not implement d_type for on disk
 			// format << v5. Fall back to OsStat().
 			var fi os.FileInfo
-			fi, err = os.Stat(path.Join(dirPath, name))
+			fi, err = os.Stat(pathJoin(dirPath, name))
 			if err != nil {
 				// If file does not exist, we continue and skip it.
 				// Could happen if it was deleted in the middle while

cmd/posix-list-dir_windows.go

@@ -20,7 +20,6 @@ package cmd
 
 import (
 	"os"
-	"path"
 	"strings"
 	"syscall"
 )
@@ -82,7 +81,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 		case data.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0:
 			// If its symbolic link, follow the link using os.Stat()
 			var fi os.FileInfo
-			fi, err = os.Stat(path.Join(dirPath, name))
+			fi, err = os.Stat(pathJoin(dirPath, name))
 			if err != nil {
 				// If file does not exist, we continue and skip it.
 				// Could happen if it was deleted in the middle while

cmd/posix.go

@@ -37,12 +37,15 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/mountinfo"
+	"github.com/ncw/directio"
 )
 
 const (
 	diskMinFreeSpace    = 900 * humanize.MiByte // Min 900MiB free space.
 	diskMinTotalSpace   = diskMinFreeSpace      // Min 900MiB total space.
 	maxAllowedIOError   = 5
+	posixWriteBlockSize = 4 * humanize.MiByte
+	directioAlignSize   = 4096 // DirectIO alignment needs to be 4K. Defined here as directio.AlignSize is defined as 0 in MacOS causing divide by 0 error.
 )
 
 // isValidVolname verifies a volname name in accordance with object
@@ -71,7 +74,6 @@ type posix struct {
 	connected bool
 
 	diskMount bool // indicates if the path is an actual mount.
-	driveSync bool // indicates if the backend is synchronous.
 
 	diskFileInfo os.FileInfo
 	// Disk usage metrics
@@ -185,10 +187,10 @@ func newPosix(path string) (*posix, error) {
 	p := &posix{
 		connected: true,
 		diskPath:  path,
-		// 1MiB buffer pool for posix internal operations.
+		// 4MiB buffer pool for posix internal operations.
 		pool: sync.Pool{
 			New: func() interface{} {
-				b := make([]byte, readSizeV1)
+				b := directio.AlignedBlock(posixWriteBlockSize)
 				return &b
 			},
 		},
@@ -197,15 +199,6 @@ func newPosix(path string) (*posix, error) {
 		diskMount:    mountinfo.IsLikelyMountPoint(path),
 	}
 
-	var pf BoolFlag
-	if driveSync := os.Getenv("MINIO_DRIVE_SYNC"); driveSync != "" {
-		pf, err = ParseBoolFlag(driveSync)
-		if err != nil {
-			return nil, err
-		}
-		p.driveSync = bool(pf)
-	}
-
 	if !p.diskMount {
 		go p.diskUsage(GlobalServiceDoneCh)
 	}
@@ -651,7 +644,7 @@ func (s *posix) DeleteVol(volume string) (err error) {
 
 // ListDir - return all the entries at the given directory path.
 // If an entry is a directory it will be returned with a trailing "/".
-func (s *posix) ListDir(volume, dirPath string, count int) (entries []string, err error) {
+func (s *posix) ListDir(volume, dirPath string, count int, leafFile string) (entries []string, err error) {
 	defer func() {
 		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
@@ -684,9 +677,21 @@ func (s *posix) ListDir(volume, dirPath string, count int) (entries []string, er
 
 	dirPath = pathJoin(volumeDir, dirPath)
 	if count > 0 {
-		return readDirN(dirPath, count)
+		entries, err = readDirN(dirPath, count)
+	} else {
+		entries, err = readDir(dirPath)
 	}
-	return readDir(dirPath)
+
+	// If leaf file is specified, filter out the entries.
+	if leafFile != "" {
+		for i, entry := range entries {
+			if _, serr := os.Stat(pathJoin(dirPath, entry, leafFile)); serr == nil {
+				entries[i] = strings.TrimSuffix(entry, slashSeparator)
+			}
+		}
+	}
+
+	return entries, err
 }
 
 // ReadAll reads from r until an error or EOF and returns the data it read.
@@ -721,6 +726,8 @@ func (s *posix) ReadAll(volume, path string) (buf []byte, err error) {
 			return nil, errVolumeNotFound
 		} else if isSysErrIO(err) {
 			return nil, errFaultyDisk
+		} else if isSysErrTooManyFiles(err) {
+			return nil, errTooManyOpenFiles
 		}
 		return nil, err
 	}
@@ -824,6 +831,8 @@ func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verif
 			return 0, errFileAccessDenied
 		case isSysErrIO(err):
 			return 0, errFaultyDisk
+		case isSysErrTooManyFiles(err):
+			return 0, errTooManyOpenFiles
 		default:
 			return 0, err
 		}
@@ -934,6 +943,8 @@ func (s *posix) openFile(volume, path string, mode int) (f *os.File, err error)
 			return nil, errFileAccessDenied
 		case isSysErrIO(err):
 			return nil, errFaultyDisk
+		case isSysErrTooManyFiles(err):
+			return nil, errTooManyOpenFiles
 		default:
 			return nil, err
 		}
@@ -996,6 +1007,8 @@ func (s *posix) ReadFileStream(volume, path string, offset, length int64) (io.Re
 			return nil, errFileAccessDenied
 		case isSysErrIO(err):
 			return nil, errFaultyDisk
+		case isSysErrTooManyFiles(err):
+			return nil, errTooManyOpenFiles
 		default:
 			return nil, err
 		}
@@ -1023,10 +1036,9 @@ func (s *posix) ReadFileStream(volume, path string, offset, length int64) (io.Re
 
 // CreateFile - creates the file.
 func (s *posix) CreateFile(volume, path string, fileSize int64, r io.Reader) (err error) {
-	if fileSize < 0 {
+	if fileSize < -1 {
 		return errInvalidArgument
 	}
-
 	defer func() {
 		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
@@ -1045,13 +1057,49 @@ func (s *posix) CreateFile(volume, path string, fileSize int64, r io.Reader) (er
 		return err
 	}
 
-	// Create file if not found. Note that it is created with os.O_EXCL flag as the file
-	// always is supposed to be created in the tmp directory with a unique file name.
-	w, err := s.openFile(volume, path, os.O_CREATE|os.O_APPEND|os.O_WRONLY|os.O_EXCL)
-	if err != nil {
+	if err = s.checkDiskFound(); err != nil {
 		return err
 	}
 
+	volumeDir, err := s.getVolDir(volume)
+	if err != nil {
+		return err
+	}
+	// Stat a volume entry.
+	_, err = os.Stat((volumeDir))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
+		}
+		return err
+	}
+
+	filePath := pathJoin(volumeDir, path)
+	if err = checkPathLength((filePath)); err != nil {
+		return err
+	}
+
+	// Create top level directories if they don't exist.
+	// with mode 0777 mkdir honors system umask.
+	if err = mkdirAll(slashpath.Dir(filePath), 0777); err != nil {
+		return err
+	}
+
+	w, err := disk.OpenFileDirectIO(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL|os.O_SYNC, 0666)
+	if err != nil {
+		switch {
+		case os.IsPermission(err):
+			return errFileAccessDenied
+		case os.IsExist(err):
+			return errFileAccessDenied
+		case isSysErrIO(err):
+			return errFaultyDisk
+		default:
+			return err
+		}
+	}
 	defer w.Close()
 
 	var e error
@@ -1078,17 +1126,70 @@ func (s *posix) CreateFile(volume, path string, fileSize int64, r io.Reader) (er
 	bufp := s.pool.Get().(*[]byte)
 	defer s.pool.Put(bufp)
 
-	n, err := io.CopyBuffer(w, r, *bufp)
+	buf := *bufp
+
+	// Writes remaining bytes in the buffer.
+	writeRemaining := func(w *os.File, buf []byte) (remainingWritten int, err error) {
+		var n int
+		remaining := len(buf)
+		// The following logic writes the remainging data such that it writes whatever best is possible (aligned buffer)
+		// in O_DIRECT mode and remaining (unaligned buffer) in non-O_DIRECT mode.
+		remainingAligned := (remaining / directioAlignSize) * directioAlignSize
+		remainingAlignedBuf := buf[:remainingAligned]
+		remainingUnalignedBuf := buf[remainingAligned:]
+		if len(remainingAlignedBuf) > 0 {
+			n, err = w.Write(remainingAlignedBuf)
+			if err != nil {
+				return 0, err
+			}
+			remainingWritten += n
+		}
+		if len(remainingUnalignedBuf) > 0 {
+			// Write on O_DIRECT fds fail if buffer is not 4K aligned, hence disable O_DIRECT.
+			if err = disk.DisableDirectIO(w); err != nil {
+				return 0, err
+			}
+			n, err = w.Write(remainingUnalignedBuf)
+			if err != nil {
+				return 0, err
+			}
+			remainingWritten += n
+		}
+		return remainingWritten, nil
+	}
+
+	var written int
+	for {
+		var n int
+		n, err = io.ReadFull(r, buf)
+		switch err {
+		case nil:
+			n, err = w.Write(buf)
 			if err != nil {
 				return err
 			}
-	if n < fileSize {
+			written += n
+		case io.ErrUnexpectedEOF:
+			n, err = writeRemaining(w, buf[:n])
+			if err != nil {
+				return err
+			}
+			written += n
+			fallthrough
+		case io.EOF:
+			if fileSize != -1 {
+				if written < int(fileSize) {
 					return errLessData
 				}
-	if n > fileSize {
+				if written > int(fileSize) {
 					return errMoreData
 				}
+			}
 			return nil
+		default:
+			return err
+		}
+	}
 }
 
 func (s *posix) WriteAll(volume, path string, buf []byte) (err error) {
@@ -1130,13 +1231,9 @@ func (s *posix) AppendFile(volume, path string, buf []byte) (err error) {
 	}
 
 	var w *os.File
-	// Create file if not found, additionally also enables synchronous
-	// operation if asked by the user.
-	if s.driveSync {
+	// Create file if not found. Not doing O_DIRECT here to avoid the code that does buffer aligned writes.
+	// AppendFile() is only used by healing code to heal objects written in old format.
 	w, err = s.openFile(volume, path, os.O_CREATE|os.O_SYNC|os.O_APPEND|os.O_WRONLY)
-	} else {
-		w, err = s.openFile(volume, path, os.O_CREATE|os.O_APPEND|os.O_WRONLY)
-	}
 	if err != nil {
 		return err
 	}

cmd/posix_test.go

@@ -839,7 +839,7 @@ func TestPosixPosixListDir(t *testing.T) {
 		} else {
 			t.Errorf("Expected the StorageAPI to be of type *posix")
 		}
-		dirList, err = posixStorage.ListDir(testCase.srcVol, testCase.srcPath, -1)
+		dirList, err = posixStorage.ListDir(testCase.srcVol, testCase.srcPath, -1, "")
 		if err != testCase.expectedErr {
 			t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}

cmd/rest/client.go

@@ -89,12 +89,7 @@ func newCustomDialContext(timeout time.Duration) func(ctx context.Context, netwo
 			DualStack: true,
 		}
 
-		conn, err := dialer.DialContext(ctx, network, addr)
-		if err != nil {
-			return nil, err
-		}
-
-		return xhttp.NewTimeoutConn(conn, timeout, timeout), nil
+		return dialer.DialContext(ctx, network, addr)
 	}
 }
 

cmd/server-main.go

@@ -167,6 +167,8 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	}
 	logger.FatalIf(err, "Invalid command line arguments")
 
+	logger.LogIf(context.Background(), checkEndpointsSubOptimal(ctx, setupType, globalEndpoints))
+
 	globalMinioHost, globalMinioPort = mustSplitHostPort(globalMinioAddr)
 
 	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
@@ -200,6 +202,8 @@ func serverMain(ctx *cli.Context) {
 		cli.ShowCommandHelpAndExit(ctx, "server", 1)
 	}
 
+	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)
+
 	// Disable logging until server initialization is complete, any
 	// error during initialization will be shown as a fatal message
 	logger.Disable = true
@@ -305,8 +309,6 @@ func serverMain(ctx *cli.Context) {
 		globalHTTPServerErrorCh <- globalHTTPServer.Start()
 	}()
 
-	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)
-
 	newObject, err := newObjectLayer(globalEndpoints)
 	if err != nil {
 		// Stop watching for any certificate changes.

cmd/storage-errors.go

@@ -48,6 +48,9 @@ var errDiskAccessDenied = errors.New("disk access denied")
 // errFileNotFound - cannot find the file.
 var errFileNotFound = errors.New("file not found")
 
+// errTooManyOpenFiles - too many open files.
+var errTooManyOpenFiles = errors.New("too many open files")
+
 // errFileNameTooLong - given file name is too long than supported length.
 var errFileNameTooLong = errors.New("file name too long")
 

cmd/storage-interface.go

@@ -39,7 +39,7 @@ type StorageAPI interface {
 	DeleteVol(volume string) (err error)
 
 	// File operations.
-	ListDir(volume, dirPath string, count int) ([]string, error)
+	ListDir(volume, dirPath string, count int, leafFile string) ([]string, error)
 	ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error)
 	AppendFile(volume string, path string, buf []byte) (err error)
 	CreateFile(volume, path string, size int64, reader io.Reader) error

cmd/storage-rest-client.go

@@ -44,6 +44,9 @@ func isNetworkError(err error) bool {
 	if err.Error() == errConnectionStale.Error() {
 		return true
 	}
+	if strings.Contains(err.Error(), "connection reset by peer") {
+		return true
+	}
 	if uerr, isURLError := err.(*url.Error); isURLError {
 		if uerr.Timeout() {
 			return true
@@ -56,6 +59,19 @@ func isNetworkError(err error) bool {
 	return isNetOpError
 }
 
+// Attempt to approximate network error with a
+// typed network error, otherwise default to
+// errDiskNotFound
+func toNetworkError(err error) error {
+	if err == nil {
+		return err
+	}
+	if strings.Contains(err.Error(), "connection reset by peer") {
+		return errNetworkConnReset
+	}
+	return errDiskNotFound
+}
+
 // Converts rpc.ServerError to underlying error. This function is
 // written so that the storageAPI errors are consistent across network
 // disks as well.
@@ -65,7 +81,7 @@ func toStorageErr(err error) error {
 	}
 
 	if isNetworkError(err) {
-		return errDiskNotFound
+		return toNetworkError(err)
 	}
 
 	switch err.Error() {
@@ -234,7 +250,7 @@ func (client *storageRESTClient) CreateFile(volume, path string, length int64, r
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTFilePath, path)
 	values.Set(storageRESTLength, strconv.Itoa(int(length)))
-	respBody, err := client.call(storageRESTMethodCreateFile, values, r, length)
+	respBody, err := client.call(storageRESTMethodCreateFile, values, ioutil.NopCloser(r), length)
 	defer http.DrainBody(respBody)
 	return err
 }
@@ -315,11 +331,12 @@ func (client *storageRESTClient) ReadFile(volume, path string, offset int64, buf
 }
 
 // ListDir - lists a directory.
-func (client *storageRESTClient) ListDir(volume, dirPath string, count int) (entries []string, err error) {
+func (client *storageRESTClient) ListDir(volume, dirPath string, count int, leafFile string) (entries []string, err error) {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTDirPath, dirPath)
 	values.Set(storageRESTCount, strconv.Itoa(count))
+	values.Set(storageRESTLeafFile, leafFile)
 	respBody, err := client.call(storageRESTMethodListDir, values, nil, -1)
 	if err != nil {
 		return nil, err

cmd/storage-rest-common.go

@@ -16,7 +16,7 @@
 
 package cmd
 
-const storageRESTVersion = "v4"
+const storageRESTVersion = "v5"
 const storageRESTPath = minioReservedBucketPath + "/storage/" + storageRESTVersion + "/"
 
 const (
@@ -50,6 +50,7 @@ const (
 	storageRESTOffset     = "offset"
 	storageRESTLength     = "length"
 	storageRESTCount      = "count"
+	storageRESTLeafFile   = "leaf-file"
 	storageRESTBitrotAlgo = "bitrot-algo"
 	storageRESTBitrotHash = "bitrot-hash"
 	storageRESTInstanceID = "instance-id"

cmd/storage-rest-server.go

@@ -361,12 +361,13 @@ func (s *storageRESTServer) ListDirHandler(w http.ResponseWriter, r *http.Reques
 	vars := mux.Vars(r)
 	volume := vars[storageRESTVolume]
 	dirPath := vars[storageRESTDirPath]
+	leafFile := vars[storageRESTLeafFile]
 	count, err := strconv.Atoi(vars[storageRESTCount])
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
-	entries, err := s.storage.ListDir(volume, dirPath, count)
+	entries, err := s.storage.ListDir(volume, dirPath, count, leafFile)
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
@@ -443,7 +444,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpoints EndpointList) {
 		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodReadFileStream).HandlerFunc(httpTraceHdrs(server.ReadFileStreamHandler)).
 			Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTOffset, storageRESTLength)...)
 		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodListDir).HandlerFunc(httpTraceHdrs(server.ListDirHandler)).
-			Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTCount)...)
+			Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTCount, storageRESTLeafFile)...)
 		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodDeleteFile).HandlerFunc(httpTraceHdrs(server.DeleteFileHandler)).
 			Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
 		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodRenameFile).HandlerFunc(httpTraceHdrs(server.RenameFileHandler)).

cmd/storage-rest_test.go

@@ -260,7 +260,7 @@ func testStorageAPIListDir(t *testing.T, storage StorageAPI) {
 	}
 
 	for i, testCase := range testCases {
-		result, err := storage.ListDir(testCase.volumeName, testCase.prefix, -1)
+		result, err := storage.ListDir(testCase.volumeName, testCase.prefix, -1, "")
 		expectErr := (err != nil)
 
 		if expectErr != testCase.expectErr {
@@ -508,6 +508,10 @@ func newStorageRESTHTTPServerClient(t *testing.T) (*httptest.Server, *storageRES
 		t.Fatalf("NewEndpoint failed %v", endpoint)
 	}
 
+	if err := endpoint.UpdateIsLocal(); err != nil {
+		t.Fatalf("UpdateIsLocal failed %v", err)
+	}
+
 	registerStorageRESTHandlers(router, EndpointList{endpoint})
 	restClient, err := newStorageRESTClient(endpoint)
 	if err != nil {

cmd/sts-handlers.go

@@ -25,6 +25,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/iam/validator"
+	"github.com/minio/minio/pkg/wildcard"
 )
 
 const (
@@ -49,13 +50,19 @@ func registerSTSRouter(router *mux.Router) {
 	stsRouter := router.NewRoute().PathPrefix("/").Subrouter()
 
 	// Assume roles with no JWT, handles AssumeRole.
-	stsRouter.Methods("POST").HeadersRegexp("Content-Type", "application/x-www-form-urlencoded*").
-		HeadersRegexp("Authorization", "AWS4-HMAC-SHA256*").
-		HandlerFunc(httpTraceAll(sts.AssumeRole))
+	stsRouter.Methods("POST").MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
+		ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get("Content-Type"))
+		authOk := wildcard.MatchSimple("AWS4-HMAC-SHA256*", r.Header.Get("Authorization"))
+		noQueries := len(r.URL.Query()) == 0
+		return ctypeOk && authOk && noQueries
+	}).HandlerFunc(httpTraceAll(sts.AssumeRole))
 
 	// Assume roles with JWT handler, handles both ClientGrants and WebIdentity.
-	stsRouter.Methods("POST").HeadersRegexp("Content-Type", "application/x-www-form-urlencoded*").
-		HandlerFunc(httpTraceAll(sts.AssumeRoleWithJWT))
+	stsRouter.Methods("POST").MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
+		ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get("Content-Type"))
+		noQueries := len(r.URL.Query()) == 0
+		return ctypeOk && noQueries
+	}).HandlerFunc(httpTraceAll(sts.AssumeRoleWithJWT))
 
 	// AssumeRoleWithClientGrants
 	stsRouter.Methods("POST").HandlerFunc(httpTraceAll(sts.AssumeRoleWithClientGrants)).

cmd/tree-walk.go

@@ -25,40 +25,9 @@ import (
 // TreeWalkResult - Tree walk result carries results of tree walking.
 type TreeWalkResult struct {
 	entry string
-	err   error
 	end   bool
 }
 
-// posix.ListDir returns entries with trailing "/" for directories. At the object layer
-// we need to remove this trailing "/" for objects and retain "/" for prefixes before
-// sorting because the trailing "/" can affect the sorting results for certain cases.
-// Ex. lets say entries = ["a-b/", "a/"] and both are objects.
-//     sorting with out trailing "/" = ["a", "a-b"]
-//     sorting with trailing "/"     = ["a-b/", "a/"]
-// Hence if entries[] does not have a case like the above example then isLeaf() check
-// can be delayed till the entry is pushed into the TreeWalkResult channel.
-// delayIsLeafCheck() returns true if isLeaf can be delayed or false if
-// isLeaf should be done in listDir()
-func delayIsLeafCheck(entries []string) bool {
-	for i, entry := range entries {
-		if i == len(entries)-1 {
-			break
-		}
-		// If any byte in the "entry" string is less than '/' then the
-		// next "entry" should not contain '/' at the same same byte position.
-		for j := 0; j < len(entry); j++ {
-			if entry[j] < '/' {
-				if len(entries[i+1]) > j {
-					if entries[i+1][j] == '/' {
-						return false
-					}
-				}
-			}
-		}
-	}
-	return true
-}
-
 // Return entries that have prefix prefixEntry.
 // Note: input entries are expected to be sorted.
 func filterMatchingPrefix(entries []string, prefixEntry string) []string {
@@ -82,51 +51,15 @@ func filterMatchingPrefix(entries []string, prefixEntry string) []string {
 		}
 		end--
 	}
+	sort.Strings(entries[start:end])
 	return entries[start:end]
 }
 
 // ListDirFunc - "listDir" function of type listDirFunc returned by listDirFactory() - explained below.
-type ListDirFunc func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool)
-
-// IsLeafFunc - A function isLeaf of type isLeafFunc is used to detect if an entry is a leaf entry. There are four scenarios
-// where isLeaf should behave differently:
-// 1. FS backend object listing - isLeaf is true if the entry has a trailing "/"
-// 2. FS backend multipart listing - isLeaf is true if the entry is a directory and contains uploads.json
-// 3. XL backend object listing - isLeaf is true if the entry is a directory and contains xl.json
-// 4. XL backend multipart listing - isLeaf is true if the entry is a directory and contains uploads.json
-type IsLeafFunc func(string, string) bool
-
-// IsLeafDirFunc - A function isLeafDir of type isLeafDirFunc is used to detect if an entry represents an empty directory.
-type IsLeafDirFunc func(string, string) bool
-
-func filterListEntries(bucket, prefixDir string, entries []string, prefixEntry string, isLeaf IsLeafFunc) ([]string, bool) {
-	// Listing needs to be sorted.
-	sort.Strings(entries)
-
-	// Filter entries that have the prefix prefixEntry.
-	entries = filterMatchingPrefix(entries, prefixEntry)
-
-	// Can isLeaf() check be delayed till when it has to be sent down the
-	// TreeWalkResult channel?
-	delayIsLeaf := delayIsLeafCheck(entries)
-	if delayIsLeaf {
-		return entries, true
-	}
-
-	// isLeaf() check has to happen here so that trailing "/" for objects can be removed.
-	for i, entry := range entries {
-		if isLeaf(bucket, pathJoin(prefixDir, entry)) {
-			entries[i] = strings.TrimSuffix(entry, slashSeparator)
-		}
-	}
-	// Sort again after removing trailing "/" for objects as the previous sort
-	// does not hold good anymore.
-	sort.Strings(entries)
-	return entries, false
-}
+type ListDirFunc func(bucket, prefixDir, prefixEntry string) (entries []string)
 
 // treeWalk walks directory tree recursively pushing TreeWalkResult into the channel as and when it encounters files.
-func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker string, recursive bool, listDir ListDirFunc, isLeaf IsLeafFunc, isLeafDir IsLeafDirFunc, resultCh chan TreeWalkResult, endWalkCh chan struct{}, isEnd bool) error {
+func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker string, recursive bool, listDir ListDirFunc, resultCh chan TreeWalkResult, endWalkCh chan struct{}, isEnd bool) error {
 	// Example:
 	// if prefixDir="one/two/three/" and marker="four/five.txt" treeWalk is recursively
 	// called with prefixDir="one/two/three/four/" and marker="five.txt"
@@ -142,12 +75,7 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
 		}
 	}
 
-	entries, delayIsLeaf := listDir(bucket, prefixDir, entryPrefixMatch)
-	// When isleaf check is delayed, make sure that it is set correctly here.
-	if delayIsLeaf && isLeaf == nil {
-		return errInvalidArgument
-	}
-
+	entries := listDir(bucket, prefixDir, entryPrefixMatch)
 	// For an empty list return right here.
 	if len(entries) == 0 {
 		return nil
@@ -166,20 +94,12 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
 	}
 
 	for i, entry := range entries {
-		var leaf, leafDir bool
+		pentry := pathJoin(prefixDir, entry)
 
-		// Decision to do isLeaf check was pushed from listDir() to here.
-		if delayIsLeaf {
-			leaf = isLeaf(bucket, pathJoin(prefixDir, entry))
-			if leaf {
-				entry = strings.TrimSuffix(entry, slashSeparator)
-			}
-		} else {
-			leaf = !strings.HasSuffix(entry, slashSeparator)
-		}
-
-		if strings.HasSuffix(entry, slashSeparator) {
-			leafDir = isLeafDir(bucket, pathJoin(prefixDir, entry))
+		leaf := !hasSuffix(pentry, slashSeparator)
+		var leafDir bool
+		if !leaf {
+			leafDir = false
 		}
 
 		isDir := !leafDir && !leaf
@@ -209,17 +129,19 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
 			// markIsEnd is passed to this entry's treeWalk() so that treeWalker.end can be marked
 			// true at the end of the treeWalk stream.
 			markIsEnd := i == len(entries)-1 && isEnd
-			if tErr := doTreeWalk(ctx, bucket, pathJoin(prefixDir, entry), prefixMatch, markerArg, recursive, listDir, isLeaf, isLeafDir, resultCh, endWalkCh, markIsEnd); tErr != nil {
-				return tErr
+			if err := doTreeWalk(ctx, bucket, pentry, prefixMatch, markerArg, recursive,
+				listDir, resultCh, endWalkCh, markIsEnd); err != nil {
+				return err
 			}
 			continue
 		}
+
 		// EOF is set if we are at last entry and the caller indicated we at the end.
 		isEOF := ((i == len(entries)-1) && isEnd)
 		select {
 		case <-endWalkCh:
 			return errWalkAbort
-		case resultCh <- TreeWalkResult{entry: pathJoin(prefixDir, entry), end: isEOF}:
+		case resultCh <- TreeWalkResult{entry: pentry, end: isEOF}:
 		}
 	}
 
@@ -228,7 +150,7 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
 }
 
 // Initiate a new treeWalk in a goroutine.
-func startTreeWalk(ctx context.Context, bucket, prefix, marker string, recursive bool, listDir ListDirFunc, isLeaf IsLeafFunc, isLeafDir IsLeafDirFunc, endWalkCh chan struct{}) chan TreeWalkResult {
+func startTreeWalk(ctx context.Context, bucket, prefix, marker string, recursive bool, listDir ListDirFunc, endWalkCh chan struct{}) chan TreeWalkResult {
 	// Example 1
 	// If prefix is "one/two/three/" and marker is "one/two/three/four/five.txt"
 	// treeWalk is called with prefixDir="one/two/three/" and marker="four/five.txt"
@@ -250,7 +172,7 @@ func startTreeWalk(ctx context.Context, bucket, prefix, marker string, recursive
 	marker = strings.TrimPrefix(marker, prefixDir)
 	go func() {
 		isEnd := true // Indication to start walking the tree with end as true.
-		doTreeWalk(ctx, bucket, prefixDir, entryPrefixMatch, marker, recursive, listDir, isLeaf, isLeafDir, resultCh, endWalkCh, isEnd)
+		doTreeWalk(ctx, bucket, prefixDir, entryPrefixMatch, marker, recursive, listDir, resultCh, endWalkCh, isEnd)
 		close(resultCh)
 	}()
 	return resultCh

cmd/tree-walk_test.go

@@ -30,49 +30,6 @@ import (
 // Fixed volume name that could be used across tests
 const volume = "testvolume"
 
-// Test for delayIsLeafCheck.
-func TestDelayIsLeafCheck(t *testing.T) {
-	testCases := []struct {
-		entries []string
-		delay   bool
-	}{
-		// Test cases where isLeaf check can't be delayed.
-		{
-			[]string{"a-b/", "a/"},
-			false,
-		},
-		{
-			[]string{"a%b/", "a/"},
-			false,
-		},
-		{
-			[]string{"a-b-c", "a-b/"},
-			false,
-		},
-
-		// Test cases where isLeaf check can be delayed.
-		{
-			[]string{"a-b/", "aa/"},
-			true,
-		},
-		{
-			[]string{"a", "a-b"},
-			true,
-		},
-		{
-			[]string{"aaa", "bbb"},
-			true,
-		},
-	}
-	for i, testCase := range testCases {
-		expected := testCase.delay
-		got := delayIsLeafCheck(testCase.entries)
-		if expected != got {
-			t.Errorf("Test %d : Expected %t got %t", i+1, expected, got)
-		}
-	}
-}
-
 // Test for filterMatchingPrefix.
 func TestFilterMatchingPrefix(t *testing.T) {
 	entries := []string{"a", "aab", "ab", "abbbb", "zzz"}
@@ -128,11 +85,11 @@ func createNamespace(disk StorageAPI, volume string, files []string) error {
 
 // Test if tree walker returns entries matching prefix alone are received
 // when a non empty prefix is supplied.
-func testTreeWalkPrefix(t *testing.T, listDir ListDirFunc, isLeaf IsLeafFunc, isLeafDir IsLeafDirFunc) {
+func testTreeWalkPrefix(t *testing.T, listDir ListDirFunc) {
 	// Start the tree walk go-routine.
 	prefix := "d/"
 	endWalkCh := make(chan struct{})
-	twResultCh := startTreeWalk(context.Background(), volume, prefix, "", true, listDir, isLeaf, isLeafDir, endWalkCh)
+	twResultCh := startTreeWalk(context.Background(), volume, prefix, "", true, listDir, endWalkCh)
 
 	// Check if all entries received on the channel match the prefix.
 	for res := range twResultCh {
@@ -143,11 +100,11 @@ func testTreeWalkPrefix(t *testing.T, listDir ListDirFunc, isLeaf IsLeafFunc, is
 }
 
 // Test if entries received on tree walk's channel appear after the supplied marker.
-func testTreeWalkMarker(t *testing.T, listDir ListDirFunc, isLeaf IsLeafFunc, isLeafDir IsLeafDirFunc) {
+func testTreeWalkMarker(t *testing.T, listDir ListDirFunc) {
 	// Start the tree walk go-routine.
 	prefix := ""
 	endWalkCh := make(chan struct{})
-	twResultCh := startTreeWalk(context.Background(), volume, prefix, "d/g", true, listDir, isLeaf, isLeafDir, endWalkCh)
+	twResultCh := startTreeWalk(context.Background(), volume, prefix, "d/g", true, listDir, endWalkCh)
 
 	// Check if only 3 entries, namely d/g/h, i/j/k, lmn are received on the channel.
 	expectedCount := 3
@@ -184,23 +141,13 @@ func TestTreeWalk(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	isLeaf := func(volume, prefix string) bool {
-		return !hasSuffix(prefix, slashSeparator)
-	}
+	listDir := listDirFactory(context.Background(), disk)
 
-	isLeafDir := func(volume, prefix string) bool {
-		entries, listErr := disk.ListDir(volume, prefix, 1)
-		if listErr != nil {
-			return false
-		}
-		return len(entries) == 0
-	}
-
-	listDir := listDirFactory(context.Background(), isLeaf, disk)
 	// Simple test for prefix based walk.
-	testTreeWalkPrefix(t, listDir, isLeaf, isLeafDir)
+	testTreeWalkPrefix(t, listDir)
 	// Simple test when marker is set.
-	testTreeWalkMarker(t, listDir, isLeaf, isLeafDir)
+	testTreeWalkMarker(t, listDir)
+
 	err = os.RemoveAll(fsDir)
 	if err != nil {
 		t.Fatal(err)
@@ -228,19 +175,7 @@ func TestTreeWalkTimeout(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	isLeaf := func(volume, prefix string) bool {
-		return !hasSuffix(prefix, slashSeparator)
-	}
-
-	isLeafDir := func(volume, prefix string) bool {
-		entries, listErr := disk.ListDir(volume, prefix, 1)
-		if listErr != nil {
-			return false
-		}
-		return len(entries) == 0
-	}
-
-	listDir := listDirFactory(context.Background(), isLeaf, disk)
+	listDir := listDirFactory(context.Background(), disk)
 
 	// TreeWalk pool with 2 seconds timeout for tree-walk go routines.
 	pool := NewTreeWalkPool(2 * time.Second)
@@ -249,7 +184,7 @@ func TestTreeWalkTimeout(t *testing.T) {
 	prefix := ""
 	marker := ""
 	recursive := true
-	resultCh := startTreeWalk(context.Background(), volume, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
+	resultCh := startTreeWalk(context.Background(), volume, prefix, marker, recursive, listDir, endWalkCh)
 
 	params := listParams{
 		bucket:    volume,
@@ -313,9 +248,7 @@ func TestListDir(t *testing.T) {
 	}
 
 	// create listDir function.
-	listDir := listDirFactory(context.Background(), func(volume, prefix string) bool {
-		return !hasSuffix(prefix, slashSeparator)
-	}, disk1, disk2)
+	listDir := listDirFactory(context.Background(), disk1, disk2)
 
 	// Create file1 in fsDir1 and file2 in fsDir2.
 	disks := []StorageAPI{disk1, disk2}
@@ -327,7 +260,7 @@ func TestListDir(t *testing.T) {
 	}
 
 	// Should list "file1" from fsDir1.
-	entries, _ := listDir(volume, "", "")
+	entries := listDir(volume, "", "")
 	if len(entries) != 2 {
 		t.Fatal("Expected the number of entries to be 2")
 	}
@@ -345,7 +278,7 @@ func TestListDir(t *testing.T) {
 	}
 
 	// Should list "file2" from fsDir2.
-	entries, _ = listDir(volume, "", "")
+	entries = listDir(volume, "", "")
 	if len(entries) != 1 {
 		t.Fatal("Expected the number of entries to be 1")
 	}
@@ -373,21 +306,8 @@ func TestRecursiveTreeWalk(t *testing.T) {
 		t.Fatalf("Unable to create StorageAPI: %s", err)
 	}
 
-	// Simple isLeaf check, returns true if there is no trailing "/"
-	isLeaf := func(volume, prefix string) bool {
-		return !hasSuffix(prefix, slashSeparator)
-	}
-
-	isLeafDir := func(volume, prefix string) bool {
-		entries, listErr := disk1.ListDir(volume, prefix, 1)
-		if listErr != nil {
-			return false
-		}
-		return len(entries) == 0
-	}
-
 	// Create listDir function.
-	listDir := listDirFactory(context.Background(), isLeaf, disk1)
+	listDir := listDirFactory(context.Background(), disk1)
 
 	// Create the namespace.
 	var files = []string{
@@ -461,13 +381,16 @@ func TestRecursiveTreeWalk(t *testing.T) {
 		}},
 	}
 	for i, testCase := range testCases {
+		testCase := testCase
+		t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
 			for entry := range startTreeWalk(context.Background(), volume,
 				testCase.prefix, testCase.marker, testCase.recursive,
-			listDir, isLeaf, isLeafDir, endWalkCh) {
+				listDir, endWalkCh) {
 				if _, found := testCase.expected[entry.entry]; !found {
-				t.Errorf("Test %d: Expected %s, but couldn't find", i+1, entry.entry)
+					t.Errorf("Expected %s, but couldn't find", entry.entry)
 				}
 			}
+		})
 	}
 	err = os.RemoveAll(fsDir1)
 	if err != nil {
@@ -488,21 +411,8 @@ func TestSortedness(t *testing.T) {
 		t.Fatalf("Unable to create StorageAPI: %s", err)
 	}
 
-	// Simple isLeaf check, returns true if there is no trailing "/"
-	isLeaf := func(volume, prefix string) bool {
-		return !hasSuffix(prefix, slashSeparator)
-	}
-
-	isLeafDir := func(volume, prefix string) bool {
-		entries, listErr := disk1.ListDir(volume, prefix, 1)
-		if listErr != nil {
-			return false
-		}
-		return len(entries) == 0
-	}
-
 	// Create listDir function.
-	listDir := listDirFactory(context.Background(), isLeaf, disk1)
+	listDir := listDirFactory(context.Background(), disk1)
 
 	// Create the namespace.
 	var files = []string{
@@ -544,7 +454,7 @@ func TestSortedness(t *testing.T) {
 		var actualEntries []string
 		for entry := range startTreeWalk(context.Background(), volume,
 			test.prefix, test.marker, test.recursive,
-			listDir, isLeaf, isLeafDir, endWalkCh) {
+			listDir, endWalkCh) {
 			actualEntries = append(actualEntries, entry.entry)
 		}
 		if !sort.IsSorted(sort.StringSlice(actualEntries)) {
@@ -572,20 +482,8 @@ func TestTreeWalkIsEnd(t *testing.T) {
 		t.Fatalf("Unable to create StorageAPI: %s", err)
 	}
 
-	isLeaf := func(volume, prefix string) bool {
-		return !hasSuffix(prefix, slashSeparator)
-	}
-
-	isLeafDir := func(volume, prefix string) bool {
-		entries, listErr := disk1.ListDir(volume, prefix, 1)
-		if listErr != nil {
-			return false
-		}
-		return len(entries) == 0
-	}
-
 	// Create listDir function.
-	listDir := listDirFactory(context.Background(), isLeaf, disk1)
+	listDir := listDirFactory(context.Background(), disk1)
 
 	// Create the namespace.
 	var files = []string{
@@ -626,7 +524,8 @@ func TestTreeWalkIsEnd(t *testing.T) {
 	}
 	for i, test := range testCases {
 		var entry TreeWalkResult
-		for entry = range startTreeWalk(context.Background(), volume, test.prefix, test.marker, test.recursive, listDir, isLeaf, isLeafDir, endWalkCh) {
+		for entry = range startTreeWalk(context.Background(), volume, test.prefix,
+			test.marker, test.recursive, listDir, endWalkCh) {
 		}
 		if entry.entry != test.expectedEntry {
 			t.Errorf("Test %d: Expected entry %s, but received %s with the EOF marker", i, test.expectedEntry, entry.entry)

cmd/typed-errors.go

@@ -85,3 +85,6 @@ var errNoSuchPolicy = errors.New("Specified canned policy does not exist")
 
 // error returned when access is denied.
 var errAccessDenied = errors.New("Do not have enough permissions to access this resource")
+
+// errNetworkConnReset - connection reset by peer
+var errNetworkConnReset = errors.New("connection reset by peer")

cmd/web-handlers.go

@@ -583,8 +583,23 @@ func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs,
 
 	claims, owner, authErr := webRequestAuthenticate(r)
 	if authErr != nil {
+		if authErr == errNoAuthToken {
+			// Check if all objects are allowed to be deleted anonymously
+			for _, object := range args.Objects {
+				if !globalPolicySys.IsAllowed(policy.Args{
+					Action:          policy.DeleteObjectAction,
+					BucketName:      args.BucketName,
+					ConditionValues: getConditionValues(r, "", ""),
+					IsOwner:         false,
+					ObjectName:      object,
+				}) {
+					return toJSONError(errAuthentication)
+				}
+			}
+		} else {
 			return toJSONError(authErr)
 		}
+	}
 
 	if args.BucketName == "" || len(args.Objects) == 0 {
 		return toJSONError(errInvalidArgument)
@@ -640,7 +655,10 @@ next:
 					return toJSONError(errMethodNotAllowed)
 				}
 			}
-
+			// Check for permissions only in the case of
+			// non-anonymous login. For anonymous login, policy has already
+			// been checked.
+			if authErr != errNoAuthToken {
 				if !globalIAMSys.IsAllowed(iampolicy.Args{
 					AccountName:     claims.Subject,
 					Action:          iampolicy.DeleteObjectAction,
@@ -651,6 +669,7 @@ next:
 				}) {
 					return toJSONError(errAccessDenied)
 				}
+			}
 
 			if err = deleteObject(context.Background(), objectAPI, web.CacheAPI(), args.BucketName, objectName, r); err != nil {
 				break next

cmd/xl-sets.go

@@ -643,7 +643,7 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke
 // Returns function "listDir" of the type listDirFunc.
 // isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
 // disks - used for doing disk.ListDir(). Sets passes set of disks.
-func listDirSetsFactory(ctx context.Context, isLeaf IsLeafFunc, isLeafDir IsLeafDirFunc, sets ...*xlObjects) ListDirFunc {
+func listDirSetsFactory(ctx context.Context, sets ...*xlObjects) ListDirFunc {
 	listDirInternal := func(bucket, prefixDir, prefixEntry string, disks []StorageAPI) (mergedEntries []string) {
 		var diskEntries = make([][]string, len(disks))
 		var wg sync.WaitGroup
@@ -654,7 +654,7 @@ func listDirSetsFactory(ctx context.Context, isLeaf IsLeafFunc, isLeafDir IsLeaf
 			wg.Add(1)
 			go func(index int, disk StorageAPI) {
 				defer wg.Done()
-				diskEntries[index], _ = disk.ListDir(bucket, prefixDir, -1)
+				diskEntries[index], _ = disk.ListDir(bucket, prefixDir, -1, xlMetaJSONFile)
 			}(index, disk)
 		}
 
@@ -684,7 +684,7 @@ func listDirSetsFactory(ctx context.Context, isLeaf IsLeafFunc, isLeafDir IsLeaf
 	}
 
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string) {
 		for _, set := range sets {
 			var newEntries []string
 			// Find elements in entries which are not in mergedEntries
@@ -703,7 +703,7 @@ func listDirSetsFactory(ctx context.Context, isLeaf IsLeafFunc, isLeafDir IsLeaf
 				sort.Strings(mergedEntries)
 			}
 		}
-		return filterListEntries(bucket, prefixDir, mergedEntries, prefixEntry, isLeaf)
+		return filterMatchingPrefix(mergedEntries, prefixEntry)
 	}
 	return listDir
 }
@@ -712,26 +712,7 @@ func listDirSetsFactory(ctx context.Context, isLeaf IsLeafFunc, isLeafDir IsLeaf
 // listed and subsequently merge lexically sorted inside listDirSetsFactory(). Resulting
 // value through the walk channel receives the data properly lexically sorted.
 func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
-	isLeaf := func(bucket, entry string) bool {
-		entry = strings.TrimSuffix(entry, slashSeparator)
-		// Verify if we are at the leaf, a leaf is where we
-		// see `xl.json` inside a directory.
-		return s.getHashedSet(entry).isObject(bucket, entry)
-	}
-
-	isLeafDir := func(bucket, entry string) bool {
-		// Verify prefixes in all sets.
-		var ok bool
-		for _, set := range s.sets {
-			ok = set.isObjectDir(bucket, entry)
-			if ok {
-				return true
-			}
-		}
-		return false
-	}
-
-	listDir := listDirSetsFactory(ctx, isLeaf, isLeafDir, s.sets...)
+	listDir := listDirSetsFactory(ctx, s.sets...)
 
 	var getObjectInfoDirs []func(context.Context, string, string) (ObjectInfo, error)
 	// Verify prefixes in all sets.
@@ -743,7 +724,7 @@ func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimi
 		return s.getHashedSet(entry).getObjectInfo(ctx, bucket, entry)
 	}
 
-	return listObjects(ctx, s, bucket, prefix, marker, delimiter, maxKeys, s.listPool, isLeaf, isLeafDir, listDir, getObjectInfo, getObjectInfoDirs...)
+	return listObjects(ctx, s, bucket, prefix, marker, delimiter, maxKeys, s.listPool, listDir, getObjectInfo, getObjectInfoDirs...)
 }
 
 func (s *xlSets) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
@@ -1256,25 +1237,13 @@ func (s *xlSets) HealObjects(ctx context.Context, bucket, prefix string, healObj
 	recursive := true
 
 	endWalkCh := make(chan struct{})
-	isLeaf := func(bucket, entry string) bool {
-		return hasSuffix(entry, xlMetaJSONFile)
-	}
-
-	isLeafDir := func(bucket, entry string) bool {
-		return false
-	}
-
-	listDir := listDirSetsFactory(ctx, isLeaf, isLeafDir, s.sets...)
-	walkResultCh := startTreeWalk(ctx, bucket, prefix, "", recursive, listDir, isLeaf, isLeafDir, endWalkCh)
+	listDir := listDirSetsFactory(ctx, s.sets...)
+	walkResultCh := startTreeWalk(ctx, bucket, prefix, "", recursive, listDir, endWalkCh)
 	for {
 		walkResult, ok := <-walkResultCh
 		if !ok {
 			break
 		}
-		// For any walk error return right away.
-		if walkResult.err != nil {
-			return toObjectErr(walkResult.err, bucket, prefix)
-		}
 		if err := healObjectFn(bucket, strings.TrimSuffix(walkResult.entry, slashSeparator+xlMetaJSONFile)); err != nil {
 			return toObjectErr(err, bucket, walkResult.entry)
 		}

cmd/xl-v1-common.go

@@ -85,34 +85,3 @@ func (xl xlObjects) isObject(bucket, prefix string) (ok bool) {
 
 	return reduceReadQuorumErrs(context.Background(), errs, objectOpIgnoredErrs, readQuorum) == nil
 }
-
-// isObjectDir returns if the specified path represents an empty directory.
-func (xl xlObjects) isObjectDir(bucket, prefix string) (ok bool) {
-	var errs = make([]error, len(xl.getDisks()))
-	var wg sync.WaitGroup
-	for index, disk := range xl.getDisks() {
-		if disk == nil {
-			continue
-		}
-		wg.Add(1)
-		go func(index int, disk StorageAPI) {
-			defer wg.Done()
-			// Check if 'prefix' is an object on this 'disk', else continue the check the next disk
-			entries, err := disk.ListDir(bucket, prefix, 1)
-			if err != nil {
-				errs[index] = err
-				return
-			}
-			if len(entries) > 0 {
-				errs[index] = errVolumeNotEmpty
-				return
-			}
-		}(index, disk)
-	}
-
-	wg.Wait()
-
-	readQuorum := len(xl.getDisks()) / 2
-
-	return reduceReadQuorumErrs(context.Background(), errs, objectOpIgnoredErrs, readQuorum) == nil
-}

cmd/xl-v1-healing.go

@@ -341,7 +341,7 @@ func (xl xlObjects) healObject(ctx context.Context, bucket string, object string
 		}
 
 		// List and delete the object directory,
-		files, derr := disk.ListDir(bucket, object, -1)
+		files, derr := disk.ListDir(bucket, object, -1, "")
 		if derr == nil {
 			for _, entry := range files {
 				_ = disk.DeleteFile(bucket,
@@ -425,6 +425,10 @@ func (xl xlObjects) healObject(ctx context.Context, bucket string, object string
 		}
 	}
 
+	// Cleanup in case of xl.json writing failure
+	writeQuorum := latestMeta.Erasure.DataBlocks + 1
+	defer xl.deleteObject(ctx, minioMetaTmpBucket, tmpID, writeQuorum, false)
+
 	// Generate and write `xl.json` generated from other disks.
 	outDatedDisks, aErr := writeUniqueXLMetadata(ctx, outDatedDisks, minioMetaTmpBucket, tmpID,
 		partsMetadata, diskCount(outDatedDisks))
@@ -478,52 +482,51 @@ func (xl xlObjects) healObjectDir(ctx context.Context, bucket, object string, dr
 	hr.Before.Drives = make([]madmin.HealDriveInfo, len(storageDisks))
 	hr.After.Drives = make([]madmin.HealDriveInfo, len(storageDisks))
 
-	var wg sync.WaitGroup
+	errs := statAllDirs(ctx, storageDisks, bucket, object)
+	if isObjectDirDangling(errs) {
+		for i, err := range errs {
+			if err == nil {
+				storageDisks[i].DeleteFile(bucket, object)
+			}
+		}
+	}
 
 	// Prepare object creation in all disks
-	for i, d := range storageDisks {
-		wg.Add(1)
-		go func(idx int, disk StorageAPI) {
-			defer wg.Done()
-			if disk == nil {
-				hr.Before.Drives[idx] = madmin.HealDriveInfo{State: madmin.DriveStateOffline}
-				hr.After.Drives[idx] = madmin.HealDriveInfo{State: madmin.DriveStateOffline}
-				return
+	for i, err := range errs {
+		var drive string
+		if storageDisks[i] != nil {
+			drive = storageDisks[i].String()
 		}
-
-			drive := disk.String()
-			hr.Before.Drives[idx] = madmin.HealDriveInfo{UUID: "", Endpoint: drive, State: madmin.DriveStateOffline}
-			hr.After.Drives[idx] = madmin.HealDriveInfo{UUID: "", Endpoint: drive, State: madmin.DriveStateOffline}
-
-			_, statErr := disk.StatVol(pathJoin(bucket, object))
-			switch statErr {
-			case nil:
-				hr.Before.Drives[idx].State = madmin.DriveStateOk
-				hr.After.Drives[idx].State = madmin.DriveStateOk
-				// Object is fine in this disk, nothing to be done anymore, exiting
-				return
+		switch err {
+		case errDiskNotFound:
+			hr.Before.Drives[i] = madmin.HealDriveInfo{State: madmin.DriveStateOffline}
+			hr.After.Drives[i] = madmin.HealDriveInfo{State: madmin.DriveStateOffline}
 		case errVolumeNotFound:
-				hr.Before.Drives[idx].State = madmin.DriveStateMissing
-				hr.After.Drives[idx].State = madmin.DriveStateMissing
+			hr.Before.Drives[i] = madmin.HealDriveInfo{Endpoint: drive, State: madmin.DriveStateMissing}
+			hr.After.Drives[i] = madmin.HealDriveInfo{Endpoint: drive, State: madmin.DriveStateMissing}
 		default:
-				logger.LogIf(ctx, err)
-				return
+			hr.Before.Drives[i] = madmin.HealDriveInfo{Endpoint: drive, State: madmin.DriveStateCorrupt}
+			hr.After.Drives[i] = madmin.HealDriveInfo{Endpoint: drive, State: madmin.DriveStateCorrupt}
+		}
 	}
-
 	if dryRun {
-				return
+		return hr, nil
+	}
+	for i, err := range errs {
+		switch err {
+		case errVolumeNotFound:
+			merr := storageDisks[i].MakeVol(pathJoin(bucket, object))
+			switch merr {
+			case nil, errVolumeExists:
+				hr.After.Drives[i].State = madmin.DriveStateOk
+			case errDiskNotFound:
+				hr.After.Drives[i].State = madmin.DriveStateOffline
+			default:
+				logger.LogIf(ctx, merr)
+				hr.After.Drives[i].State = madmin.DriveStateCorrupt
 			}
-
-			if err := disk.MakeVol(pathJoin(bucket, object)); err == nil || err == errVolumeExists {
-				hr.After.Drives[idx].State = madmin.DriveStateOk
-			} else {
-				logger.LogIf(ctx, err)
-				hr.After.Drives[idx].State = madmin.DriveStateOffline
 		}
-		}(i, d)
 	}
-
-	wg.Wait()
 	return hr, nil
 }
 
@@ -587,6 +590,46 @@ func defaultHealResult(latestXLMeta xlMetaV1, storageDisks []StorageAPI, errs []
 	return result
 }
 
+// Stat all directories.
+func statAllDirs(ctx context.Context, storageDisks []StorageAPI, bucket, prefix string) []error {
+	var errs = make([]error, len(storageDisks))
+	var wg sync.WaitGroup
+	for index, disk := range storageDisks {
+		if disk == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(index int, disk StorageAPI) {
+			defer wg.Done()
+			entries, err := disk.ListDir(bucket, prefix, 1, "")
+			if err != nil {
+				errs[index] = err
+				return
+			}
+			if len(entries) > 0 {
+				errs[index] = errVolumeNotEmpty
+				return
+			}
+		}(index, disk)
+	}
+
+	wg.Wait()
+	return errs
+}
+
+// ObjectDir is considered dangling/corrupted if any only
+// if total disks - a combination of corrupted and missing
+// files is lesser than N/2+1 number of disks.
+func isObjectDirDangling(errs []error) (ok bool) {
+	var notFoundDir int
+	for _, readErr := range errs {
+		if readErr == errFileNotFound {
+			notFoundDir++
+		}
+	}
+	return notFoundDir > len(errs)/2
+}
+
 // Object is considered dangling/corrupted if any only
 // if total disks - a combination of corrupted and missing
 // files is lesser than number of data blocks.

cmd/xl-v1-list-objects.go

@@ -22,11 +22,10 @@ import (
 )
 
 // Returns function "listDir" of the type listDirFunc.
-// isLeaf - is used by listDir function to check if an entry is a leaf or non-leaf entry.
 // disks - used for doing disk.ListDir()
-func listDirFactory(ctx context.Context, isLeaf IsLeafFunc, disks ...StorageAPI) ListDirFunc {
+func listDirFactory(ctx context.Context, disks ...StorageAPI) ListDirFunc {
 	// Returns sorted merged entries from all the disks.
-	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string, delayIsLeaf bool) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (mergedEntries []string) {
 		for _, disk := range disks {
 			if disk == nil {
 				continue
@@ -34,7 +33,7 @@ func listDirFactory(ctx context.Context, isLeaf IsLeafFunc, disks ...StorageAPI)
 			var entries []string
 			var newEntries []string
 			var err error
-			entries, err = disk.ListDir(bucket, prefixDir, -1)
+			entries, err = disk.ListDir(bucket, prefixDir, -1, xlMetaJSONFile)
 			if err != nil {
 				continue
 			}
@@ -55,8 +54,7 @@ func listDirFactory(ctx context.Context, isLeaf IsLeafFunc, disks ...StorageAPI)
 				sort.Strings(mergedEntries)
 			}
 		}
-		mergedEntries, delayIsLeaf = filterListEntries(bucket, prefixDir, mergedEntries, prefixEntry, isLeaf)
-		return mergedEntries, delayIsLeaf
+		return filterMatchingPrefix(mergedEntries, prefixEntry)
 	}
 	return listDir
 }
@@ -72,10 +70,8 @@ func (xl xlObjects) listObjects(ctx context.Context, bucket, prefix, marker, del
 	walkResultCh, endWalkCh := xl.listPool.Release(listParams{bucket, recursive, marker, prefix})
 	if walkResultCh == nil {
 		endWalkCh = make(chan struct{})
-		isLeaf := xl.isObject
-		isLeafDir := xl.isObjectDir
-		listDir := listDirFactory(ctx, isLeaf, xl.getLoadBalancedDisks()...)
-		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
+		listDir := listDirFactory(ctx, xl.getLoadBalancedDisks()...)
+		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, endWalkCh)
 	}
 
 	var objInfos []ObjectInfo
@@ -89,10 +85,6 @@ func (xl xlObjects) listObjects(ctx context.Context, bucket, prefix, marker, del
 			eof = true
 			break
 		}
-		// For any walk error return right away.
-		if walkResult.err != nil {
-			return loi, toObjectErr(walkResult.err, bucket, prefix)
-		}
 		entry := walkResult.entry
 		var objInfo ObjectInfo
 		if hasSuffix(entry, slashSeparator) {
@@ -108,8 +100,10 @@ func (xl xlObjects) listObjects(ctx context.Context, bucket, prefix, marker, del
 				// Ignore errFileNotFound as the object might have got
 				// deleted in the interim period of listing and getObjectInfo(),
 				// ignore quorum error as it might be an entry from an outdated disk.
-				switch err {
-				case errFileNotFound, errXLReadQuorum:
+				if IsErrIgnored(err, []error{
+					errFileNotFound,
+					errXLReadQuorum,
+				}...) {
 					continue
 				}
 				return loi, toObjectErr(err, bucket, prefix)

cmd/xl-v1-metadata.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016, 2017, 2017 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,6 +27,7 @@ import (
 	"sync"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/logger"
 )
 
@@ -74,8 +75,8 @@ func (c ChecksumInfo) MarshalJSON() ([]byte, error) {
 
 // UnmarshalJSON - should never be called, instead xlMetaV1UnmarshalJSON() should be used.
 func (c *ChecksumInfo) UnmarshalJSON(data []byte) error {
-	logger.LogIf(context.Background(), errUnexpected)
 	var info checksumInfoJSON
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	if err := json.Unmarshal(data, &info); err != nil {
 		return err
 	}
@@ -424,14 +425,6 @@ func (xl xlObjects) readXLMetaStat(ctx context.Context, bucket, object string) (
 	return statInfo{}, nil, reduceReadQuorumErrs(ctx, ignoredErrs, nil, readQuorum)
 }
 
-// deleteXLMetadata - deletes `xl.json` on a single disk.
-func deleteXLMetdata(ctx context.Context, disk StorageAPI, bucket, prefix string) error {
-	jsonFile := path.Join(prefix, xlMetaJSONFile)
-	err := disk.DeleteFile(bucket, jsonFile)
-	logger.LogIf(ctx, err)
-	return err
-}
-
 // writeXLMetadata - writes `xl.json` to a single disk.
 func writeXLMetadata(ctx context.Context, disk StorageAPI, bucket, prefix string, xlMeta xlMetaV1) error {
 	jsonFile := path.Join(prefix, xlMetaJSONFile)
@@ -449,27 +442,6 @@ func writeXLMetadata(ctx context.Context, disk StorageAPI, bucket, prefix string
 	return err
 }
 
-// deleteAllXLMetadata - deletes all partially written `xl.json` depending on errs.
-func deleteAllXLMetadata(ctx context.Context, disks []StorageAPI, bucket, prefix string, errs []error) {
-	var wg = &sync.WaitGroup{}
-	// Delete all the `xl.json` left over.
-	for index, disk := range disks {
-		if disk == nil {
-			continue
-		}
-		// Undo rename object in parallel.
-		wg.Add(1)
-		go func(index int, disk StorageAPI) {
-			defer wg.Done()
-			if errs[index] != nil {
-				return
-			}
-			_ = deleteXLMetdata(ctx, disk, bucket, prefix)
-		}(index, disk)
-	}
-	wg.Wait()
-}
-
 // Rename `xl.json` content to destination location for each disk in order.
 func renameXLMetadata(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, quorum int) ([]StorageAPI, error) {
 	isDir := false
@@ -509,10 +481,6 @@ func writeUniqueXLMetadata(ctx context.Context, disks []StorageAPI, bucket, pref
 	wg.Wait()
 
 	err := reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, quorum)
-	if err == errXLWriteQuorum {
-		// Delete all `xl.json` successfully renamed.
-		deleteAllXLMetadata(ctx, disks, bucket, prefix, mErrs)
-	}
 	return evalDisks(disks, mErrs), err
 }
 
@@ -547,9 +515,5 @@ func writeSameXLMetadata(ctx context.Context, disks []StorageAPI, bucket, prefix
 	wg.Wait()
 
 	err := reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, writeQuorum)
-	if err == errXLWriteQuorum {
-		// Delete all `xl.json` successfully renamed.
-		deleteAllXLMetadata(ctx, disks, bucket, prefix, mErrs)
-	}
 	return evalDisks(disks, mErrs), err
 }

cmd/xl-v1-multipart.go

@@ -47,9 +47,10 @@ func (xl xlObjects) getMultipartSHADir(bucket, object string) string {
 	return getSHA256Hash([]byte(pathJoin(bucket, object)))
 }
 
-// isUploadIDExists - verify if a given uploadID exists and is valid.
-func (xl xlObjects) isUploadIDExists(ctx context.Context, bucket, object, uploadID string) bool {
-	return xl.isObject(minioMetaMultipartBucket, xl.getUploadIDDir(bucket, object, uploadID))
+// checkUploadIDExists - verify if a given uploadID exists and is valid.
+func (xl xlObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string) error {
+	_, err := xl.getObjectInfo(ctx, minioMetaMultipartBucket, xl.getUploadIDDir(bucket, object, uploadID))
+	return err
 }
 
 // Removes part given by partName belonging to a mulitpart upload from minioMetaBucket
@@ -135,10 +136,6 @@ func commitXLMetadata(ctx context.Context, disks []StorageAPI, srcBucket, srcPre
 	wg.Wait()
 
 	err := reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, quorum)
-	if err == errXLWriteQuorum {
-		// Delete all `xl.json` successfully renamed.
-		deleteAllXLMetadata(ctx, disks, dstBucket, dstPrefix, mErrs)
-	}
 	return evalDisks(disks, mErrs), err
 }
 
@@ -163,7 +160,7 @@ func (xl xlObjects) ListMultipartUploads(ctx context.Context, bucket, object, ke
 		if disk == nil {
 			continue
 		}
-		uploadIDs, err := disk.ListDir(minioMetaMultipartBucket, xl.getMultipartSHADir(bucket, object), -1)
+		uploadIDs, err := disk.ListDir(minioMetaMultipartBucket, xl.getMultipartSHADir(bucket, object), -1, "")
 		if err != nil {
 			if err == errFileNotFound {
 				return result, nil
@@ -216,15 +213,16 @@ func (xl xlObjects) newMultipartUpload(ctx context.Context, bucket string, objec
 	uploadIDPath := xl.getUploadIDDir(bucket, object, uploadID)
 	tempUploadIDPath := uploadID
 
+	// Delete the tmp path later in case we fail to commit (ignore
+	// returned errors) - this will be a no-op in case of a commit
+	// success.
+	defer xl.deleteObject(ctx, minioMetaTmpBucket, tempUploadIDPath, writeQuorum, false)
+
 	// Write updated `xl.json` to all disks.
 	disks, err := writeSameXLMetadata(ctx, xl.getDisks(), minioMetaTmpBucket, tempUploadIDPath, xlMeta, writeQuorum)
 	if err != nil {
 		return "", toObjectErr(err, minioMetaTmpBucket, tempUploadIDPath)
 	}
-	// delete the tmp path later in case we fail to rename (ignore
-	// returned errors) - this will be a no-op in case of a rename
-	// success.
-	defer xl.deleteObject(ctx, minioMetaTmpBucket, tempUploadIDPath, writeQuorum, false)
 
 	// Attempt to rename temp upload object to actual upload path object
 	_, rErr := rename(ctx, disks, minioMetaTmpBucket, tempUploadIDPath, minioMetaMultipartBucket, uploadIDPath, true, writeQuorum, nil)
@@ -308,9 +306,9 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	}
 
 	// Validates if upload ID exists.
-	if !xl.isUploadIDExists(ctx, bucket, object, uploadID) {
+	if err := xl.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		preUploadIDLock.RUnlock()
-		return pi, InvalidUploadID{UploadID: uploadID}
+		return pi, toObjectErr(err, bucket, object, uploadID)
 	}
 
 	// Read metadata associated with the object from all disks.
@@ -406,9 +404,9 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	}
 	defer postUploadIDLock.Unlock()
 
-	// Validate again if upload ID still exists.
-	if !xl.isUploadIDExists(ctx, bucket, object, uploadID) {
-		return pi, InvalidUploadID{UploadID: uploadID}
+	// Validates if upload ID exists.
+	if err := xl.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+		return pi, toObjectErr(err, bucket, object, uploadID)
 	}
 
 	// Rename temporary part file to its final location.
@@ -452,8 +450,10 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	}
 
 	// Write all the checksum metadata.
-	newUUID := mustGetUUID()
-	tempXLMetaPath := newUUID
+	tempXLMetaPath := mustGetUUID()
+
+	// Cleanup in case of xl.json writing failure
+	defer xl.deleteObject(ctx, minioMetaTmpBucket, tempXLMetaPath, writeQuorum, false)
 
 	// Writes a unique `xl.json` each disk carrying new checksum related information.
 	if onlineDisks, err = writeUniqueXLMetadata(ctx, onlineDisks, minioMetaTmpBucket, tempXLMetaPath, partsMetadata, writeQuorum); err != nil {
@@ -499,8 +499,8 @@ func (xl xlObjects) ListObjectParts(ctx context.Context, bucket, object, uploadI
 	}
 	defer uploadIDLock.Unlock()
 
-	if !xl.isUploadIDExists(ctx, bucket, object, uploadID) {
-		return result, InvalidUploadID{UploadID: uploadID}
+	if err := xl.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+		return result, toObjectErr(err, bucket, object, uploadID)
 	}
 
 	uploadIDPath := xl.getUploadIDDir(bucket, object, uploadID)
@@ -617,8 +617,8 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 	defer uploadIDLock.Unlock()
 
-	if !xl.isUploadIDExists(ctx, bucket, object, uploadID) {
-		return oi, InvalidUploadID{UploadID: uploadID}
+	if err := xl.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+		return oi, toObjectErr(err, bucket, object, uploadID)
 	}
 
 	// Check if an object is present as one of the parent dir.
@@ -731,8 +731,6 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	// Save the consolidated actual size.
 	xlMeta.Meta[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)
 
-	tempUploadIDPath := uploadID
-
 	// Update all xl metadata, make sure to not modify fields like
 	// checksum which are different on each disks.
 	for index := range partsMetadata {
@@ -741,13 +739,18 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		partsMetadata[index].Parts = xlMeta.Parts
 	}
 
+	tempXLMetaPath := mustGetUUID()
+
+	// Cleanup in case of failure
+	defer xl.deleteObject(ctx, minioMetaTmpBucket, tempXLMetaPath, writeQuorum, false)
+
 	// Write unique `xl.json` for each disk.
-	if onlineDisks, err = writeUniqueXLMetadata(ctx, onlineDisks, minioMetaTmpBucket, tempUploadIDPath, partsMetadata, writeQuorum); err != nil {
-		return oi, toObjectErr(err, minioMetaTmpBucket, tempUploadIDPath)
+	if onlineDisks, err = writeUniqueXLMetadata(ctx, onlineDisks, minioMetaTmpBucket, tempXLMetaPath, partsMetadata, writeQuorum); err != nil {
+		return oi, toObjectErr(err, minioMetaTmpBucket, tempXLMetaPath)
 	}
 
 	var rErr error
-	onlineDisks, rErr = commitXLMetadata(ctx, onlineDisks, minioMetaTmpBucket, tempUploadIDPath, minioMetaMultipartBucket, uploadIDPath, writeQuorum)
+	onlineDisks, rErr = commitXLMetadata(ctx, onlineDisks, minioMetaTmpBucket, tempXLMetaPath, minioMetaMultipartBucket, uploadIDPath, writeQuorum)
 	if rErr != nil {
 		return oi, toObjectErr(rErr, minioMetaMultipartBucket, uploadIDPath)
 	}
@@ -821,8 +824,9 @@ func (xl xlObjects) AbortMultipartUpload(ctx context.Context, bucket, object, up
 	}
 	defer uploadIDLock.Unlock()
 
-	if !xl.isUploadIDExists(ctx, bucket, object, uploadID) {
-		return InvalidUploadID{UploadID: uploadID}
+	// Validates if upload ID exists.
+	if err := xl.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+		return toObjectErr(err, bucket, object, uploadID)
 	}
 
 	// Read metadata associated with the object from all disks.
@@ -871,12 +875,12 @@ func (xl xlObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInt
 // Remove the old multipart uploads on the given disk.
 func (xl xlObjects) cleanupStaleMultipartUploadsOnDisk(ctx context.Context, disk StorageAPI, expiry time.Duration) {
 	now := time.Now()
-	shaDirs, err := disk.ListDir(minioMetaMultipartBucket, "", -1)
+	shaDirs, err := disk.ListDir(minioMetaMultipartBucket, "", -1, "")
 	if err != nil {
 		return
 	}
 	for _, shaDir := range shaDirs {
-		uploadIDDirs, err := disk.ListDir(minioMetaMultipartBucket, shaDir, -1)
+		uploadIDDirs, err := disk.ListDir(minioMetaMultipartBucket, shaDir, -1, "")
 		if err != nil {
 			continue
 		}

cmd/xl-v1-object.go

@@ -103,6 +103,9 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc
 
 		tempObj := mustGetUUID()
 
+		// Cleanup in case of xl.json writing failure
+		defer xl.deleteObject(ctx, minioMetaTmpBucket, tempObj, writeQuorum, false)
+
 		// Write unique `xl.json` for each disk.
 		if onlineDisks, err = writeUniqueXLMetadata(ctx, storageDisks, minioMetaTmpBucket, tempObj, metaArr, writeQuorum); err != nil {
 			return oi, toObjectErr(err, srcBucket, srcObject)
@@ -173,10 +176,6 @@ func (xl xlObjects) GetObjectNInfo(ctx context.Context, bucket, object string, r
 	// Handler directory request by returning a reader that
 	// returns no bytes.
 	if hasSuffix(object, slashSeparator) {
-		if !xl.isObjectDir(bucket, object) {
-			nsUnlocker()
-			return nil, toObjectErr(errFileNotFound, bucket, object)
-		}
 		var objInfo ObjectInfo
 		if objInfo, err = xl.getObjectInfoDir(ctx, bucket, object); err != nil {
 			nsUnlocker()
@@ -375,19 +374,16 @@ func (xl xlObjects) getObjectInfoDir(ctx context.Context, bucket, object string)
 		wg.Add(1)
 		go func(index int, disk StorageAPI) {
 			defer wg.Done()
-			if _, err := disk.StatVol(pathJoin(bucket, object)); err != nil {
-				// Since we are re-purposing StatVol, an object which
-				// is a directory if it doesn't exist should be
-				// returned as errFileNotFound instead, convert
-				// the error right here accordingly.
-				if err == errVolumeNotFound {
-					err = errFileNotFound
-				} else if err == errVolumeAccessDenied {
-					err = errFileAccessDenied
-				}
-
-				// Save error to reduce it later
+			// Check if 'prefix' is an object on this 'disk'.
+			entries, err := disk.ListDir(bucket, object, 1, "")
+			if err != nil {
 				errs[index] = err
+				return
+			}
+			if len(entries) > 0 {
+				// Not a directory if not empty.
+				errs[index] = errFileNotFound
+				return
 			}
 		}(index, disk)
 	}
@@ -412,13 +408,11 @@ func (xl xlObjects) GetObjectInfo(ctx context.Context, bucket, object string, op
 	}
 
 	if hasSuffix(object, slashSeparator) {
-		if !xl.isObjectDir(bucket, object) {
-			return oi, toObjectErr(errFileNotFound, bucket, object)
+		info, err := xl.getObjectInfoDir(ctx, bucket, object)
+		if err != nil {
+			return oi, toObjectErr(err, bucket, object)
 		}
-		if oi, e = xl.getObjectInfoDir(ctx, bucket, object); e != nil {
-			return oi, toObjectErr(e, bucket, object)
-		}
-		return oi, nil
+		return info, nil
 	}
 
 	info, err := xl.getObjectInfo(ctx, bucket, object)
@@ -879,8 +873,17 @@ func (xl xlObjects) DeleteObject(ctx context.Context, bucket, object string) (er
 	var writeQuorum int
 	var isObjectDir = hasSuffix(object, slashSeparator)
 
-	if isObjectDir && !xl.isObjectDir(bucket, object) {
-		return toObjectErr(errFileNotFound, bucket, object)
+	if isObjectDir {
+		_, err = xl.getObjectInfoDir(ctx, bucket, object)
+		if err == errXLReadQuorum {
+			if isObjectDirDangling(statAllDirs(ctx, xl.getDisks(), bucket, object)) {
+				// If object is indeed dangling, purge it.
+				err = nil
+			}
+		}
+		if err != nil {
+			return toObjectErr(err, bucket, object)
+		}
 	}
 
 	if isObjectDir {

cmd/xl-v1-utils.go

@@ -18,13 +18,13 @@ package cmd
 
 import (
 	"context"
-	"encoding/hex"
 	"errors"
 	"hash/crc32"
 	"path"
 	"sync"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/tidwall/gjson"
 )
@@ -139,59 +139,6 @@ func parseXLFormat(xlMetaBuf []byte) string {
 	return gjson.GetBytes(xlMetaBuf, "format").String()
 }
 
-func parseXLRelease(xlMetaBuf []byte) string {
-	return gjson.GetBytes(xlMetaBuf, "minio.release").String()
-}
-
-func parseXLErasureInfo(ctx context.Context, xlMetaBuf []byte) (ErasureInfo, error) {
-	erasure := ErasureInfo{}
-	erasureResult := gjson.GetBytes(xlMetaBuf, "erasure")
-	// parse the xlV1Meta.Erasure.Distribution.
-	disResult := erasureResult.Get("distribution").Array()
-
-	distribution := make([]int, len(disResult))
-	for i, dis := range disResult {
-		distribution[i] = int(dis.Int())
-	}
-	erasure.Distribution = distribution
-
-	erasure.Algorithm = erasureResult.Get("algorithm").String()
-	erasure.DataBlocks = int(erasureResult.Get("data").Int())
-	erasure.ParityBlocks = int(erasureResult.Get("parity").Int())
-	erasure.BlockSize = erasureResult.Get("blockSize").Int()
-	erasure.Index = int(erasureResult.Get("index").Int())
-
-	checkSumsResult := erasureResult.Get("checksum").Array()
-
-	// Check for scenario where checksum information missing for some parts.
-	partsResult := gjson.GetBytes(xlMetaBuf, "parts").Array()
-	if len(checkSumsResult) != len(partsResult) {
-		return erasure, errCorruptedFormat
-	}
-
-	// Parse xlMetaV1.Erasure.Checksum array.
-	checkSums := make([]ChecksumInfo, len(checkSumsResult))
-	for i, v := range checkSumsResult {
-		algorithm := BitrotAlgorithmFromString(v.Get("algorithm").String())
-		if !algorithm.Available() {
-			logger.LogIf(ctx, errBitrotHashAlgoInvalid)
-			return erasure, errBitrotHashAlgoInvalid
-		}
-		hash, err := hex.DecodeString(v.Get("hash").String())
-		if err != nil {
-			logger.LogIf(ctx, err)
-			return erasure, err
-		}
-		name := v.Get("name").String()
-		if name == "" {
-			return erasure, errCorruptedFormat
-		}
-		checkSums[i] = ChecksumInfo{Name: name, Algorithm: algorithm, Hash: hash}
-	}
-	erasure.Checksums = checkSums
-	return erasure, nil
-}
-
 func parseXLParts(xlMetaBuf []byte) []ObjectPartInfo {
 	// Parse the XL Parts.
 	partsResult := gjson.GetBytes(xlMetaBuf, "parts").Array()
@@ -220,32 +167,9 @@ func parseXLMetaMap(xlMetaBuf []byte) map[string]string {
 
 // Constructs XLMetaV1 using `gjson` lib to retrieve each field.
 func xlMetaV1UnmarshalJSON(ctx context.Context, xlMetaBuf []byte) (xlMeta xlMetaV1, e error) {
-	// obtain version.
-	xlMeta.Version = parseXLVersion(xlMetaBuf)
-	// obtain format.
-	xlMeta.Format = parseXLFormat(xlMetaBuf)
-	// Parse xlMetaV1.Stat .
-	stat, err := parseXLStat(xlMetaBuf)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return xlMeta, err
-	}
-
-	xlMeta.Stat = stat
-	// parse the xlV1Meta.Erasure fields.
-	xlMeta.Erasure, err = parseXLErasureInfo(ctx, xlMetaBuf)
-	if err != nil {
-		return xlMeta, err
-	}
-
-	// Parse the XL Parts.
-	xlMeta.Parts = parseXLParts(xlMetaBuf)
-	// Get the xlMetaV1.Realse field.
-	xlMeta.Minio.Release = parseXLRelease(xlMetaBuf)
-	// parse xlMetaV1.
-	xlMeta.Meta = parseXLMetaMap(xlMetaBuf)
-
-	return xlMeta, nil
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	e = json.Unmarshal(xlMetaBuf, &xlMeta)
+	return xlMeta, e
 }
 
 // read xl.json from the given disk, parse and return xlV1MetaV1.Parts.
@@ -426,6 +350,9 @@ func calculatePartSizeFromIdx(ctx context.Context, totalSize int64, partSize int
 		logger.LogIf(ctx, errPartSizeIndex)
 		return 0, errPartSizeIndex
 	}
+	if totalSize == -1 {
+		return -1, nil
+	}
 	if totalSize > 0 {
 		// Compute the total count of parts
 		partsCount := totalSize/partSize + 1

docs/minio-limits.md

@@ -4,7 +4,8 @@
 
 |Item|Specification|
 |:---|:---|
-|Maximum number of servers| 32|
+|Maximum number of servers per cluster| 32|
+|Maximum number of Federated clusters | Unlimited|
 |Minimum number of servers| 02|
 |Maximum number of drives per server| Unlimited|
 |Read quorum| N/2|

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '2'
 # 9001 through 9004.
 services:
  minio1:
-  image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+  image: minio/minio:RELEASE.2019-04-23T23-50-36Z
   volumes:
    - data1:/data
   ports:
@@ -15,7 +15,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio2:
-  image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+  image: minio/minio:RELEASE.2019-04-23T23-50-36Z
   volumes:
    - data2:/data
   ports:
@@ -25,7 +25,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio3:
-  image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+  image: minio/minio:RELEASE.2019-04-23T23-50-36Z
   volumes:
    - data3:/data
   ports:
@@ -35,7 +35,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio4:
-  image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+  image: minio/minio:RELEASE.2019-04-23T23-50-36Z
   volumes:
    - data4:/data
   ports:

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.1'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+    image: minio/minio:RELEASE.2019-04-23T23-50-36Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -24,7 +24,7 @@ services:
       - access_key
 
   minio2:
-    image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+    image: minio/minio:RELEASE.2019-04-23T23-50-36Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -46,7 +46,7 @@ services:
       - access_key
 
   minio3:
-    image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+    image: minio/minio:RELEASE.2019-04-23T23-50-36Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -68,7 +68,7 @@ services:
       - access_key
 
   minio4:
-    image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+    image: minio/minio:RELEASE.2019-04-23T23-50-36Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+    image: minio/minio:RELEASE.2019-04-23T23-50-36Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -24,7 +24,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio2:
-    image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+    image: minio/minio:RELEASE.2019-04-23T23-50-36Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -46,7 +46,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio3:
-    image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+    image: minio/minio:RELEASE.2019-04-23T23-50-36Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -68,7 +68,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio4:
-    image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+    image: minio/minio:RELEASE.2019-04-23T23-50-36Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/kubernetes/minio-distributed-daemonset.yaml

@@ -30,7 +30,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+        image: minio/minio:RELEASE.2019-04-23T23-50-36Z
         # Unfortunately you must manually define each server. Perhaps autodiscovery via DNS can be implemented in the future.
         args:
         - server

docs/orchestration/kubernetes/minio-distributed-statefulset.yaml

@@ -21,7 +21,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+        image: minio/minio:RELEASE.2019-04-23T23-50-36Z
         args:
         - server
         - http://minio-0.minio.default.svc.cluster.local/data

docs/orchestration/kubernetes/minio-gcs-gateway-deployment.yaml

@@ -21,7 +21,7 @@ spec:
       containers:
       - name: minio
         # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+        image: minio/minio:RELEASE.2019-04-23T23-50-36Z
         args:
         - gateway
         - gcs

docs/orchestration/kubernetes/minio-standalone-deployment.yaml

@@ -29,7 +29,7 @@ spec:
         - name: data 
           mountPath: "/data"
         # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2019-04-18T01-15-57Z
+        image: minio/minio:RELEASE.2019-04-23T23-50-36Z
         args:
         - server
         - /data

docs/tls/README.md

@@ -22,7 +22,7 @@ Copy the existing private key and public certificate to the `certs` directory. T
 > NOTE: Location of custom certs directory can be specified using `--certs-dir` command line option.
 
 **Note:** 
-* The key and certificate files must be appended with `.key` and `.crt`, respectively.
+* Inside the `certs` directory, the private key must by named `private.key` and the public key must be named `public.crt`.
 * A certificate signed by a CA contains information about the issued identity (e.g. name, expiry, public key) and any intermediate certificates. The root CA is not included.
 
 ## <a name="generate-use-self-signed-keys-certificates"></a>3. Generate and use Self-signed Keys and Certificates with MinIO

go.mod

@@ -10,9 +10,9 @@ require (
 	github.com/DataDog/zstd v1.3.5 // indirect
 	github.com/alecthomas/participle v0.2.1
 	github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5
+	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
 	github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2
 	github.com/cheggaaa/pb v1.0.28
-	github.com/colinmarc/hdfs/v2 v2.0.0
 	github.com/coredns/coredns v1.4.0
 	github.com/coreos/bbolt v1.3.2 // indirect
 	github.com/coreos/etcd v3.3.12+incompatible
@@ -37,12 +37,15 @@ require (
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.8.5 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
-	github.com/hashicorp/go-retryablehttp v0.5.3 // indirect
+	github.com/hashicorp/go-msgpack v0.5.4 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.0 // indirect
+	github.com/hashicorp/raft v1.0.1 // indirect
 	github.com/hashicorp/vault v1.1.0
 	github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c // indirect
 	github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf
+	github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 // indirect
 	github.com/jonboulle/clockwork v0.1.0 // indirect
+	github.com/json-iterator/go v1.1.6
 	github.com/klauspost/compress v1.4.1 // indirect
 	github.com/klauspost/cpuid v1.2.0 // indirect
 	github.com/klauspost/pgzip v1.2.1
@@ -53,7 +56,8 @@ require (
 	github.com/miekg/dns v1.1.8
 	github.com/minio/blazer v0.0.0-20171126203752-2081f5bf0465
 	github.com/minio/cli v1.3.0
-	github.com/minio/dsync v0.0.0-20190131060523-fb604afd87b2
+	github.com/minio/dsync v1.0.0
+	github.com/minio/hdfs/v3 v3.0.0
 	github.com/minio/highwayhash v1.0.0
 	github.com/minio/lsync v0.0.0-20190207022115-a4e43e3d0887
 	github.com/minio/mc v0.0.0-20190401030144-a1355e50e2e8
@@ -66,11 +70,11 @@ require (
 	github.com/nats-io/go-nats v1.7.2 // indirect
 	github.com/nats-io/go-nats-streaming v0.4.2
 	github.com/nats-io/nats v1.7.2
+	github.com/nats-io/nats-streaming-server v0.14.1 // indirect
 	github.com/nats-io/nkeys v0.0.2 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/ncw/directio v1.0.5
 	github.com/nsqio/go-nsq v1.0.7
-	github.com/pascaldekloe/goe v0.1.0 // indirect
-	github.com/pkg/errors v0.8.1 // indirect
 	github.com/pkg/profile v1.3.0
 	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
 	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect
@@ -93,12 +97,12 @@ require (
 	go.uber.org/atomic v1.3.2
 	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.9.1 // indirect
-	golang.org/x/crypto v0.0.0-20190417174047-f416ebab96af
-	golang.org/x/lint v0.0.0-20190409202823-959b441ac422 // indirect
-	golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2
-	golang.org/x/sys v0.0.0-20190416152802-12500544f89f
-	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
-	golang.org/x/tools v0.0.0-20190417223002-a5870b403859 // indirect
+	golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734
+	golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
+	golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872
+	golang.org/x/text v0.3.2 // indirect
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
 	google.golang.org/api v0.3.0
 	gopkg.in/Shopify/sarama.v1 v1.20.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect

go.sum

@@ -19,6 +19,7 @@ github.com/Azure/go-autorest v11.5.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW
 github.com/Azure/go-autorest v11.7.0+incompatible h1:gzma19dc9ejB75D90E5S+/wXouzpZyA+CV+/MJPSD/k=
 github.com/Azure/go-autorest v11.7.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/zstd v1.3.5 h1:DtpNbljikUepEPD16hD4LvIcmhnhdLTiW/5pHgbmp14=
 github.com/DataDog/zstd v1.3.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/Jeffail/gabs v1.1.1/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc=
@@ -45,6 +46,8 @@ github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61/go.mod h1:ikc1XA58M+Rx
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
+github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.15.31/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
@@ -69,9 +72,9 @@ github.com/cheggaaa/pb v0.0.0-20160713104425-73ae1d68fe0b/go.mod h1:pQciLPpbU0ox
 github.com/cheggaaa/pb v1.0.28 h1:kWGpdAcSp3MxMU9CCHOwz/8V0kCHN4+9yQm2MzWuI98=
 github.com/cheggaaa/pb v1.0.28/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
 github.com/chrismalek/oktasdk-go v0.0.0-20181212195951-3430665dfaa0/go.mod h1:5d8DqS60xkj9k3aXfL3+mXBH0DPYO0FQjcKosxl+b/Q=
+github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
+github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/colinmarc/hdfs/v2 v2.0.0 h1:zXrYPcGbdTS8li50eel904btC8XI35dwDYlTvtFKQe4=
-github.com/colinmarc/hdfs/v2 v2.0.0/go.mod h1:uU5kspcuAB3y3XVcBew3IEXsYDTeyOHBbiBFSqCQHnk=
 github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/coredns/coredns v0.0.0-20180121192821-d4bf076ccf4e/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0=
 github.com/coredns/coredns v1.4.0 h1:RubBkYmkByUqZWWkjRHvNLnUHgkRVqAWgSMmRFvpE1A=
@@ -244,6 +247,8 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh
 github.com/hashicorp/go-memdb v0.0.0-20190306140544-eea0b16292ad/go.mod h1:kbfItVoBJwCfKXDXN4YoAXjxcFVZ7MRrJzyTX6H4giE=
 github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-msgpack v0.5.4 h1:SFT72YqIkOcLdWJUYcriVX7hbrZpwc/f7h8aW2NUqrA=
+github.com/hashicorp/go-msgpack v0.5.4/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/go-plugin v0.0.0-20190220160451-3f118e8ee104/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY=
@@ -275,6 +280,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
 github.com/hashicorp/nomad v0.8.7/go.mod h1:WRaKjdO1G2iqi86TvTjIYtKTyxg4pl7NLr9InxtWaI0=
 github.com/hashicorp/raft v1.0.0 h1:htBVktAOtGs4Le5Z7K8SF5H2+oWsQFYVmOgH5loro7Y=
 github.com/hashicorp/raft v1.0.0/go.mod h1:DVSAWItjLjTOkVbSpWQ0j0kUADIvDaCtBxIcbNAQLkI=
+github.com/hashicorp/raft v1.0.1 h1:94uRdS11oEneUkxmXq6Vg9shNhBILh2UTb9crQjJWl0=
+github.com/hashicorp/raft v1.0.1/go.mod h1:DVSAWItjLjTOkVbSpWQ0j0kUADIvDaCtBxIcbNAQLkI=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hashicorp/vault v0.0.0-20181121181053-d4367e581fe1/go.mod h1:KfSyffbKxoVyspOdlaGVjIuwLobi07qD1bAbosPMpP0=
 github.com/hashicorp/vault v1.1.0 h1:v79NUgO5xCZnXVzUkIqFOXtP8YhpnHAi1fk3eo9cuOE=
@@ -299,12 +306,15 @@ github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7V
 github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg=
 github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930 h1:v4CYlQ+HeysPHsr2QFiEO60gKqnvn1xwvuKhhAhuEkk=
 github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
+github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjLBdCp5PRlCFijNjvcYANOZXzCfXwCM=
+github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
 github.com/jeffchao/backoff v0.0.0-20140404060208-9d7fd7aa17f2/go.mod h1:xkfESuHriIekR+4RoV+fu91j/CfnYM29Zi2tMFw5iD4=
 github.com/jefferai/jsonx v1.0.0/go.mod h1:OGmqmi2tTeI/PS+qQfBDToLHHJIy/RMp24fPo8vFvoQ=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
@@ -367,8 +377,10 @@ github.com/minio/cli v0.0.0-20170227073228-b8ae5507c0ce/go.mod h1:hLsWNQy2wIf3FK
 github.com/minio/cli v1.3.0 h1:vB0iUpmyaH54+1jJJj62Aa0qFF3xO3i0J3IcKiM6bHM=
 github.com/minio/cli v1.3.0/go.mod h1:hLsWNQy2wIf3FKFnMlH69f4RdEyn8nbRA2shaulTjGY=
 github.com/minio/dsync v0.0.0-20190104003057-61c41ffdeea2/go.mod h1:eLQe3mXL0h02kNpPtBJiLr1fIEIJftgXRAjncjQbxJo=
-github.com/minio/dsync v0.0.0-20190131060523-fb604afd87b2 h1:5Aq4Aro/PSNVgoWWTLPX+zcfDY87VhtKPv+7x1ERJ1w=
-github.com/minio/dsync v0.0.0-20190131060523-fb604afd87b2/go.mod h1:eLQe3mXL0h02kNpPtBJiLr1fIEIJftgXRAjncjQbxJo=
+github.com/minio/dsync v1.0.0 h1:l6pQgUPBM41idlR0UOcpAP+EYim9MCwIAUh6sQQI1gk=
+github.com/minio/dsync v1.0.0/go.mod h1:eLQe3mXL0h02kNpPtBJiLr1fIEIJftgXRAjncjQbxJo=
+github.com/minio/hdfs/v3 v3.0.0 h1:yHa9ugB2UeazkiphO6Q2dmbqiKR5lZssN/1vda4gwoY=
+github.com/minio/hdfs/v3 v3.0.0/go.mod h1:k04lEYpgeojX3o1vQep6rQs4MCTD+qlh2xHFEa/BH6A=
 github.com/minio/highwayhash v0.0.0-20181220011308-93ed73d64169/go.mod h1:NL8wme5P5MoscwAkXfGroz3VgpCdhBw3KYOu5mEsvpU=
 github.com/minio/highwayhash v1.0.0 h1:iMSDhgUILCr0TNm8LWlSjF8N0ZIj2qbO8WHp6Q/J2BA=
 github.com/minio/highwayhash v1.0.0/go.mod h1:xQboMTeM9nY9v/LlAOxFctujiv5+Aq2hR5dxBpaMbdc=
@@ -407,7 +419,9 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/pointerstructure v0.0.0-20170205204203-f2329fcfa9e2/go.mod h1:KMNPMpc0BU/kZEgyDhBplsDn/mjnJMhyMjq4MWboN20=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nats-io/gnatsd v1.4.1 h1:RconcfDeWpKCD6QIIwiVFcvForlXpWeJP7i5/lDLy44=
@@ -423,12 +437,16 @@ github.com/nats-io/nats v1.7.2 h1:8hasuHgNFiDGc7MTkGpPFc0v1Jz1nEoQwIHKAH2hFuo=
 github.com/nats-io/nats v1.7.2/go.mod h1:PpmYZwlgTfBI56QypJLfIMOfLnMRuVs+VL6r8mQ2SoQ=
 github.com/nats-io/nats-streaming-server v0.12.2 h1:EpyLfUBZgwu5c0mdSSytQsapm615AyitPssq7jgafdw=
 github.com/nats-io/nats-streaming-server v0.12.2/go.mod h1:RyqtDJZvMZO66YmyjIYdIvS69zu/wDAkyNWa8PIUa5c=
+github.com/nats-io/nats-streaming-server v0.14.1 h1:NLZNqluNLmsijog4x0VXofHGaSdAIH16Ge55maG5ItM=
+github.com/nats-io/nats-streaming-server v0.14.1/go.mod h1:RyqtDJZvMZO66YmyjIYdIvS69zu/wDAkyNWa8PIUa5c=
 github.com/nats-io/nkeys v0.0.2 h1:+qM7QpgXnvDDixitZtQUBDY9w/s9mu1ghS+JIbsrx6M=
 github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4=
 github.com/nats-io/nuid v1.0.0 h1:44QGdhbiANq8ZCbUkdn6W5bqtg+mHuDE4wOUuxxndFs=
 github.com/nats-io/nuid v1.0.0/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/ncw/directio v1.0.5 h1:JSUBhdjEvVaJvOoyPAbcW0fnd0tvRXD76wEfZ1KcQz4=
+github.com/ncw/directio v1.0.5/go.mod h1:rX/pKEYkOXBGOggmcyJeJGloCkleSvphPx2eV3t6ROk=
 github.com/nsqio/go-nsq v0.0.0-20181028195256-0527e80f3ba5/go.mod h1:XP5zaUs3pqf+Q71EqUJs3HYfBIqfK6G83WQMdNN+Ito=
 github.com/nsqio/go-nsq v1.0.7 h1:O0pIZJYTf+x7cZBA0UMY8WxFG79lYTURmWzAAh48ljY=
 github.com/nsqio/go-nsq v1.0.7/go.mod h1:XP5zaUs3pqf+Q71EqUJs3HYfBIqfK6G83WQMdNN+Ito=
@@ -553,6 +571,7 @@ github.com/tidwall/sjson v1.0.4 h1:UcdIRXff12Lpnu3OLtZvnc03g4vH2suXDXhBwBqmzYg=
 github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
 github.com/ugorji/go v0.0.0-20180628102755-7d51bbe6161d/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
 github.com/ugorji/go v1.1.2 h1:JON3E2/GPW2iDNGoSAusl1KDf5TRQ8k8q7Tp097pZGs=
 github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
@@ -592,10 +611,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 h1:bselrhR0Or1vomJZC8ZIjWtbDmn9OYFLX5Ik9alpJpE=
-golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
-golang.org/x/crypto v0.0.0-20190417174047-f416ebab96af h1:6qGQw30u837TXZbCmLFR9AVA+RjJU1LIbvk0oIkDZGY=
-golang.org/x/crypto v0.0.0-20190417174047-f416ebab96af/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -604,8 +621,6 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422 h1:QzoH/1pFpZguR8NrRHLcO6jKqfv2zpuSqZLgdm7ZmjI=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -625,8 +640,8 @@ golang.org/x/net v0.0.0-20190318221613-d196dffd7c2b/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190324223953-e3b2ff56ed87/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2 h1:iC0Y6EDq+rhnAePxGvJs2kzUAYcwESqdcGRPzEUfzTU=
-golang.org/x/net v0.0.0-20190415214537-1da14a5a36f2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6 h1:FP8hkuE6yUEaJnK7O2eTuejKWwW+Rhfj80dQ2JcKxCU=
+golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/oauth2 v0.0.0-20180603041954-1e0a3fa8ba9a/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -639,6 +654,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180828065106-d99a578cf41b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -661,21 +678,21 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190304154630-e844e0132e93/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190318195719-6c81ef8f67ca/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e h1:nFYrTHrdrAOpShe27kaFHjsqYSEQ0KWqdWLu3xuZJts=
-golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67 h1:1Fzlr8kkDLQwqMP8GxrhptBLqZG/EDpiATneiZHY998=
-golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190416152802-12500544f89f h1:1ZH9RnjNgLzh6YrsRp/c6ddZ8Lq0fq9xztNOoWJ2sz4=
-golang.org/x/sys v0.0.0-20190416152802-12500544f89f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872 h1:cGjJzUd8RgBw428LXP65YXni0aiGNA4Bl+ls8SmLOm8=
+golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -683,8 +700,6 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190318200714-bb1270c20edf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190417223002-a5870b403859 h1:9IwAPkJAVzY3ES7jydtY2zc5YltyS6lQuXUJoqoWQYg=
-golang.org/x/tools v0.0.0-20190417223002-a5870b403859/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 google.golang.org/api v0.0.0-20180603000442-8e296ef26005/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20180916000451-19ff8768a5c0/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=

pkg/disk/directio_darwin.go

@@ -0,0 +1,36 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package disk
+
+import (
+	"os"
+
+	"github.com/ncw/directio"
+	"golang.org/x/sys/unix"
+)
+
+// OpenFileDirectIO - bypass kernel cache.
+func OpenFileDirectIO(filePath string, flag int, perm os.FileMode) (*os.File, error) {
+	return directio.OpenFile(filePath, flag, perm)
+}
+
+// DisableDirectIO - disables directio mode.
+func DisableDirectIO(f *os.File) error {
+	fd := f.Fd()
+	_, err := unix.FcntlInt(fd, unix.F_NOCACHE, 0)
+	return err
+}

pkg/disk/directio_unix.go

@@ -0,0 +1,44 @@
+// +build linux netbsd freebsd
+
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package disk
+
+import (
+	"os"
+	"syscall"
+
+	"github.com/ncw/directio"
+	"golang.org/x/sys/unix"
+)
+
+// OpenFileDirectIO - bypass kernel cache.
+func OpenFileDirectIO(filePath string, flag int, perm os.FileMode) (*os.File, error) {
+	return directio.OpenFile(filePath, flag, perm)
+}
+
+// DisableDirectIO - disables directio mode.
+func DisableDirectIO(f *os.File) error {
+	fd := f.Fd()
+	flag, err := unix.FcntlInt(fd, unix.F_GETFL, 0)
+	if err != nil {
+		return err
+	}
+	flag = flag & ^(syscall.O_DIRECT)
+	_, err = unix.FcntlInt(fd, unix.F_SETFL, flag)
+	return err
+}

pkg/disk/directio_unsupported.go

@@ -0,0 +1,35 @@
+// +build !linux,!netbsd,!freebsd,!darwin
+
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package disk
+
+import (
+	"os"
+)
+
+// OpenBSD and Windows not supported.
+// On OpenBSD O_DIRECT is not supported
+// On Windows there is no documentation on disabling O_DIRECT
+
+func OpenFileDirectIO(filePath string, flag int, perm os.FileMode) (*os.File, error) {
+	return os.OpenFile(filePath, flag, perm)
+}
+
+func DisableDirectIO(f *os.File) error {
+	return nil
+}

pkg/madmin/API.md

@@ -1,549 +0,0 @@
-# Golang Admin Client API Reference [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
-
-## Initialize MinIO Admin Client object.
-
-##  MinIO
-
-```go
-
-package main
-
-import (
-    "fmt"
-
-    "github.com/minio/minio/pkg/madmin"
-)
-
-func main() {
-    // Use a secure connection.
-    ssl := true
-
-    // Initialize minio client object.
-    mdmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETKEY", ssl)
-    if err != nil {
-        fmt.Println(err)
-        return
-    }
-
-    // Fetch service status.
-    st, err := mdmClnt.ServiceStatus()
-    if err != nil {
-        fmt.Println(err)
-        return
-    }
-    fmt.Printf("%#v\n", st)
-}
-
-```
-
-| Service operations         | Info operations  | Healing operations                    | Config operations        | Top operations        | IAM operations | Misc                                |
-|:----------------------------|:----------------------------|:--------------------------------------|:--------------------------|:--------------------------|:------------------------------------|:------------------------------------|
-| [`ServiceStatus`](#ServiceStatus) | [`ServerInfo`](#ServerInfo) | [`Heal`](#Heal) | [`GetConfig`](#GetConfig) | [`TopLocks`](#TopLocks) | [`AddUser`](#AddUser) | [`SetAdminCredentials`](#SetAdminCredentials) |
-| [`ServiceSendAction`](#ServiceSendAction) | [`ServerCPULoadInfo`](#ServerCPULoadInfo) | | [`SetConfig`](#SetConfig) | |  [`SetUserPolicy`](#SetUserPolicy) | [`StartProfiling`](#StartProfiling) |
-| |[`ServerMemUsageInfo`](#ServerMemUsageInfo) |            | [`GetConfigKeys`](#GetConfigKeys) | | [`ListUsers`](#ListUsers) | [`DownloadProfilingData`](#DownloadProfilingData) |
-| | |            | [`SetConfigKeys`](#SetConfigKeys) | | [`AddCannedPolicy`](#AddCannedPolicy) | |
-
-
-## 1. Constructor
-<a name="MinIO"></a>
-
-### New(endpoint string, accessKeyID string, secretAccessKey string, ssl bool) (*AdminClient, error)
-Initializes a new admin client object.
-
-__Parameters__
-
-|Param   |Type   |Description   |
-|:---|:---| :---|
-|`endpoint`   | _string_  |MinIO endpoint.   |
-|`accessKeyID`  |_string_   | Access key for the object storage endpoint.  |
-|`secretAccessKey`  | _string_  |Secret key for the object storage endpoint.   |
-|`ssl`   | _bool_  | Set this value to 'true' to enable secure (HTTPS) access.  |
-
-## 2. Admin API Version
-
-<a name="VersionInfo"></a>
-### VersionInfo() (AdminAPIVersionInfo, error)
-Fetch server's supported Administrative API version.
-
- __Example__
-
-``` go
-
-	info, err := madmClnt.VersionInfo()
-	if err != nil {
-		log.Fatalln(err)
-	}
-	log.Printf("%s\n", info.Version)
-
-```
-
-## 3. Service operations
-
-<a name="ServiceStatus"></a>
-### ServiceStatus() (ServiceStatusMetadata, error)
-Fetch service status, replies disk space used, backend type and total disks offline/online (applicable in distributed mode).
-
-| Param  | Type  | Description  |
-|---|---|---|
-|`serviceStatus`  | _ServiceStatusMetadata_  | Represents current server status info in following format: |
-
-
-| Param  | Type  | Description  |
-|---|---|---|
-|`st.ServerVersion.Version`  | _string_  | Server version. |
-|`st.ServerVersion.CommitID`  | _string_  | Server commit id. |
-|`st.Uptime` | _time.Duration_ | Server uptime duration in seconds. |
-
- __Example__
-
- ```go
-
-	st, err := madmClnt.ServiceStatus()
-	if err != nil {
-		log.Fatalln(err)
-	}
-	log.Printf("%#v\n", st)
-
- ```
-
-<a name="ServiceSendAction"></a>
-### ServiceSendAction(act ServiceActionValue) (error)
-Sends a service action command to service - possible actions are restarting and stopping the server.
-
- __Example__
-
-
- ```go
-        // to restart
-	st, err := madmClnt.ServiceSendAction(ServiceActionValueRestart)
-        // or to stop
-        // st, err := madmClnt.ServiceSendAction(ServiceActionValueStop)
-	if err != nil {
-		log.Fatalln(err)
-	}
-	log.Printf("Success")
- ```
-
-## 4. Info operations
-
-<a name="ServerInfo"></a>
-### ServerInfo() ([]ServerInfo, error)
-Fetches information for all cluster nodes, such as server properties, storage information, network statistics, etc.
-
-| Param                           | Type               | Description                                                        |
-|---------------------------------|--------------------|--------------------------------------------------------------------|
-| `si.Addr`                       | _string_           | Address of the server the following information is retrieved from. |
-| `si.ConnStats`                  | _ServerConnStats_  | Connection statistics from the given server.                       |
-| `si.HTTPStats`                  | _ServerHTTPStats_  | HTTP connection statistics from the given server.                  |
-| `si.Properties`                 | _ServerProperties_ | Server properties such as region, notification targets.            |
-| `si.Data.StorageInfo.Used`      | _int64_            | Used disk space.                                                   |
-| `si.Data.StorageInfo.Total`     | _int64_            | Total disk space.                                                  |
-| `si.Data.StorageInfo.Available` | _int64_            | Available disk space.                                              |
-| `si.Data.StorageInfo.Backend`   | _struct{}_         | Represents backend type embedded structure.                        |
-
-| Param                       | Type            | Description                                        |
-|-----------------------------|-----------------|----------------------------------------------------|
-| `ServerProperties.Uptime`   | _time.Duration_ | Total duration in seconds since server is running. |
-| `ServerProperties.Version`  | _string_        | Current server version.                            |
-| `ServerProperties.CommitID` | _string_        | Current server commitID.                           |
-| `ServerProperties.Region`   | _string_        | Configured server region.                          |
-| `ServerProperties.SQSARN`   | _[]string_      | List of notification target ARNs.                  |
-
-| Param                              | Type     | Description                         |
-|------------------------------------|----------|-------------------------------------|
-| `ServerConnStats.TotalInputBytes`  | _uint64_ | Total bytes received by the server. |
-| `ServerConnStats.TotalOutputBytes` | _uint64_ | Total bytes sent by the server.     |
-
-| Param                                | Type                    | Description                                             |
-|--------------------------------------|-------------------------|---------------------------------------------------------|
-| `ServerHTTPStats.TotalHEADStats`     | _ServerHTTPMethodStats_ | Total statistics regarding HEAD operations              |
-| `ServerHTTPStats.SuccessHEADStats`   | _ServerHTTPMethodStats_ | Total statistics regarding successful HEAD operations   |
-| `ServerHTTPStats.TotalGETStats`      | _ServerHTTPMethodStats_ | Total statistics regarding GET operations               |
-| `ServerHTTPStats.SuccessGETStats`    | _ServerHTTPMethodStats_ | Total statistics regarding successful GET operations    |
-| `ServerHTTPStats.TotalPUTStats`      | _ServerHTTPMethodStats_ | Total statistics regarding PUT operations               |
-| `ServerHTTPStats.SuccessPUTStats`    | _ServerHTTPMethodStats_ | Total statistics regarding successful PUT operations    |
-| `ServerHTTPStats.TotalPOSTStats`     | _ServerHTTPMethodStats_ | Total statistics regarding POST operations              |
-| `ServerHTTPStats.SuccessPOSTStats`   | _ServerHTTPMethodStats_ | Total statistics regarding successful POST operations   |
-| `ServerHTTPStats.TotalDELETEStats`   | _ServerHTTPMethodStats_ | Total statistics regarding DELETE operations            |
-| `ServerHTTPStats.SuccessDELETEStats` | _ServerHTTPMethodStats_ | Total statistics regarding successful DELETE operations |
-
-
-| Param                               | Type     | Description                                     |
-|-------------------------------------|----------|-------------------------------------------------|
-| `ServerHTTPMethodStats.Count`       | _uint64_ | Total number of operations.                     |
-| `ServerHTTPMethodStats.AvgDuration` | _string_ | Average duration of Count number of operations. |
-
-| Param                      | Type            | Description                                                                       |
-|----------------------------|-----------------|-----------------------------------------------------------------------------------|
-| `Backend.Type`             | _BackendType_   | Type of backend used by the server currently only FS or Erasure.                  |
-| `Backend.OnlineDisks`      | _int_           | Total number of disks online (only applies to Erasure backend), is empty for FS.  |
-| `Backend.OfflineDisks`     | _int_           | Total number of disks offline (only applies to Erasure backend), is empty for FS. |
-| `Backend.StandardSCData`   | _int_           | Data disks set for standard storage class, is empty for FS.                       |
-| `Backend.StandardSCParity` | _int_           | Parity disks set for standard storage class, is empty for FS.                     |
-| `Backend.RRSCData`         | _int_           | Data disks set for reduced redundancy storage class, is empty for FS.             |
-| `Backend.RRSCParity`       | _int_           | Parity disks set for reduced redundancy storage class, is empty for FS.           |
-| `Backend.Sets`             | _[][]DriveInfo_ | Represents topology of drives in erasure coded sets.                              |
-
-| Param                | Type     | Description                                           |
-|----------------------|----------|-------------------------------------------------------|
-| `DriveInfo.UUID`     | _string_ | Unique ID for each disk provisioned by server format. |
-| `DriveInfo.Endpoint` | _string_ | Endpoint location of the remote/local disk.           |
-| `DriveInfo.State`    | _string_ | Current state of the disk at endpoint.                |
-
- __Example__
-
- ```go
-
-	serversInfo, err := madmClnt.ServerInfo()
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	for _, peerInfo := range serversInfo {
-		log.Printf("Node: %s, Info: %v\n", peerInfo.Addr, peerInfo.Data)
-	}
-
- ```
-
-<a name="ServerDrivesPerfInfo"></a>
-### ServerDrivesPerfInfo() ([]ServerDrivesPerfInfo, error)
-
-Fetches drive performance information for all cluster nodes. Returned value is in Bytes/s.
-
-| Param | Type | Description |
-|---|---|---|
-|`di.Addr` | _string_ | Address of the server the following information is retrieved from. |
-|`di.Error` | _string _ | Errors (if any) encountered while reaching this node |
-|`di.DrivesPerf` | _disk.Performance_ | Path of the drive mount on above server and read, write speed. |
-
-| Param | Type | Description |
-|---|---|---|
-|`disk.Performance.Path` | _string_ | Path of drive mount. |
-|`disk.Performance.Error` | _string_ | Error (if any) encountered while accessing this drive. |
-|`disk.Performance.WriteSpeed` | _float64_ | Write speed on above path in Bytes/s. |
-|`disk.Performance.ReadSpeed` | _float64_ | Read speed on above path in Bytes/s. |
-
-<a name="ServerCPULoadInfo"></a>
-### ServerCPULoadInfo() ([]ServerCPULoadInfo, error)
-
-Fetches CPU utilization for all cluster nodes. Returned value is in Bytes.
-
-| Param | Type | Description |
-|-------|------|-------------|
-|`cpui.Addr` | _string_ | Address of the server the following information  is retrieved from. |
-|`cpui.Error` | _string_ | Errors (if any) encountered while reaching this node |
-|`cpui.CPULoad` | _cpu.Load_ | The load on the CPU. |
-
-| Param | Type | Description |
-|-------|------|-------------|
-|`cpu.Load.Avg` | _float64_ | The average utilization of the CPU measured in a 200ms interval |
-|`cpu.Load.Min` | _float64_ | The minimum utilization of the CPU measured in a 200ms interval |
-|`cpu.Load.Max` | _float64_ | The maximum utilization of the CPU measured in a 200ms interval |
-|`cpu.Load.Error` | _string_ | Error (if any) encountered while accesing the CPU info |
-
-<a name="ServerMemUsageInfo"></a>
-### ServerMemUsageInfo() ([]ServerMemUsageInfo, error)
-
-Fetches Mem utilization for all cluster nodes. Returned value is in Bytes.
-
-| Param | Type | Description |
-|-------|------|-------------|
-|`memi.Addr` | _string_ | Address of the server the following information  is retrieved from. |
-|`memi.Error` | _string_ | Errors (if any) encountered while reaching this node |
-|`memi.MemUsage` | _mem.Usage_ | The utilitzation of Memory |
-
-| Param | Type | Description |
-|-------|------|-------------|
-|`mem.Usage.Mem` | _uint64_ | The total number of bytes obtained from the OS |
-|`mem.Usage.Error` | _string_ | Error (if any) encountered while accesing the CPU info |
-
-## 6. Heal operations
-
-<a name="Heal"></a>
-### Heal(bucket, prefix string, healOpts HealOpts, clientToken string, forceStart bool, forceStop bool) (start HealStartSuccess, status HealTaskStatus, err error)
-
-Start a heal sequence that scans data under given (possible empty)
-`bucket` and `prefix`. The `recursive` bool turns on recursive
-traversal under the given path. `dryRun` does not mutate on-disk data,
-but performs data validation.
-
-Two heal sequences on overlapping paths may not be initiated.
-
-The progress of a heal should be followed using the same API `Heal`
-by providing the `clientToken` previously obtained from a `Heal`
-API. The server accumulates results of the heal traversal and waits
-for the client to receive and acknowledge them using the status
-request by providing `clientToken`.
-
-__Example__
-
-``` go
-
-    opts := madmin.HealOpts{
-            Recursive: true,
-            DryRun:    false,
-    }
-    forceStart := false
-    forceStop := false
-    healPath, err := madmClnt.Heal("", "", opts, "", forceStart, forceStop)
-    if err != nil {
-        log.Fatalln(err)
-    }
-    log.Printf("Heal sequence started at %s", healPath)
-
-```
-
-#### HealStartSuccess structure
-
-| Param | Type | Description |
-|----|--------|--------|
-| s.ClientToken | _string_ | A unique token for a successfully started heal operation, this token is used to request realtime progress of the heal operation. |
-| s.ClientAddress | _string_ | Address of the client which initiated the heal operation, the client address has the form "host:port".|
-| s.StartTime | _time.Time_ | Time when heal was initially started.|
-
-#### HealTaskStatus structure
-
-| Param | Type | Description |
-|----|--------|--------|
-| s.Summary | _string_ | Short status of heal sequence |
-| s.FailureDetail | _string_ | Error message in case of heal sequence failure |
-| s.HealSettings | _HealOpts_ | Contains the booleans set in the `HealStart` call |
-| s.Items | _[]HealResultItem_ | Heal records for actions performed by server |
-
-#### HealResultItem structure
-
-| Param | Type | Description |
-|------|-------|---------|
-| ResultIndex | _int64_ | Index of the heal-result record |
-| Type | _HealItemType_ | Represents kind of heal operation in the heal record |
-| Bucket | _string_ | Bucket name |
-| Object | _string_ | Object name |
-| Detail | _string_ | Details about heal operation |
-| DiskInfo.AvailableOn | _[]int_ | List of disks on which the healed entity is present and healthy |
-| DiskInfo.HealedOn | _[]int_ | List of disks on which the healed entity was restored |
-
-## 7. Config operations
-
-<a name="GetConfig"></a>
-### GetConfig() ([]byte, error)
-Get current `config.json` of a MinIO server.
-
-__Example__
-
-``` go
-    configBytes, err := madmClnt.GetConfig()
-    if err != nil {
-        log.Fatalf("failed due to: %v", err)
-    }
-
-    // Pretty-print config received as json.
-    var buf bytes.Buffer
-    err = json.Indent(buf, configBytes, "", "\t")
-    if err != nil {
-        log.Fatalf("failed due to: %v", err)
-    }
-
-    log.Println("config received successfully: ", string(buf.Bytes()))
-```
-
-
-<a name="SetConfig"></a>
-### SetConfig(config io.Reader) error
-Set a new `config.json` for a MinIO server.
-
-__Example__
-
-``` go
-    config := bytes.NewReader([]byte(`config.json contents go here`))
-    if err := madmClnt.SetConfig(config); err != nil {
-        log.Fatalf("failed due to: %v", err)
-    }
-    log.Println("SetConfig was successful")
-```
-
-<a name="GetConfigKeys"></a>
-### GetConfigKeys(keys []string) ([]byte, error)
-Get a json document which contains a set of keys and their values from config.json.
-
-__Example__
-
-``` go
-    configBytes, err := madmClnt.GetConfigKeys([]string{"version", "notify.amqp.1"})
-    if err != nil {
-        log.Fatalf("failed due to: %v", err)
-    }
-
-    // Pretty-print config received as json.
-    var buf bytes.Buffer
-    err = json.Indent(buf, configBytes, "", "\t")
-    if err != nil {
-        log.Fatalf("failed due to: %v", err)
-    }
-
-    log.Println("config received successfully: ", string(buf.Bytes()))
-```
-
-
-<a name="SetConfigKeys"></a>
-### SetConfigKeys(params map[string]string) error
-Set a set of keys and values for MinIO server or distributed setup and restart the MinIO
-server for the new configuration changes to take effect.
-
-__Example__
-
-``` go
-    err := madmClnt.SetConfigKeys(map[string]string{"notify.webhook.1": "{\"enable\": true, \"endpoint\": \"http://example.com/api\"}"})
-    if err != nil {
-        log.Fatalf("failed due to: %v", err)
-    }
-
-    log.Println("New configuration successfully set")
-```
-
-## 8. Top operations
-
-<a name="TopLocks"></a>
-### TopLocks() (LockEntries, error)
-Get the oldest locks from MinIO server.
-
-__Example__
-
-``` go
-    locks, err := madmClnt.TopLocks()
-    if err != nil {
-        log.Fatalf("failed due to: %v", err)
-    }
-
-    out, err := json.Marshal(locks)
-    if err != nil {
-        log.Fatalf("Marshal failed due to: %v", err)
-    }
-
-    log.Println("TopLocks received successfully: ", string(out))
-```
-
-## 9. IAM operations
-
-<a name="AddCannedPolicy"></a>
-### AddCannedPolicy(policyName string, policy string) error
-Create a new canned policy on MinIO server.
-
-__Example__
-
-```
-	policy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject"],"Effect": "Allow","Resource": ["arn:aws:s3:::my-bucketname/*"],"Sid": ""}]}`
-
-    if err = madmClnt.AddCannedPolicy("get-only", policy); err != nil {
-		log.Fatalln(err)
-	}
-```
-
-<a name="AddUser"></a>
-### AddUser(user string, secret string) error
-Add a new user on a MinIO server.
-
-__Example__
-
-``` go
-	if err = madmClnt.AddUser("newuser", "newstrongpassword"); err != nil {
-		log.Fatalln(err)
-	}
-```
-
-<a name="SetUserPolicy"></a>
-### SetUserPolicy(user string, policyName string) error
-Enable a canned policy `get-only` for a given user on MinIO server.
-
-__Example__
-
-``` go
-	if err = madmClnt.SetUserPolicy("newuser", "get-only"); err != nil {
-		log.Fatalln(err)
-	}
-```
-
-<a name="ListUsers"></a>
-### ListUsers() (map[string]UserInfo, error)
-Lists all users on MinIO server.
-
-__Example__
-
-``` go
-	users, err := madmClnt.ListUsers();
-    if err != nil {
-		log.Fatalln(err)
-	}
-    for k, v := range users {
-        fmt.Printf("User %s Status %s\n", k, v.Status)
-    }
-```
-
-## 10. Misc operations
-
-<a name="SetAdminCredentials"></a>
-### SetAdminCredentials() error
-Set new credentials of a MinIO setup.
-
-__Example__
-
-``` go
-    err = madmClnt.SetAdminCredentials("YOUR-NEW-ACCESSKEY", "YOUR-NEW-SECRETKEY")
-    if err != nil {
-            log.Fatalln(err)
-    }
-    log.Println("New credentials successfully set.")
-
-```
-
-<a name="StartProfiling"></a>
-### StartProfiling(profiler string) error
-Ask all nodes to start profiling using the specified profiler mode
-
-__Example__
-
-``` go
-    startProfilingResults, err = madmClnt.StartProfiling("cpu")
-    if err != nil {
-            log.Fatalln(err)
-    }
-    for _, result := range startProfilingResults {
-        if !result.Success {
-            log.Printf("Unable to start profiling on node `%s`, reason = `%s`\n", result.NodeName, result.Error)
-        } else {
-            log.Printf("Profiling successfully started on node `%s`\n", result.NodeName)
-        }
-    }
-
-```
-
-<a name="DownloadProfilingData"></a>
-### DownloadProfilingData() ([]byte, error)
-Download profiling data of all nodes in a zip format.
-
-__Example__
-
-``` go
-    profilingData, err := madmClnt.DownloadProfilingData()
-    if err != nil {
-            log.Fatalln(err)
-    }
-
-    profilingFile, err := os.Create("/tmp/profiling-data.zip")
-    if err != nil {
-            log.Fatal(err)
-    }
-
-    if _, err := io.Copy(profilingFile, profilingData); err != nil {
-            log.Fatal(err)
-    }
-
-    if err := profilingFile.Close(); err != nil {
-            log.Fatal(err)
-    }
-
-    if err := profilingData.Close(); err != nil {
-            log.Fatal(err)
-    }
-
-    log.Println("Profiling data successfully downloaded.")
-```

pkg/madmin/README.md

@@ -1,120 +1,539 @@
-# MinIO Admin Library. [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
+# Golang Admin Client API Reference [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
 The MinIO Admin Golang Client SDK provides APIs to manage MinIO services.
 
 This quickstart guide will show you how to install the MinIO Admin client SDK, connect to MinIO admin service, and provide a walkthrough of a simple file uploader.
 
 This document assumes that you have a working [Golang setup](https://docs.min.io/docs/how-to-install-golang).
 
-## Download from GitHub
-
-```sh
-
-go get -u github.com/minio/minio/pkg/madmin
-
-```
-
-## Initialize MinIO Admin Client
-
-You need four items to connect to MinIO admin services.
-
-
-| Parameter  | Description|
-| :---         |     :---     |
-| endpoint   | URL to object storage service.   |
-| accessKeyID | Access key is the user ID that uniquely identifies your account. |
-| secretAccessKey | Secret key is the password to your account. |
-| secure | Set this value to 'true' to enable secure (HTTPS) access. |
+## Initialize MinIO Admin Client object.
 
+##  MinIO
 
 ```go
 
 package main
 
 import (
-	"github.com/minio/minio/pkg/madmin"
-	"log"
-)
-
-func main() {
-	endpoint := "your-minio.example.com:9000"
-	accessKeyID := "YOUR-ACCESSKEYID"
-	secretAccessKey := "YOUR-SECRETKEY"
-	useSSL := true
-
-	// Initialize minio admin client object.
-	madmClnt, err := madmin.New(endpoint, accessKeyID, secretAccessKey, useSSL)
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	log.Println("%v", madmClnt) // MinIO admin client is now setup
-}
-```
-
-## Quick Start Example - Server Info
-
-This example program connects to minio server, gets the current disk status and other useful server information.
-
-#### ServiceStatus.go
-
-```go
-package main
-
-import (
-	"encoding/json"
     "fmt"
-	"log"
 
     "github.com/minio/minio/pkg/madmin"
 )
 
 func main() {
-	endpoint := "your-minio.example.com:9000"
-	accessKeyID := "YOUR-ACCESSKEYID"
-	secretAccessKey := "YOUR-SECRETKEY"
-	useSSL := true
+    // Use a secure connection.
+    ssl := true
 
-	// Initialize minio admin client.
-	mdmClnt, err := madmin.New(endpoint, accessKeyID, secretAccessKey, useSSL)
+    // Initialize minio client object.
+    mdmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETKEY", ssl)
     if err != nil {
-		log.Fatalln(err)
+        fmt.Println(err)
+        return
     }
 
-	si, err := mdmClnt.ServerInfo()
+    // Fetch service status.
+    st, err := mdmClnt.ServiceStatus()
     if err != nil {
-		log.Fatalln(err)
+        fmt.Println(err)
+        return
     }
-	b, err := json.Marshal(si)
-	fmt.Printf("%s\n", string(b))
+    fmt.Printf("%#v\n", st)
 }
+
 ```
 
-Replace the endpoint and access credentials above according to an actual setup.
+| Service operations                        | Info operations                             | Healing operations | Config operations                 | Top operations          | IAM operations                        | Misc                                              |
+|:------------------------------------------|:--------------------------------------------|:-------------------|:----------------------------------|:------------------------|:--------------------------------------|:--------------------------------------------------|
+| [`ServiceStatus`](#ServiceStatus)         | [`ServerInfo`](#ServerInfo)                 | [`Heal`](#Heal)    | [`GetConfig`](#GetConfig)         | [`TopLocks`](#TopLocks) | [`AddUser`](#AddUser)                 |                                                   |
+| [`ServiceSendAction`](#ServiceSendAction) | [`ServerCPULoadInfo`](#ServerCPULoadInfo)   |                    | [`SetConfig`](#SetConfig)         |                         | [`SetUserPolicy`](#SetUserPolicy)     | [`StartProfiling`](#StartProfiling)               |
+|                                           | [`ServerMemUsageInfo`](#ServerMemUsageInfo) |                    | [`GetConfigKeys`](#GetConfigKeys) |                         | [`ListUsers`](#ListUsers)             | [`DownloadProfilingData`](#DownloadProfilingData) |
+|                                           |                                             |                    | [`SetConfigKeys`](#SetConfigKeys) |                         | [`AddCannedPolicy`](#AddCannedPolicy) |                                                   |
 
-#### Run ServiceStatus
 
-The sample output below shows the result of executing the above program against a locally hosted server.
+## 1. Constructor
+<a name="MinIO"></a>
+
+### New(endpoint string, accessKeyID string, secretAccessKey string, ssl bool) (*AdminClient, error)
+Initializes a new admin client object.
+
+__Parameters__
+
+| Param             | Type     | Description                                               |
+|:------------------|:---------|:----------------------------------------------------------|
+| `endpoint`        | _string_ | MinIO endpoint.                                           |
+| `accessKeyID`     | _string_ | Access key for the object storage endpoint.               |
+| `secretAccessKey` | _string_ | Secret key for the object storage endpoint.               |
+| `ssl`             | _bool_   | Set this value to 'true' to enable secure (HTTPS) access. |
+
+## 2. Admin API Version
+
+<a name="VersionInfo"></a>
+### VersionInfo() (AdminAPIVersionInfo, error)
+Fetch server's supported Administrative API version.
+
+ __Example__
+
+``` go
+
+	info, err := madmClnt.VersionInfo()
+	if err != nil {
+		log.Fatalln(err)
+	}
+	log.Printf("%s\n", info.Version)
 
-```sh
-[{"error":"","addr":"localhost:9000","data":{"storage":{"Total":460373336064,"Free":77001187328,"Backend":{"Type":2,"OnlineDisks":4,"OfflineDisks":0,"StandardSCParity":2,"RRSCParity":2}},"network":{"transferred":30599,"received":36370},"http":{"totalHEADs":{"count":0,"avgDuration":"0s"},"successHEADs":{"count":0,"avgDuration":"0s"},"totalGETs":{"count":11,"avgDuration":"0s"},"successGETs":{"count":11,"avgDuration":"0s"},"totalPUTs":{"count":0,"avgDuration":"0s"},"successPUTs":{"count":0,"avgDuration":"0s"},"totalPOSTs":{"count":0,"avgDuration":"0s"},"successPOSTs":{"count":0,"avgDuration":"0s"},"totalDELETEs":{"count":0,"avgDuration":"0s"},"successDELETEs":{"count":0,"avgDuration":"0s"}},"server":{"uptime":596915001694,"version":"2018-01-18T20:33:21Z","commitID":"e2d5a87b2676e3e01f0f4fa7ebd01205364cfb16","region":"us-east-1","sqsARN":null}}},{"error":"","addr":"minio2:9000","data":{"storage":{"Total":460373336064,"Free":77001187328,"Backend":{"Type":2,"OnlineDisks":4,"OfflineDisks":0,"StandardSCParity":2,"RRSCParity":2}},"network":{"transferred":28538,"received":11845},"http":{"totalHEADs":{"count":0,"avgDuration":"0s"},"successHEADs":{"count":0,"avgDuration":"0s"},"totalGETs":{"count":0,"avgDuration":"0s"},"successGETs":{"count":0,"avgDuration":"0s"},"totalPUTs":{"count":0,"avgDuration":"0s"},"successPUTs":{"count":0,"avgDuration":"0s"},"totalPOSTs":{"count":0,"avgDuration":"0s"},"successPOSTs":{"count":0,"avgDuration":"0s"},"totalDELETEs":{"count":0,"avgDuration":"0s"},"successDELETEs":{"count":0,"avgDuration":"0s"}},"server":{"uptime":595852367296,"version":"2018-01-18T20:33:21Z","commitID":"e2d5a87b2676e3e01f0f4fa7ebd01205364cfb16","region":"us-east-1","sqsARN":null}}},{"error":"","addr":"minio3:9000","data":{"storage":{"Total":460373336064,"Free":77001187328,"Backend":{"Type":2,"OnlineDisks":4,"OfflineDisks":0,"StandardSCParity":2,"RRSCParity":2}},"network":{"transferred":27624,"received":11708},"http":{"totalHEADs":{"count":0,"avgDuration":"0s"},"successHEADs":{"count":0,"avgDuration":"0s"},"totalGETs":{"count":0,"avgDuration":"0s"},"successGETs":{"count":0,"avgDuration":"0s"},"totalPUTs":{"count":0,"avgDuration":"0s"},"successPUTs":{"count":0,"avgDuration":"0s"},"totalPOSTs":{"count":0,"avgDuration":"0s"},"successPOSTs":{"count":0,"avgDuration":"0s"},"totalDELETEs":{"count":0,"avgDuration":"0s"},"successDELETEs":{"count":0,"avgDuration":"0s"}},"server":{"uptime":595831126778,"version":"2018-01-18T20:33:21Z","commitID":"e2d5a87b2676e3e01f0f4fa7ebd01205364cfb16","region":"us-east-1","sqsARN":null}}},{"error":"","addr":"minio4:9000","data":{"storage":{"Total":460373336064,"Free":77001187328,"Backend":{"Type":2,"OnlineDisks":4,"OfflineDisks":0,"StandardSCParity":2,"RRSCParity":2}},"network":{"transferred":27740,"received":12116},"http":{"totalHEADs":{"count":0,"avgDuration":"0s"},"successHEADs":{"count":0,"avgDuration":"0s"},"totalGETs":{"count":0,"avgDuration":"0s"},"successGETs":{"count":0,"avgDuration":"0s"},"totalPUTs":{"count":0,"avgDuration":"0s"},"successPUTs":{"count":0,"avgDuration":"0s"},"totalPOSTs":{"count":0,"avgDuration":"0s"},"successPOSTs":{"count":0,"avgDuration":"0s"},"totalDELETEs":{"count":0,"avgDuration":"0s"},"successDELETEs":{"count":0,"avgDuration":"0s"}},"server":{"uptime":595349958375,"version":"2018-01-18T20:33:21Z","commitID":"e2d5a87b2676e3e01f0f4fa7ebd01205364cfb16","region":"us-east-1","sqsARN":null}}}]
 ```
 
-## API Reference
+## 3. Service operations
 
-### API Reference : Service Operations
+<a name="ServiceStatus"></a>
+### ServiceStatus() (ServiceStatusMetadata, error)
+Fetch service status, replies disk space used, backend type and total disks offline/online (applicable in distributed mode).
 
-* [`ServiceStatus`](./API.md#ServiceStatus)
-* [`ServiceRestart`](./API.md#ServiceRestart)
-* [`ServiceSetCredentials`](./API.md#ServiceSetCredentials)
+| Param           | Type                    | Description                                                |
+|-----------------|-------------------------|------------------------------------------------------------|
+| `serviceStatus` | _ServiceStatusMetadata_ | Represents current server status info in following format: |
 
-## Full Examples
 
-#### Full Examples : Service Operations
+| Param                       | Type            | Description                        |
+|-----------------------------|-----------------|------------------------------------|
+| `st.ServerVersion.Version`  | _string_        | Server version.                    |
+| `st.ServerVersion.CommitID` | _string_        | Server commit id.                  |
+| `st.Uptime`                 | _time.Duration_ | Server uptime duration in seconds. |
 
-* [service-status.go](https://github.com/minio/minio/blob/master/pkg/madmin/examples/service-status.go)
-* [service-restart.go](https://github.com/minio/minio/blob/master/pkg/madmin/examples/service-restart.go)
-* [set-credentials.go](https://github.com/minio/minio/blob/master/pkg/madmin/examples/set-credentials.go)
+ __Example__
 
-## Contribute
+ ```go
 
-[Contributors Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
+	st, err := madmClnt.ServiceStatus()
+	if err != nil {
+		log.Fatalln(err)
+	}
+	log.Printf("%#v\n", st)
+
+ ```
+
+<a name="ServiceSendAction"></a>
+### ServiceSendAction(act ServiceActionValue) (error)
+Sends a service action command to service - possible actions are restarting and stopping the server.
+
+ __Example__
+
+
+ ```go
+        // to restart
+	st, err := madmClnt.ServiceSendAction(ServiceActionValueRestart)
+        // or to stop
+        // st, err := madmClnt.ServiceSendAction(ServiceActionValueStop)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	log.Printf("Success")
+ ```
+
+## 4. Info operations
+
+<a name="ServerInfo"></a>
+### ServerInfo() ([]ServerInfo, error)
+Fetches information for all cluster nodes, such as server properties, storage information, network statistics, etc.
+
+| Param                           | Type               | Description                                                        |
+|---------------------------------|--------------------|--------------------------------------------------------------------|
+| `si.Addr`                       | _string_           | Address of the server the following information is retrieved from. |
+| `si.ConnStats`                  | _ServerConnStats_  | Connection statistics from the given server.                       |
+| `si.HTTPStats`                  | _ServerHTTPStats_  | HTTP connection statistics from the given server.                  |
+| `si.Properties`                 | _ServerProperties_ | Server properties such as region, notification targets.            |
+| `si.Data.StorageInfo.Used`      | _int64_            | Used disk space.                                                   |
+| `si.Data.StorageInfo.Total`     | _int64_            | Total disk space.                                                  |
+| `si.Data.StorageInfo.Available` | _int64_            | Available disk space.                                              |
+| `si.Data.StorageInfo.Backend`   | _struct{}_         | Represents backend type embedded structure.                        |
+
+| Param                       | Type            | Description                                        |
+|-----------------------------|-----------------|----------------------------------------------------|
+| `ServerProperties.Uptime`   | _time.Duration_ | Total duration in seconds since server is running. |
+| `ServerProperties.Version`  | _string_        | Current server version.                            |
+| `ServerProperties.CommitID` | _string_        | Current server commitID.                           |
+| `ServerProperties.Region`   | _string_        | Configured server region.                          |
+| `ServerProperties.SQSARN`   | _[]string_      | List of notification target ARNs.                  |
+
+| Param                              | Type     | Description                         |
+|------------------------------------|----------|-------------------------------------|
+| `ServerConnStats.TotalInputBytes`  | _uint64_ | Total bytes received by the server. |
+| `ServerConnStats.TotalOutputBytes` | _uint64_ | Total bytes sent by the server.     |
+
+| Param                                | Type                    | Description                                             |
+|--------------------------------------|-------------------------|---------------------------------------------------------|
+| `ServerHTTPStats.TotalHEADStats`     | _ServerHTTPMethodStats_ | Total statistics regarding HEAD operations              |
+| `ServerHTTPStats.SuccessHEADStats`   | _ServerHTTPMethodStats_ | Total statistics regarding successful HEAD operations   |
+| `ServerHTTPStats.TotalGETStats`      | _ServerHTTPMethodStats_ | Total statistics regarding GET operations               |
+| `ServerHTTPStats.SuccessGETStats`    | _ServerHTTPMethodStats_ | Total statistics regarding successful GET operations    |
+| `ServerHTTPStats.TotalPUTStats`      | _ServerHTTPMethodStats_ | Total statistics regarding PUT operations               |
+| `ServerHTTPStats.SuccessPUTStats`    | _ServerHTTPMethodStats_ | Total statistics regarding successful PUT operations    |
+| `ServerHTTPStats.TotalPOSTStats`     | _ServerHTTPMethodStats_ | Total statistics regarding POST operations              |
+| `ServerHTTPStats.SuccessPOSTStats`   | _ServerHTTPMethodStats_ | Total statistics regarding successful POST operations   |
+| `ServerHTTPStats.TotalDELETEStats`   | _ServerHTTPMethodStats_ | Total statistics regarding DELETE operations            |
+| `ServerHTTPStats.SuccessDELETEStats` | _ServerHTTPMethodStats_ | Total statistics regarding successful DELETE operations |
+
+
+| Param                               | Type     | Description                                     |
+|-------------------------------------|----------|-------------------------------------------------|
+| `ServerHTTPMethodStats.Count`       | _uint64_ | Total number of operations.                     |
+| `ServerHTTPMethodStats.AvgDuration` | _string_ | Average duration of Count number of operations. |
+
+| Param                      | Type            | Description                                                                       |
+|----------------------------|-----------------|-----------------------------------------------------------------------------------|
+| `Backend.Type`             | _BackendType_   | Type of backend used by the server currently only FS or Erasure.                  |
+| `Backend.OnlineDisks`      | _int_           | Total number of disks online (only applies to Erasure backend), is empty for FS.  |
+| `Backend.OfflineDisks`     | _int_           | Total number of disks offline (only applies to Erasure backend), is empty for FS. |
+| `Backend.StandardSCData`   | _int_           | Data disks set for standard storage class, is empty for FS.                       |
+| `Backend.StandardSCParity` | _int_           | Parity disks set for standard storage class, is empty for FS.                     |
+| `Backend.RRSCData`         | _int_           | Data disks set for reduced redundancy storage class, is empty for FS.             |
+| `Backend.RRSCParity`       | _int_           | Parity disks set for reduced redundancy storage class, is empty for FS.           |
+| `Backend.Sets`             | _[][]DriveInfo_ | Represents topology of drives in erasure coded sets.                              |
+
+| Param                | Type     | Description                                           |
+|----------------------|----------|-------------------------------------------------------|
+| `DriveInfo.UUID`     | _string_ | Unique ID for each disk provisioned by server format. |
+| `DriveInfo.Endpoint` | _string_ | Endpoint location of the remote/local disk.           |
+| `DriveInfo.State`    | _string_ | Current state of the disk at endpoint.                |
+
+ __Example__
+
+ ```go
+
+	serversInfo, err := madmClnt.ServerInfo()
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	for _, peerInfo := range serversInfo {
+		log.Printf("Node: %s, Info: %v\n", peerInfo.Addr, peerInfo.Data)
+	}
+
+ ```
+
+<a name="ServerDrivesPerfInfo"></a>
+### ServerDrivesPerfInfo() ([]ServerDrivesPerfInfo, error)
+
+Fetches drive performance information for all cluster nodes. Returned value is in Bytes/s.
+
+| Param           | Type               | Description                                                        |
+|-----------------|--------------------|--------------------------------------------------------------------|
+| `di.Addr`       | _string_           | Address of the server the following information is retrieved from. |
+| `di.Error`      | _string_           | Errors (if any) encountered while reaching this node               |
+| `di.DrivesPerf` | _disk.Performance_ | Path of the drive mount on above server and read, write speed.     |
+
+| Param                         | Type      | Description                                            |
+|-------------------------------|-----------|--------------------------------------------------------|
+| `disk.Performance.Path`       | _string_  | Path of drive mount.                                   |
+| `disk.Performance.Error`      | _string_  | Error (if any) encountered while accessing this drive. |
+| `disk.Performance.WriteSpeed` | _float64_ | Write speed on above path in Bytes/s.                  |
+| `disk.Performance.ReadSpeed`  | _float64_ | Read speed on above path in Bytes/s.                   |
+
+<a name="ServerCPULoadInfo"></a>
+### ServerCPULoadInfo() ([]ServerCPULoadInfo, error)
+
+Fetches CPU utilization for all cluster nodes. Returned value is in Bytes.
+
+| Param          | Type       | Description                                                         |
+|----------------|------------|---------------------------------------------------------------------|
+| `cpui.Addr`    | _string_   | Address of the server the following information  is retrieved from. |
+| `cpui.Error`   | _string_   | Errors (if any) encountered while reaching this node                |
+| `cpui.CPULoad` | _cpu.Load_ | The load on the CPU.                                                |
+
+| Param            | Type      | Description                                                     |
+|------------------|-----------|-----------------------------------------------------------------|
+| `cpu.Load.Avg`   | _float64_ | The average utilization of the CPU measured in a 200ms interval |
+| `cpu.Load.Min`   | _float64_ | The minimum utilization of the CPU measured in a 200ms interval |
+| `cpu.Load.Max`   | _float64_ | The maximum utilization of the CPU measured in a 200ms interval |
+| `cpu.Load.Error` | _string_  | Error (if any) encountered while accesing the CPU info          |
+
+<a name="ServerMemUsageInfo"></a>
+### ServerMemUsageInfo() ([]ServerMemUsageInfo, error)
+
+Fetches Mem utilization for all cluster nodes. Returned value is in Bytes.
+
+| Param           | Type        | Description                                                         |
+|-----------------|-------------|---------------------------------------------------------------------|
+| `memi.Addr`     | _string_    | Address of the server the following information  is retrieved from. |
+| `memi.Error`    | _string_    | Errors (if any) encountered while reaching this node                |
+| `memi.MemUsage` | _mem.Usage_ | The utilitzation of Memory                                          |
+
+| Param             | Type     | Description                                            |
+|-------------------|----------|--------------------------------------------------------|
+| `mem.Usage.Mem`   | _uint64_ | The total number of bytes obtained from the OS         |
+| `mem.Usage.Error` | _string_ | Error (if any) encountered while accesing the CPU info |
+
+## 6. Heal operations
+
+<a name="Heal"></a>
+### Heal(bucket, prefix string, healOpts HealOpts, clientToken string, forceStart bool, forceStop bool) (start HealStartSuccess, status HealTaskStatus, err error)
+
+Start a heal sequence that scans data under given (possible empty)
+`bucket` and `prefix`. The `recursive` bool turns on recursive
+traversal under the given path. `dryRun` does not mutate on-disk data,
+but performs data validation.
+
+Two heal sequences on overlapping paths may not be initiated.
+
+The progress of a heal should be followed using the same API `Heal`
+by providing the `clientToken` previously obtained from a `Heal`
+API. The server accumulates results of the heal traversal and waits
+for the client to receive and acknowledge them using the status
+request by providing `clientToken`.
+
+__Example__
+
+``` go
+
+    opts := madmin.HealOpts{
+            Recursive: true,
+            DryRun:    false,
+    }
+    forceStart := false
+    forceStop := false
+    healPath, err := madmClnt.Heal("", "", opts, "", forceStart, forceStop)
+    if err != nil {
+        log.Fatalln(err)
+    }
+    log.Printf("Heal sequence started at %s", healPath)
+
+```
+
+#### HealStartSuccess structure
+
+| Param             | Type        | Description                                                                                                                      |
+|-------------------|-------------|----------------------------------------------------------------------------------------------------------------------------------|
+| `s.ClientToken`   | _string_    | A unique token for a successfully started heal operation, this token is used to request realtime progress of the heal operation. |
+| `s.ClientAddress` | _string_    | Address of the client which initiated the heal operation, the client address has the form "host:port".                           |
+| `s.StartTime`     | _time.Time_ | Time when heal was initially started.                                                                                            |
+
+#### HealTaskStatus structure
+
+| Param             | Type               | Description                                       |
+|-------------------|--------------------|---------------------------------------------------|
+| `s.Summary`       | _string_           | Short status of heal sequence                     |
+| `s.FailureDetail` | _string_           | Error message in case of heal sequence failure    |
+| `s.HealSettings`  | _HealOpts_         | Contains the booleans set in the `HealStart` call |
+| `s.Items`         | _[]HealResultItem_ | Heal records for actions performed by server      |
+
+#### HealResultItem structure
+
+| Param                  | Type           | Description                                                     |
+|------------------------|----------------|-----------------------------------------------------------------|
+| `ResultIndex`          | _int64_        | Index of the heal-result record                                 |
+| `Type`                 | _HealItemType_ | Represents kind of heal operation in the heal record            |
+| `Bucket`               | _string_       | Bucket name                                                     |
+| `Object`               | _string_       | Object name                                                     |
+| `Detail`               | _string_       | Details about heal operation                                    |
+| `DiskInfo.AvailableOn` | _[]int_        | List of disks on which the healed entity is present and healthy |
+| `DiskInfo.HealedOn`    | _[]int_        | List of disks on which the healed entity was restored           |
+
+## 7. Config operations
+
+<a name="GetConfig"></a>
+### GetConfig() ([]byte, error)
+Get current `config.json` of a MinIO server.
+
+__Example__
+
+``` go
+    configBytes, err := madmClnt.GetConfig()
+    if err != nil {
+        log.Fatalf("failed due to: %v", err)
+    }
+
+    // Pretty-print config received as json.
+    var buf bytes.Buffer
+    err = json.Indent(buf, configBytes, "", "\t")
+    if err != nil {
+        log.Fatalf("failed due to: %v", err)
+    }
+
+    log.Println("config received successfully: ", string(buf.Bytes()))
+```
+
+
+<a name="SetConfig"></a>
+### SetConfig(config io.Reader) error
+Set a new `config.json` for a MinIO server.
+
+__Example__
+
+``` go
+    config := bytes.NewReader([]byte(`config.json contents go here`))
+    if err := madmClnt.SetConfig(config); err != nil {
+        log.Fatalf("failed due to: %v", err)
+    }
+    log.Println("SetConfig was successful")
+```
+
+<a name="GetConfigKeys"></a>
+### GetConfigKeys(keys []string) ([]byte, error)
+Get a json document which contains a set of keys and their values from config.json.
+
+__Example__
+
+``` go
+    configBytes, err := madmClnt.GetConfigKeys([]string{"version", "notify.amqp.1"})
+    if err != nil {
+        log.Fatalf("failed due to: %v", err)
+    }
+
+    // Pretty-print config received as json.
+    var buf bytes.Buffer
+    err = json.Indent(buf, configBytes, "", "\t")
+    if err != nil {
+        log.Fatalf("failed due to: %v", err)
+    }
+
+    log.Println("config received successfully: ", string(buf.Bytes()))
+```
+
+
+<a name="SetConfigKeys"></a>
+### SetConfigKeys(params map[string]string) error
+Set a set of keys and values for MinIO server or distributed setup and restart the MinIO
+server for the new configuration changes to take effect.
+
+__Example__
+
+``` go
+    err := madmClnt.SetConfigKeys(map[string]string{"notify.webhook.1": "{\"enable\": true, \"endpoint\": \"http://example.com/api\"}"})
+    if err != nil {
+        log.Fatalf("failed due to: %v", err)
+    }
+
+    log.Println("New configuration successfully set")
+```
+
+## 8. Top operations
+
+<a name="TopLocks"></a>
+### TopLocks() (LockEntries, error)
+Get the oldest locks from MinIO server.
+
+__Example__
+
+``` go
+    locks, err := madmClnt.TopLocks()
+    if err != nil {
+        log.Fatalf("failed due to: %v", err)
+    }
+
+    out, err := json.Marshal(locks)
+    if err != nil {
+        log.Fatalf("Marshal failed due to: %v", err)
+    }
+
+    log.Println("TopLocks received successfully: ", string(out))
+```
+
+## 9. IAM operations
+
+<a name="AddCannedPolicy"></a>
+### AddCannedPolicy(policyName string, policy string) error
+Create a new canned policy on MinIO server.
+
+__Example__
+
+```
+	policy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:GetObject"],"Effect": "Allow","Resource": ["arn:aws:s3:::my-bucketname/*"],"Sid": ""}]}`
+
+    if err = madmClnt.AddCannedPolicy("get-only", policy); err != nil {
+		log.Fatalln(err)
+	}
+```
+
+<a name="AddUser"></a>
+### AddUser(user string, secret string) error
+Add a new user on a MinIO server.
+
+__Example__
+
+``` go
+	if err = madmClnt.AddUser("newuser", "newstrongpassword"); err != nil {
+		log.Fatalln(err)
+	}
+```
+
+<a name="SetUserPolicy"></a>
+### SetUserPolicy(user string, policyName string) error
+Enable a canned policy `get-only` for a given user on MinIO server.
+
+__Example__
+
+``` go
+	if err = madmClnt.SetUserPolicy("newuser", "get-only"); err != nil {
+		log.Fatalln(err)
+	}
+```
+
+<a name="ListUsers"></a>
+### ListUsers() (map[string]UserInfo, error)
+Lists all users on MinIO server.
+
+__Example__
+
+``` go
+	users, err := madmClnt.ListUsers();
+    if err != nil {
+		log.Fatalln(err)
+	}
+    for k, v := range users {
+        fmt.Printf("User %s Status %s\n", k, v.Status)
+    }
+```
+
+## 10. Misc operations
+
+<a name="StartProfiling"></a>
+### StartProfiling(profiler string) error
+Ask all nodes to start profiling using the specified profiler mode
+
+__Example__
+
+``` go
+    startProfilingResults, err = madmClnt.StartProfiling("cpu")
+    if err != nil {
+            log.Fatalln(err)
+    }
+    for _, result := range startProfilingResults {
+        if !result.Success {
+            log.Printf("Unable to start profiling on node `%s`, reason = `%s`\n", result.NodeName, result.Error)
+        } else {
+            log.Printf("Profiling successfully started on node `%s`\n", result.NodeName)
+        }
+    }
+
+```
+
+<a name="DownloadProfilingData"></a>
+### DownloadProfilingData() ([]byte, error)
+Download profiling data of all nodes in a zip format.
+
+__Example__
+
+``` go
+    profilingData, err := madmClnt.DownloadProfilingData()
+    if err != nil {
+            log.Fatalln(err)
+    }
+
+    profilingFile, err := os.Create("/tmp/profiling-data.zip")
+    if err != nil {
+            log.Fatal(err)
+    }
+
+    if _, err := io.Copy(profilingFile, profilingData); err != nil {
+            log.Fatal(err)
+    }
+
+    if err := profilingFile.Close(); err != nil {
+            log.Fatal(err)
+    }
+
+    if err := profilingData.Close(); err != nil {
+            log.Fatal(err)
+    }
+
+    log.Println("Profiling data successfully downloaded.")
+```

pkg/madmin/examples/set-credentials.go

@@ -1,44 +0,0 @@
-// +build ignore
-
-/*
- * MinIO Cloud Storage, (C) 2016 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package main
-
-import (
-	"log"
-
-	"github.com/minio/minio/pkg/madmin"
-)
-
-func main() {
-	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
-	// dummy values, please replace them with original values.
-
-	// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
-	// New returns an MinIO Admin client object.
-	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	err = madmClnt.SetAdminCredentials("YOUR-NEW-ACCESSKEY", "YOUR-NEW-SECRETKEY")
-	if err != nil {
-		log.Fatalln(err)
-	}
-	log.Println("New credentials successfully set.")
-}

pkg/madmin/generic-commands.go

@@ -1,66 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2016, 2017 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package madmin
-
-import (
-	"encoding/json"
-	"net/http"
-)
-
-// SetCredsReq - xml to send to the server to set new credentials
-type SetCredsReq struct {
-	AccessKey string `json:"accessKey"`
-	SecretKey string `json:"secretKey"`
-}
-
-// SetAdminCredentials - Call Set Credentials API to set new access and
-// secret keys in the specified MinIO server
-func (adm *AdminClient) SetAdminCredentials(access, secret string) error {
-	// Setup request's body
-	body, err := json.Marshal(SetCredsReq{access, secret})
-	if err != nil {
-		return err
-	}
-
-	ebody, err := EncryptData(adm.secretAccessKey, body)
-	if err != nil {
-		return err
-	}
-
-	// Setup new request
-	reqData := requestData{
-		relPath: "/v1/config/credential",
-		content: ebody,
-	}
-
-	// Execute GET on bucket to list objects.
-	resp, err := adm.executeMethod("PUT", reqData)
-
-	defer closeResponse(resp)
-	if err != nil {
-		return err
-	}
-
-	// Return error to the caller if http response code is
-	// different from 200
-	if resp.StatusCode != http.StatusOK {
-		return httpRespToErrorResponse(resp)
-	}
-
-	return nil
-}