posix: do not upstream errors in deleteFile (#4771)

This commit changes posix's deleteFile() to not upstream errors from
removing parent directories. This fixes a race condition.

The race condition occurs when multiple deleteFile()s are called on the
same parent directory, but different child files. Because deleteFile()
recursively removes parent directories if they are empty, but
deleteFile() errors if the selected deletePath does not exist, there was
an opportunity for a race condition. The two processes would remove the
child directories successfully, then depend on the parent directory
still existing. In some cases this is an invalid assumption, because
other processes can remove the parent directory beforehand. This commit
changes deleteFile() to not upstream an error if one occurs, because the
only required error should be from the immediate deletePath, not from a
parent path.

In the specific bug report, multiple CompleteMultipartUpload requests
would launch multiple deleteFile() requests. Because they chain up on
parent directories, ultimately at the end, there would be multiple
remove files for the ultimate parent directory,
.minio.sys/multipart/{bucket}. Because only one will succeed and one
will fail, an error would be upstreamed saying that the file does not
exist, and the CompleteMultipartUpload code interpreted this as
NoSuchKey, or that the object/part id doesn't exist. This was faulty
behavior and is now fixed.

The added test fails before this change and passes after this change.

Fixes: https://github.com/minio/minio/issues/4727

browser/app/js/components/Browse.js

@@ -150,7 +150,16 @@ export default class Browse extends React.Component {
       if (prefix === currentPath) return
       browserHistory.push(utils.pathJoin(currentBucket, encPrefix))
     } else {
-      window.location = `${window.location.origin}/minio/download/${currentBucket}/${encPrefix}?token=${storage.getItem('token')}`
+      // Download the selected file.
+      web.CreateURLToken()
+        .then(res => {
+          let url = `${window.location.origin}/minio/download/${currentBucket}/${encPrefix}?token=${res.token}`
+          window.location = url
+        })
+        .catch(err => dispatch(actions.showAlert({
+          type: 'danger',
+          message: err.message
+        })))
     }
   }
 
@@ -406,16 +415,24 @@ export default class Browse extends React.Component {
   }
 
   downloadSelected() {
-    const {dispatch} = this.props
+    const {dispatch, web} = this.props
     let req = {
       bucketName: this.props.currentBucket,
       objects: this.props.checkedObjects,
       prefix: this.props.currentPath
     }
-    let requestUrl = location.origin + "/minio/zip?token=" + localStorage.token
 
-    this.xhr = new XMLHttpRequest()
-    dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
+    web.CreateURLToken()
+      .then(res => {
+        let requestUrl = location.origin + "/minio/zip?token=" + res.token
+
+        this.xhr = new XMLHttpRequest()
+        dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
+      })
+      .catch(err => dispatch(actions.showAlert({
+        type: 'danger',
+        message: err.message
+      })))
   }
 
   clearSelected() {

browser/app/js/web.js

@@ -112,6 +112,9 @@ export default class Web {
         return res
       })
   }
+  CreateURLToken() {
+    return this.makeCall('CreateURLToken')
+  }
   GetBucketPolicy(args) {
     return this.makeCall('GetBucketPolicy', args)
   }

buildscripts/healthcheck.sh

@@ -16,17 +16,28 @@
 #
 
 _init () {
-    address="http://127.0.0.1:9000"
+    scheme="http://"
+    address="127.0.0.1:9000"
     resource="/minio/index.html"
 }
 
 HealthCheckMain () {
     # Get the http response code
-    http_response=$(curl -s -o /dev/null -I -w "%{http_code}" ${address}${resource})
+    http_response=$(curl -s -k -o /dev/null -I -w "%{http_code}" ${scheme}${address}${resource})
+
+    # Get the http response body
+    http_response_body=$(curl -k -s ${scheme}${address}${resource})
+
+    # server returns response 403 and body "SSL required" if non-TLS connection is attempted on a TLS-configured server. 
+    # change the scheme and try again
+    if [ "$http_response" = "403" ] && [ "$http_response_body" = "SSL required" ]; then
+        scheme="https://"
+        http_response=$(curl -s -k -o /dev/null -I -w "%{http_code}" ${scheme}${address}${resource})
+    fi
 
     # If http_repsonse is 200 - server is up. 
     # When MINIO_BROWSER is set to off, curl responds with 404. We assume that the the server is up
-    [ "$http_response" == "200" ] || [ "$http_response" == "404" ]
+    [ "$http_response" = "200" ] || [ "$http_response" = "404" ]
 }
 
 _init && HealthCheckMain

cmd/credential.go

@@ -29,6 +29,7 @@ const (
 	accessKeyMinLen = 5
 
 	// Maximum length for Minio access key.
+	// There is no max length enforcement for access keys
 	accessKeyMaxLen = 20
 
 	// Minimum length for Minio secret key for both server and gateway mode.
@@ -36,11 +37,8 @@ const (
 
 	// Maximum secret key length for Minio, this
 	// is used when autogenerating new credentials.
-	secretKeyMaxLenMinio = 40
-
-	// Maximum secret key length allowed from client side
-	// caters for both server and gateway mode.
-	secretKeyMaxLen = 100
+	// There is no max length enforcement for secret keys
+	secretKeyMaxLen = 40
 
 	// Alpha numeric table used for generating access keys.
 	alphaNumericTable = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
@@ -51,18 +49,18 @@ const (
 
 // Common errors generated for access and secret key validation.
 var (
-	errInvalidAccessKeyLength = errors.New("Invalid access key, access key should be 5 to 20 characters in length")
-	errInvalidSecretKeyLength = errors.New("Invalid secret key, secret key should be 8 to 100 characters in length")
+	errInvalidAccessKeyLength = errors.New("Invalid access key, access key should be minimum 5 characters in length")
+	errInvalidSecretKeyLength = errors.New("Invalid secret key, secret key should be minimum 8 characters in length")
 )
 
 // isAccessKeyValid - validate access key for right length.
 func isAccessKeyValid(accessKey string) bool {
-	return len(accessKey) >= accessKeyMinLen && len(accessKey) <= accessKeyMaxLen
+	return len(accessKey) >= accessKeyMinLen
 }
 
 // isSecretKeyValid - validate secret key for right length.
 func isSecretKeyValid(secretKey string) bool {
-	return len(secretKey) >= secretKeyMinLen && len(secretKey) <= secretKeyMaxLen
+	return len(secretKey) >= secretKeyMinLen
 }
 
 // credential container for access and secret keys.
@@ -116,24 +114,35 @@ func createCredential(accessKey, secretKey string) (cred credential, err error)
 }
 
 // Initialize a new credential object
-func mustGetNewCredential() credential {
+func getNewCredential(accessKeyLen, secretKeyLen int) (cred credential, err error) {
 	// Generate access key.
-	keyBytes := make([]byte, accessKeyMaxLen)
-	_, err := rand.Read(keyBytes)
-	fatalIf(err, "Unable to generate access key.")
-	for i := 0; i < accessKeyMaxLen; i++ {
+	keyBytes := make([]byte, accessKeyLen)
+	_, err = rand.Read(keyBytes)
+	if err != nil {
+		return cred, err
+	}
+
+	for i := 0; i < accessKeyLen; i++ {
 		keyBytes[i] = alphaNumericTable[keyBytes[i]%alphaNumericTableLen]
 	}
 	accessKey := string(keyBytes)
 
 	// Generate secret key.
-	keyBytes = make([]byte, secretKeyMaxLenMinio)
+	keyBytes = make([]byte, secretKeyLen)
 	_, err = rand.Read(keyBytes)
-	fatalIf(err, "Unable to generate secret key.")
-	secretKey := string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyMaxLenMinio])
+	if err != nil {
+		return cred, err
+	}
 
-	cred, err := createCredential(accessKey, secretKey)
-	fatalIf(err, "Unable to generate new credential.")
+	secretKey := string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyLen])
+	cred, err = createCredential(accessKey, secretKey)
 
+	return cred, err
+}
+
+func mustGetNewCredential() credential {
+	// Generate Minio credentials with Minio key max lengths.
+	cred, err := getNewCredential(accessKeyMaxLen, secretKeyMaxLen)
+	fatalIf(err, "Unable to generate new credentials.")
 	return cred
 }

cmd/credential_test.go

@@ -23,8 +23,8 @@ func TestMustGetNewCredential(t *testing.T) {
 	if !cred.IsValid() {
 		t.Fatalf("Failed to get new valid credential")
 	}
-	if len(cred.SecretKey) != secretKeyMaxLenMinio {
-		t.Fatalf("Invalid length %d of the secretKey credential generated, expected %d", len(cred.SecretKey), secretKeyMaxLenMinio)
+	if len(cred.SecretKey) != secretKeyMaxLen {
+		t.Fatalf("Invalid length %d of the secretKey credential generated, expected %d", len(cred.SecretKey), secretKeyMaxLen)
 	}
 }
 
@@ -36,18 +36,14 @@ func TestCreateCredential(t *testing.T) {
 		expectedResult bool
 		expectedErr    error
 	}{
-		// Access key too small.
+		// Access key too small (min 5 chars).
 		{"user", "pass", false, errInvalidAccessKeyLength},
-		// Access key too long.
-		{"user12345678901234567", "pass", false, errInvalidAccessKeyLength},
-		// Access key contains unsuppported characters.
-		{"!@#$", "pass", false, errInvalidAccessKeyLength},
-		// Secret key too small.
+		// Long access key is ok.
+		{"user123456789012345678901234567890", "password", true, nil},
+		// Secret key too small (min 8 chars).
 		{"myuser", "pass", false, errInvalidSecretKeyLength},
-		// Secret key too long.
-		{"myuser", "pass1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", false, errInvalidSecretKeyLength},
-		// Success when access key contains leading/trailing spaces.
-		{" user ", cred.SecretKey, true, nil},
+		// Long secret key is ok.
+		{"myuser", "pass1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", true, nil},
 		{"myuser", "mypassword", true, nil},
 		{cred.AccessKey, cred.SecretKey, true, nil},
 	}

cmd/fs-v1-helpers.go

@@ -363,8 +363,7 @@ func fsRenameFile(sourcePath, destPath string) error {
 	return nil
 }
 
-// Delete a file and its parent if it is empty at the destination path.
-// this function additionally protects the basePath from being deleted.
+// fsDeleteFile is a wrapper for deleteFile(), after checking the path length.
 func fsDeleteFile(basePath, deletePath string) error {
 	if err := checkPathLength(basePath); err != nil {
 		return traceError(err)
@@ -374,44 +373,7 @@ func fsDeleteFile(basePath, deletePath string) error {
 		return traceError(err)
 	}
 
-	if basePath == deletePath {
-		return nil
-	}
-
-	// Verify if the path exists.
-	pathSt, err := osStat(preparePath(deletePath))
-	if err != nil {
-		if os.IsNotExist(err) {
-			return traceError(errFileNotFound)
-		} else if os.IsPermission(err) {
-			return traceError(errFileAccessDenied)
-		}
-		return traceError(err)
-	}
-
-	if pathSt.IsDir() && !isDirEmpty(deletePath) {
-		// Verify if directory is empty.
-		return nil
-	}
-
-	// Attempt to remove path.
-	if err = os.Remove(preparePath(deletePath)); err != nil {
-		if os.IsNotExist(err) {
-			return traceError(errFileNotFound)
-		} else if os.IsPermission(err) {
-			return traceError(errFileAccessDenied)
-		} else if isSysErrNotEmpty(err) {
-			return traceError(errVolumeNotEmpty)
-		}
-		return traceError(err)
-	}
-
-	// Recursively go down the next path and delete again.
-	if err := fsDeleteFile(basePath, pathutil.Dir(deletePath)); err != nil {
-		return err
-	}
-
-	return nil
+	return deleteFile(basePath, deletePath)
 }
 
 // fsRemoveMeta safely removes a locked file and takes care of Windows special case

cmd/fs-v1-helpers_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
@@ -263,50 +264,123 @@ func TestFSDeletes(t *testing.T) {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
-	reader.Seek(0, 0)
+	reader.Seek(0, io.SeekStart)
+
+	// folder is not empty
+	err = fsMkdir(pathJoin(path, "success-vol", "not-empty"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = ioutil.WriteFile(pathJoin(path, "success-vol", "not-empty", "file"), []byte("data"), 0777)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// recursive
+	if err = fsMkdir(pathJoin(path, "success-vol", "parent")); err != nil {
+		t.Fatal(err)
+	}
+	if err = fsMkdir(pathJoin(path, "success-vol", "parent", "dir")); err != nil {
+		t.Fatal(err)
+	}
 
 	testCases := []struct {
+		basePath    string
 		srcVol      string
 		srcPath     string
 		expectedErr error
 	}{
-		// Test case - 1.
 		// valid case with existing volume and file to delete.
 		{
+			basePath:    path,
 			srcVol:      "success-vol",
 			srcPath:     "success-file",
 			expectedErr: nil,
 		},
-		// Test case - 2.
 		// The file was deleted in the last case, so DeleteFile should fail.
 		{
+			basePath:    path,
 			srcVol:      "success-vol",
 			srcPath:     "success-file",
 			expectedErr: errFileNotFound,
 		},
-		// Test case - 3.
 		// Test case with segment of the volume name > 255.
 		{
+			basePath:    path,
 			srcVol:      "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
 			srcPath:     "success-file",
 			expectedErr: errFileNameTooLong,
 		},
-		// Test case - 4.
 		// Test case with src path segment > 255.
 		{
+			basePath:    path,
 			srcVol:      "success-vol",
 			srcPath:     "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
 			expectedErr: errFileNameTooLong,
 		},
+		// Base path is way too long.
+		{
+			basePath:    "path03333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333",
+			srcVol:      "success-vol",
+			srcPath:     "object",
+			expectedErr: errFileNameTooLong,
+		},
+		// Directory is not empty. Should give nil, but won't delete.
+		{
+			basePath:    path,
+			srcVol:      "success-vol",
+			srcPath:     "not-empty",
+			expectedErr: nil,
+		},
+		// Should delete recursively.
+		{
+			basePath:    path,
+			srcVol:      "success-vol",
+			srcPath:     pathJoin("parent", "dir"),
+			expectedErr: nil,
+		},
 	}
 
 	for i, testCase := range testCases {
-		if err = fsDeleteFile(path, pathJoin(path, testCase.srcVol, testCase.srcPath)); errorCause(err) != testCase.expectedErr {
+		if err = fsDeleteFile(testCase.basePath, pathJoin(testCase.basePath, testCase.srcVol, testCase.srcPath)); errorCause(err) != testCase.expectedErr {
 			t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}
 	}
 }
 
+func BenchmarkFSDeleteFile(b *testing.B) {
+	// create posix test setup
+	_, path, err := newPosixTestSetup()
+	if err != nil {
+		b.Fatalf("Unable to create posix test setup, %s", err)
+	}
+	defer removeAll(path)
+
+	// Setup test environment.
+	if err = fsMkdir(pathJoin(path, "benchmark")); err != nil {
+		b.Fatalf("Unable to create directory, %s", err)
+	}
+
+	benchDir := pathJoin(path, "benchmark")
+	filename := pathJoin(benchDir, "file.txt")
+
+	b.ResetTimer()
+	// We need to create and delete the file sequentially inside the benchmark.
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		err = ioutil.WriteFile(filename, []byte("data"), 0777)
+		if err != nil {
+			b.Fatal(err)
+		}
+		b.StartTimer()
+
+		err = fsDeleteFile(benchDir, filename)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 // Tests fs removes.
 func TestFSRemoves(t *testing.T) {
 	// create posix test setup

cmd/fs-v1-multipart.go

@@ -31,6 +31,11 @@ import (
 	"github.com/minio/sha256-simd"
 )
 
+const (
+	fsMultipartExpiry        = time.Hour * 24 * 14
+	fsMultipartCleanupPeriod = time.Hour * 24
+)
+
 // Returns if the prefix is a multipart upload.
 func (fs fsObjects) isMultipartUpload(bucket, prefix string) bool {
 	uploadsIDPath := pathJoin(fs.fsPath, bucket, prefix, uploadsJSONFile)
@@ -302,16 +307,35 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 	return result, nil
 }
 
-// ListMultipartUploads - lists all the pending multipart uploads on a
+// ListMultipartUploads - returns empty response always. The onus is
+// on applications to remember uploadId of the multipart uploads that
+// are in progress.
+func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
+	if err := checkListMultipartArgs(bucket, prefix, keyMarker, uploadIDMarker, delimiter, fs); err != nil {
+		return lmi, err
+	}
+
+	if _, err := fs.statBucketDir(bucket); err != nil {
+		return lmi, toObjectErr(err, bucket)
+	}
+	return ListMultipartsInfo{
+		Prefix:         prefix,
+		KeyMarker:      keyMarker,
+		UploadIDMarker: uploadIDMarker,
+		Delimiter:      delimiter,
+		MaxUploads:     maxUploads,
+		IsTruncated:    false,
+	}, nil
+}
+
+// listMultipartUploadsHelper - lists all the pending multipart uploads on a
 // bucket. Additionally takes 'prefix, keyMarker, uploadIDmarker and a
 // delimiter' which allows us to list uploads match a particular
 // prefix or lexically starting from 'keyMarker' or delimiting the
 // output to get a directory like listing.
 //
-// Implements S3 compatible ListMultipartUploads API. The resulting
-// ListMultipartsInfo structure is unmarshalled directly into XML and
-// replied back to the client.
-func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
+// This function remains here to aid in listing uploads that require cleanup.
+func (fs fsObjects) listMultipartUploadsHelper(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
 	if err := checkListMultipartArgs(bucket, prefix, keyMarker, uploadIDMarker, delimiter, fs); err != nil {
 		return lmi, err
 	}
@@ -596,103 +620,37 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	}, nil
 }
 
-// listObjectParts - wrapper scanning through
-// '.minio.sys/multipart/bucket/object/UPLOADID'. Lists all the parts
-// saved inside '.minio.sys/multipart/bucket/object/UPLOADID'.
-func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (lpi ListPartsInfo, e error) {
-	result := ListPartsInfo{}
-
-	uploadIDPath := pathJoin(bucket, object, uploadID)
-	fsMetaPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadIDPath, fsMetaJSONFile)
-	metaFile, err := fs.rwPool.Open(fsMetaPath)
-	if err != nil {
-		if err == errFileNotFound || err == errFileAccessDenied {
-			// On windows oddly this is returned.
-			return lpi, traceError(InvalidUploadID{UploadID: uploadID})
-		}
-		return lpi, toObjectErr(traceError(err), bucket, object)
-	}
-	defer fs.rwPool.Close(fsMetaPath)
-
-	fsMeta := fsMetaV1{}
-	_, err = fsMeta.ReadFrom(metaFile.LockedFile)
-	if err != nil {
-		return lpi, toObjectErr(err, minioMetaBucket, fsMetaPath)
-	}
-
-	// Only parts with higher part numbers will be listed.
-	partIdx := fsMeta.ObjectPartIndex(partNumberMarker)
-	parts := fsMeta.Parts
-	if partIdx != -1 {
-		parts = fsMeta.Parts[partIdx+1:]
-	}
-
-	count := maxParts
-	for _, part := range parts {
-		var fi os.FileInfo
-		partNamePath := pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadIDPath, part.Name)
-		fi, err = fsStatFile(partNamePath)
-		if err != nil {
-			return lpi, toObjectErr(err, minioMetaMultipartBucket, partNamePath)
-		}
-		result.Parts = append(result.Parts, PartInfo{
-			PartNumber:   part.Number,
-			ETag:         part.ETag,
-			LastModified: fi.ModTime(),
-			Size:         fi.Size(),
-		})
-		count--
-		if count == 0 {
-			break
-		}
-	}
-
-	// If listed entries are more than maxParts, we set IsTruncated as true.
-	if len(parts) > len(result.Parts) {
-		result.IsTruncated = true
-		// Make sure to fill next part number marker if IsTruncated is
-		// true for subsequent listing.
-		nextPartNumberMarker := result.Parts[len(result.Parts)-1].PartNumber
-		result.NextPartNumberMarker = nextPartNumberMarker
-	}
-	result.Bucket = bucket
-	result.Object = object
-	result.UploadID = uploadID
-	result.MaxParts = maxParts
-
-	// Success.
-	return result, nil
-}
-
-// ListObjectParts - lists all previously uploaded parts for a given
-// object and uploadID.  Takes additional input of part-number-marker
-// to indicate where the listing should begin from.
-//
-// Implements S3 compatible ListObjectParts API. The resulting
-// ListPartsInfo structure is unmarshalled directly into XML and
-// replied back to the client.
+// ListObjectParts - returns empty response always. Applications are
+// expected to remember the uploaded part numbers and corresponding
+// etags.
 func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (lpi ListPartsInfo, e error) {
 	if err := checkListPartsArgs(bucket, object, fs); err != nil {
 		return lpi, err
 	}
 
+	// Check if bucket exists
 	if _, err := fs.statBucketDir(bucket); err != nil {
 		return lpi, toObjectErr(err, bucket)
 	}
 
-	// Hold the lock so that two parallel complete-multipart-uploads
-	// do not leave a stale uploads.json behind.
-	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
-	objectMPartPathLock.RLock()
-	defer objectMPartPathLock.RUnlock()
-
-	listPartsInfo, err := fs.listObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
+	// Check if uploadID exists. N B This check may race with a
+	// concurrent abortMultipartUpload on the same uploadID. It is
+	// OK to be eventually consistent w.r.t listing of objects,
+	// uploads and parts.
+	uploadIDPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket, object, uploadID)
+	_, err := fsStatDir(uploadIDPath)
 	if err != nil {
-		return lpi, toObjectErr(err, bucket, object)
+		return lpi, traceError(InvalidUploadID{UploadID: uploadID})
 	}
 
-	// Success.
-	return listPartsInfo, nil
+	// Return empty list parts response
+	return ListPartsInfo{
+		Bucket:           bucket,
+		Object:           object,
+		UploadID:         uploadID,
+		PartNumberMarker: 0,
+		MaxParts:         maxParts,
+	}, nil
 }
 
 // CompleteMultipartUpload - completes an ongoing multipart
@@ -978,3 +936,56 @@ func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error
 
 	return nil
 }
+
+// Remove multipart uploads left unattended in a given bucket older than `fsMultipartExpiry`
+func (fs fsObjects) cleanupStaleMultipartUpload(bucket string) (err error) {
+	var lmi ListMultipartsInfo
+
+	for {
+		// List multipart uploads in a bucket 1000 at a time
+		prefix := ""
+		lmi, err = fs.listMultipartUploadsHelper(bucket, prefix, lmi.KeyMarker, lmi.UploadIDMarker, slashSeparator, 1000)
+		if err != nil {
+			errorIf(err, fmt.Sprintf("Failed to list uploads of %s for cleaning up of multipart uploads older than %d weeks", bucket, fsMultipartExpiry))
+			return err
+		}
+
+		// Remove uploads (and its parts) older than 2 weeks
+		for _, upload := range lmi.Uploads {
+			uploadIDPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket, upload.Object, upload.UploadID)
+			st, err := fsStatDir(uploadIDPath)
+			if err != nil {
+				errorIf(err, "Failed to stat uploads directory", uploadIDPath)
+				return err
+			}
+			if time.Since(st.ModTime()) > fsMultipartExpiry {
+				fs.AbortMultipartUpload(bucket, upload.Object, upload.UploadID)
+			}
+		}
+
+		// No more incomplete uploads remain
+		if !lmi.IsTruncated {
+			break
+		}
+	}
+
+	return nil
+}
+
+// Remove multipart uploads left unattended for more than `fsMultipartExpiry` seconds.
+func (fs fsObjects) cleanupStaleMultipartUploads() {
+	for {
+		bucketInfos, err := fs.ListBuckets()
+		if err != nil {
+			errorIf(err, fmt.Sprintf("Failed to list buckets for cleaning up of multipart uploads older than %d weeks", fsMultipartExpiry))
+			return
+		}
+
+		for _, bucketInfo := range bucketInfos {
+			fs.cleanupStaleMultipartUpload(bucketInfo.Name)
+		}
+
+		// Schedule for the next multipart backend cleanup
+		time.Sleep(fsMultipartCleanupPeriod)
+	}
+}

cmd/fs-v1.go

@@ -157,6 +157,9 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		return nil, fmt.Errorf("Unable to initialize event notification. %s", err)
 	}
 
+	// Start background process to cleanup old files in minio.sys.tmp
+	go fs.cleanupStaleMultipartUploads()
+
 	// Return successfully initialized object layer.
 	return fs, nil
 }

cmd/jwt.go

@@ -34,6 +34,9 @@ const (
 
 	// Inter-node JWT token expiry is 100 years approx.
 	defaultInterNodeJWTExpiry = 100 * 365 * 24 * time.Hour
+
+	// URL JWT token expiry is one minute (might be exposed).
+	defaultURLJWTExpiry = time.Minute
 )
 
 var (
@@ -77,6 +80,10 @@ func authenticateWeb(accessKey, secretKey string) (string, error) {
 	return authenticateJWT(accessKey, secretKey, defaultJWTExpiry)
 }
 
+func authenticateURL(accessKey, secretKey string) (string, error) {
+	return authenticateJWT(accessKey, secretKey, defaultURLJWTExpiry)
+}
+
 func keyFuncCallback(jwtToken *jwtgo.Token) (interface{}, error) {
 	if _, ok := jwtToken.Method.(*jwtgo.SigningMethodHMAC); !ok {
 		return nil, fmt.Errorf("Unexpected signing method: %v", jwtToken.Header["alg"])

cmd/jwt_test.go

@@ -24,8 +24,12 @@ func testAuthenticate(authType string, t *testing.T) {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
 	defer removeAll(testPath)
-
-	serverCred := serverConfig.GetCredential()
+	// Create access and secret keys in length, 300 and 600
+	cred, err := getNewCredential(300, 600)
+	if err != nil {
+		t.Fatalf("unable to get new credential, %v", err)
+	}
+	serverConfig.SetCredential(cred)
 
 	// Define test cases.
 	testCases := []struct {
@@ -33,24 +37,16 @@ func testAuthenticate(authType string, t *testing.T) {
 		secretKey   string
 		expectedErr error
 	}{
-		// Access key too small.
-		{"user", "pass", errInvalidAccessKeyLength},
-		// Access key too long.
-		{"user12345678901234567", "pass", errInvalidAccessKeyLength},
-		// Access key contains unsuppported characters.
-		{"!@#$", "pass", errInvalidAccessKeyLength},
-		// Success when access key contains leading/trailing spaces.
-		{" " + serverCred.AccessKey + " ", serverCred.SecretKey, errInvalidAccessKeyLength},
-		// Secret key too small.
-		{"myuser", "pass", errInvalidSecretKeyLength},
-		// Secret key too long.
-		{"myuser", "pass1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", errInvalidSecretKeyLength},
+		// Access key (less than 5 chrs) too small.
+		{"user", cred.SecretKey, errInvalidAccessKeyLength},
+		// Secret key (less than 8 chrs) too small.
+		{cred.AccessKey, "pass", errInvalidSecretKeyLength},
 		// Authentication error.
 		{"myuser", "mypassword", errInvalidAccessKeyID},
 		// Authentication error.
-		{serverCred.AccessKey, "mypassword", errAuthentication},
+		{cred.AccessKey, "mypassword", errAuthentication},
 		// Success.
-		{serverCred.AccessKey, serverCred.SecretKey, nil},
+		{cred.AccessKey, cred.SecretKey, nil},
 	}
 
 	// Run tests.
@@ -60,6 +56,8 @@ func testAuthenticate(authType string, t *testing.T) {
 			_, err = authenticateNode(testCase.accessKey, testCase.secretKey)
 		} else if authType == "web" {
 			_, err = authenticateWeb(testCase.accessKey, testCase.secretKey)
+		} else if authType == "url" {
+			_, err = authenticateURL(testCase.accessKey, testCase.secretKey)
 		}
 
 		if testCase.expectedErr != nil {
@@ -83,6 +81,10 @@ func TestAuthenticateWeb(t *testing.T) {
 	testAuthenticate("web", t)
 }
 
+func TestAuthenticateURL(t *testing.T) {
+	testAuthenticate("url", t)
+}
+
 func BenchmarkAuthenticateNode(b *testing.B) {
 	testPath, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {

cmd/object-api-datatypes.go

@@ -34,9 +34,9 @@ const (
 // StorageInfo - represents total capacity of underlying storage.
 type StorageInfo struct {
 	// Total disk space.
-	Total int64
+	Total uint64
 	// Free available disk space.
-	Free int64
+	Free uint64
 	// Backend type.
 	Backend struct {
 		// Represents various backend types, currently on FS and Erasure.

cmd/object-api-multipart_test.go

@@ -1236,26 +1236,33 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 			if actualResult.Delimiter != expectedResult.Delimiter {
 				t.Errorf("Test %d: %s: Expected Delimiter to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Delimiter, actualResult.Delimiter)
 			}
-			// Asserting NextUploadIDMarker.
-			if actualResult.NextUploadIDMarker != expectedResult.NextUploadIDMarker {
-				t.Errorf("Test %d: %s: Expected NextUploadIDMarker to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.NextUploadIDMarker, actualResult.NextUploadIDMarker)
-			}
-			// Asserting NextKeyMarker.
-			if actualResult.NextKeyMarker != expectedResult.NextKeyMarker {
-				t.Errorf("Test %d: %s: Expected NextKeyMarker to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.NextKeyMarker, actualResult.NextKeyMarker)
-			}
 			// Asserting the keyMarker.
 			if actualResult.KeyMarker != expectedResult.KeyMarker {
 				t.Errorf("Test %d: %s: Expected keyMarker to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.KeyMarker, actualResult.KeyMarker)
 			}
-			// Asserting IsTruncated.
-			if actualResult.IsTruncated != testCase.expectedResult.IsTruncated {
-				t.Errorf("Test %d: %s: Expected Istruncated to be \"%v\", but found it to \"%v\"", i+1, instanceType, expectedResult.IsTruncated, actualResult.IsTruncated)
-			}
-			// Asserting the number of upload Metadata info.
-			if len(expectedResult.Uploads) != len(actualResult.Uploads) {
-				t.Errorf("Test %d: %s: Expected the result to contain info of %d Multipart Uploads, but found %d instead", i+1, instanceType, len(expectedResult.Uploads), len(actualResult.Uploads))
-			} else {
+
+			// ListMultipartUploads returns empty respsonse always in FS mode
+			if instanceType != FSTestStr {
+				// Asserting IsTruncated.
+				if actualResult.IsTruncated != testCase.expectedResult.IsTruncated {
+					t.Errorf("Test %d: %s: Expected Istruncated to be \"%v\", but found it to \"%v\"", i+1, instanceType, expectedResult.IsTruncated, actualResult.IsTruncated)
+					continue
+				}
+				// Asserting NextUploadIDMarker.
+				if actualResult.NextUploadIDMarker != expectedResult.NextUploadIDMarker {
+					t.Errorf("Test %d: %s: Expected NextUploadIDMarker to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.NextUploadIDMarker, actualResult.NextUploadIDMarker)
+					continue
+				}
+				// Asserting NextKeyMarker.
+				if actualResult.NextKeyMarker != expectedResult.NextKeyMarker {
+					t.Errorf("Test %d: %s: Expected NextKeyMarker to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.NextKeyMarker, actualResult.NextKeyMarker)
+					continue
+				}
+				// Asserting the number of upload Metadata info.
+				if len(expectedResult.Uploads) != len(actualResult.Uploads) {
+					t.Errorf("Test %d: %s: Expected the result to contain info of %d Multipart Uploads, but found %d instead", i+1, instanceType, len(expectedResult.Uploads), len(actualResult.Uploads))
+					continue
+				}
 				// Iterating over the uploads Metadata and asserting the fields.
 				for j, actualMetaData := range actualResult.Uploads {
 					//  Asserting the object name in the upload Metadata.
@@ -1712,10 +1719,6 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
 			if actualResult.UploadID != expectedResult.UploadID {
 				t.Errorf("Test %d: %s: Expected UploadID to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.UploadID, actualResult.UploadID)
 			}
-			// Asserting NextPartNumberMarker.
-			if actualResult.NextPartNumberMarker != expectedResult.NextPartNumberMarker {
-				t.Errorf("Test %d: %s: Expected NextPartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.NextPartNumberMarker, actualResult.NextPartNumberMarker)
-			}
 			// Asserting PartNumberMarker.
 			if actualResult.PartNumberMarker != expectedResult.PartNumberMarker {
 				t.Errorf("Test %d: %s: Expected PartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.PartNumberMarker, actualResult.PartNumberMarker)
@@ -1724,14 +1727,24 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
 			if actualResult.Bucket != expectedResult.Bucket {
 				t.Errorf("Test %d: %s: Expected Bucket to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Bucket, actualResult.Bucket)
 			}
-			// Asserting IsTruncated.
-			if actualResult.IsTruncated != testCase.expectedResult.IsTruncated {
-				t.Errorf("Test %d: %s: Expected IsTruncated to be \"%v\", but found it to \"%v\"", i+1, instanceType, expectedResult.IsTruncated, actualResult.IsTruncated)
-			}
-			// Asserting the number of Parts.
-			if len(expectedResult.Parts) != len(actualResult.Parts) {
-				t.Errorf("Test %d: %s: Expected the result to contain info of %d Parts, but found %d instead", i+1, instanceType, len(expectedResult.Parts), len(actualResult.Parts))
-			} else {
+
+			// ListObjectParts returns empty response always in FS mode
+			if instanceType != FSTestStr {
+				// Asserting IsTruncated.
+				if actualResult.IsTruncated != testCase.expectedResult.IsTruncated {
+					t.Errorf("Test %d: %s: Expected IsTruncated to be \"%v\", but found it to \"%v\"", i+1, instanceType, expectedResult.IsTruncated, actualResult.IsTruncated)
+					continue
+				}
+				// Asserting NextPartNumberMarker.
+				if actualResult.NextPartNumberMarker != expectedResult.NextPartNumberMarker {
+					t.Errorf("Test %d: %s: Expected NextPartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.NextPartNumberMarker, actualResult.NextPartNumberMarker)
+					continue
+				}
+				// Asserting the number of Parts.
+				if len(expectedResult.Parts) != len(actualResult.Parts) {
+					t.Errorf("Test %d: %s: Expected the result to contain info of %d Parts, but found %d instead", i+1, instanceType, len(expectedResult.Parts), len(actualResult.Parts))
+					continue
+				}
 				// Iterating over the partInfos and asserting the fields.
 				for j, actualMetaData := range actualResult.Parts {
 					//  Asserting the PartNumber in the PartInfo.
@@ -1747,6 +1760,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
 						t.Errorf("Test %d: %s: Part %d: Expected Etag to be \"%s\", but instead found \"%s\"", i+1, instanceType, j+1, expectedResult.Parts[j].ETag, actualMetaData.ETag)
 					}
 				}
+
 			}
 		}
 	}

cmd/object-handlers_test.go

@@ -1407,7 +1407,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 			if err != nil {
 				t.Fatalf("Test %d: %s: Failed to look for copied object part: <ERROR> %s", i+1, instanceType, err)
 			}
-			if len(results.Parts) != 1 {
+			if instanceType != FSTestStr && len(results.Parts) != 1 {
 				t.Fatalf("Test %d: %s: Expected only one entry returned %d entries", i+1, instanceType, len(results.Parts))
 			}
 		}

cmd/posix.go

@@ -35,11 +35,9 @@ import (
 )
 
 const (
-	diskMinFreeSpace   = 1 * humanize.GiByte // Min 1GiB free space.
-	diskMinTotalSpace  = diskMinFreeSpace    // Min 1GiB total space.
-	diskMinFreeInodes  = 10000               // Min 10000 free inodes.
-	diskMinTotalInodes = diskMinFreeInodes   // Min 10000 total inodes.
-	maxAllowedIOError  = 5
+	diskMinFreeSpace  = 1 * humanize.GiByte // Min 1GiB free space.
+	diskMinTotalSpace = diskMinFreeSpace    // Min 1GiB total space.
+	maxAllowedIOError = 5
 )
 
 // posix - implements StorageAPI interface.
@@ -76,24 +74,20 @@ func checkPathLength(pathName string) error {
 func isDirEmpty(dirname string) bool {
 	f, err := os.Open(preparePath(dirname))
 	if err != nil {
-		errorIf(func() error {
-			if !os.IsNotExist(err) {
-				return err
-			}
-			return nil
-		}(), "Unable to access directory.")
+		if !os.IsNotExist(err) {
+			errorIf(err, "Unable to access directory")
+		}
+
 		return false
 	}
 	defer f.Close()
 	// List one entry.
 	_, err = f.Readdirnames(1)
 	if err != io.EOF {
-		errorIf(func() error {
-			if !os.IsNotExist(err) {
-				return err
-			}
-			return nil
-		}(), "Unable to list directory.")
+		if !os.IsNotExist(err) {
+			errorIf(err, "Unable to list directory")
+		}
+
 		return false
 	}
 	// Returns true if we have reached EOF, directory is indeed empty.
@@ -177,18 +171,6 @@ func checkDiskMinTotal(di disk.Info) (err error) {
 	if int64(totalDiskSpace) <= diskMinTotalSpace {
 		return errDiskFull
 	}
-
-	// Some filesystems do not implement a way to provide total inodes available, instead
-	// inodes are allocated based on available disk space. For example CephDISK, StoreNext CVDISK,
-	// AzureFile driver. Allow for the available disk to be separately validated and we will
-	// validate inodes only if total inodes are provided by the underlying filesystem.
-	if di.Files != 0 && di.FSType != "NFS" {
-		totalFiles := int64(di.Files)
-		if totalFiles <= diskMinTotalInodes {
-			return errDiskFull
-		}
-	}
-
 	return nil
 }
 
@@ -200,17 +182,6 @@ func checkDiskMinFree(di disk.Info) error {
 		return errDiskFull
 	}
 
-	// Some filesystems do not implement a way to provide total inodes available, instead inodes
-	// are allocated based on available disk space. For example CephDISK, StoreNext CVDISK, AzureFile driver.
-	// Allow for the available disk to be separately validate and we will validate inodes only if
-	// total inodes are provided by the underlying filesystem.
-	if di.Files != 0 && di.FSType != "NFS" {
-		availableFiles := int64(di.Ffree)
-		if availableFiles <= diskMinFreeInodes {
-			return errDiskFull
-		}
-	}
-
 	// Success.
 	return nil
 }
@@ -915,27 +886,23 @@ func (s *posix) StatFile(volume, path string) (file FileInfo, err error) {
 	}, nil
 }
 
-// deleteFile - delete file path if its empty.
+// deleteFile deletes a file path if its empty. If it's successfully deleted,
+// it will recursively move up the tree, deleting empty parent directories
+// until it finds one with files in it. Returns nil for a non-empty directory.
 func deleteFile(basePath, deletePath string) error {
 	if basePath == deletePath {
 		return nil
 	}
-	// Verify if the path exists.
-	pathSt, err := osStat(preparePath(deletePath))
-	if err != nil {
-		if os.IsNotExist(err) {
-			return errFileNotFound
-		} else if os.IsPermission(err) {
-			return errFileAccessDenied
-		}
-		return err
-	}
-	if pathSt.IsDir() && !isDirEmpty(deletePath) {
-		// Verify if directory is empty.
-		return nil
-	}
+
 	// Attempt to remove path.
 	if err := os.Remove(preparePath(deletePath)); err != nil {
+		// Ignore errors if the directory is not empty. The server relies on
+		// this functionality, and sometimes uses recursion that should not
+		// error on parent directories.
+		if isSysErrNotEmpty(err) {
+			return nil
+		}
+
 		if os.IsNotExist(err) {
 			return errFileNotFound
 		} else if os.IsPermission(err) {
@@ -943,10 +910,11 @@ func deleteFile(basePath, deletePath string) error {
 		}
 		return err
 	}
+
 	// Recursively go down the next path and delete again.
-	if err := deleteFile(basePath, slashpath.Dir(deletePath)); err != nil {
-		return err
-	}
+	// Errors for parent directories shouldn't trickle down.
+	deleteFile(basePath, slashpath.Dir(deletePath))
+
 	return nil
 }
 

cmd/posix_test.go

@@ -73,6 +73,42 @@ func TestPosixGetDiskInfo(t *testing.T) {
 	}
 }
 
+func TestPosixIsDirEmpty(t *testing.T) {
+	tmp, err := ioutil.TempDir(globalTestTmpDir, "minio-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer removeAll(tmp)
+
+	// Should give false on non-existent directory.
+	dir1 := slashpath.Join(tmp, "non-existent-directory")
+	if isDirEmpty(dir1) != false {
+		t.Error("expected false for non-existent directory, got true")
+	}
+
+	// Should give false for not-a-directory.
+	dir2 := slashpath.Join(tmp, "file")
+	err = ioutil.WriteFile(dir2, []byte("hello"), 0777)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if isDirEmpty(dir2) != false {
+		t.Error("expected false for a file, got true")
+	}
+
+	// Should give true for a real empty directory.
+	dir3 := slashpath.Join(tmp, "empty")
+	err = os.Mkdir(dir3, 0777)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if isDirEmpty(dir3) != true {
+		t.Error("expected true for empty dir, got false")
+	}
+}
+
 // TestPosixReadAll - TestPosixs the functionality implemented by posix ReadAll storage API.
 func TestPosixReadAll(t *testing.T) {
 	// create posix test setup
@@ -706,6 +742,17 @@ func TestPosixDeleteFile(t *testing.T) {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 
+	if err = posixStorage.MakeVol("no-permissions"); err != nil {
+		t.Fatalf("Unable to create volume, %s", err.Error())
+	}
+	if err = posixStorage.AppendFile("no-permissions", "dir/file", []byte("Hello, world")); err != nil {
+		t.Fatalf("Unable to create file, %s", err.Error())
+	}
+	// Parent directory must have write permissions, this is read + execute.
+	if err = os.Chmod(pathJoin(path, "no-permissions"), 0555); err != nil {
+		t.Fatalf("Unable to chmod directory, %s", err.Error())
+	}
+
 	testCases := []struct {
 		srcVol      string
 		srcPath     string
@@ -760,6 +807,15 @@ func TestPosixDeleteFile(t *testing.T) {
 			ioErrCnt:    0,
 			expectedErr: errFileNameTooLong,
 		},
+		// TestPosix case - 7.
+		// TestPosix case with undeletable parent directory.
+		// File can delete, dir cannot delete because no-permissions doesn't have write perms.
+		{
+			srcVol:      "no-permissions",
+			srcPath:     "dir/file",
+			ioErrCnt:    0,
+			expectedErr: nil,
+		},
 	}
 
 	for i, testCase := range testCases {
@@ -1686,14 +1742,14 @@ func TestCheckDiskTotalMin(t *testing.T) {
 			},
 			err: nil,
 		},
-		// Test 2 - when fstype is xfs and total inodes are small.
+		// Test 2 - when fstype is xfs and total inodes are less than 10k.
 		{
 			diskInfo: disk.Info{
 				Total:  diskMinTotalSpace * 3,
 				FSType: "XFS",
 				Files:  9999,
 			},
-			err: errDiskFull,
+			err: nil,
 		},
 		// Test 3 - when fstype is btrfs and total inodes is empty.
 		{
@@ -1737,7 +1793,7 @@ func TestCheckDiskFreeMin(t *testing.T) {
 			},
 			err: nil,
 		},
-		// Test 2 - when fstype is xfs and total inodes are small.
+		// Test 2 - when fstype is xfs and total inodes are less than 10k.
 		{
 			diskInfo: disk.Info{
 				Free:   diskMinTotalSpace * 3,
@@ -1745,7 +1801,7 @@ func TestCheckDiskFreeMin(t *testing.T) {
 				Files:  9999,
 				Ffree:  9999,
 			},
-			err: errDiskFull,
+			err: nil,
 		},
 		// Test 3 - when fstype is btrfs and total inodes are empty.
 		{

cmd/server-main.go

@@ -52,8 +52,8 @@ FLAGS:
   {{end}}{{end}}
 ENVIRONMENT VARIABLES:
   ACCESS:
-     MINIO_ACCESS_KEY: Custom username or access key of 5 to 20 characters in length.
-     MINIO_SECRET_KEY: Custom password or secret key of 8 to 40 characters in length.
+     MINIO_ACCESS_KEY: Custom username or access key of minimum 5 characters in length.
+     MINIO_SECRET_KEY: Custom password or secret key of minimum 8 characters in length.
 
   BROWSER:
      MINIO_BROWSER: To disable web browser access, set this value to "off".

cmd/web-handlers.go

@@ -467,6 +467,30 @@ func (web *webAPIHandlers) GetAuth(r *http.Request, args *WebGenericArgs, reply
 	return nil
 }
 
+// URLTokenReply contains the reply for CreateURLToken.
+type URLTokenReply struct {
+	Token     string `json:"token"`
+	UIVersion string `json:"uiVersion"`
+}
+
+// CreateURLToken creates a URL token (short-lived) for GET requests.
+func (web *webAPIHandlers) CreateURLToken(r *http.Request, args *WebGenericArgs, reply *URLTokenReply) error {
+	if !isHTTPRequestValid(r) {
+		return toJSONError(errAuthentication)
+	}
+
+	creds := serverConfig.GetCredential()
+
+	token, err := authenticateURL(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		return toJSONError(err)
+	}
+
+	reply.Token = token
+	reply.UIVersion = browser.UIVersion
+	return nil
+}
+
 // Upload - file upload handler.
 func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 	objectAPI := web.ObjectAPI()

cmd/web-handlers_test.go

@@ -662,6 +662,45 @@ func testGetAuthWebHandler(obj ObjectLayer, instanceType string, t TestErrHandle
 	}
 }
 
+func TestWebCreateURLToken(t *testing.T) {
+	ExecObjectLayerTest(t, testCreateURLToken)
+}
+
+func testCreateURLToken(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	apiRouter := initTestWebRPCEndPoint(obj)
+	credentials := serverConfig.GetCredential()
+
+	authorization, err := getWebRPCToken(apiRouter, credentials.AccessKey, credentials.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	args := WebGenericArgs{}
+	tokenReply := &URLTokenReply{}
+
+	req, err := newTestWebRPCRequest("Web.CreateURLToken", authorization, args)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rec := httptest.NewRecorder()
+	apiRouter.ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("Expected the response status to be 200, but instead found `%d`", rec.Code)
+	}
+
+	err = getTestWebRPCResponse(rec, &tokenReply)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Ensure the token is valid now. It will expire later.
+	if !isAuthTokenValid(tokenReply.Token) {
+		t.Fatalf("token is not valid")
+	}
+}
+
 // Wrapper for calling Upload Handler
 func TestWebHandlerUpload(t *testing.T) {
 	ExecObjectLayerTest(t, testUploadWebHandler)
@@ -815,6 +854,32 @@ func testDownloadWebHandler(obj ObjectLayer, instanceType string, t TestErrHandl
 		t.Fatalf("The downloaded file is corrupted")
 	}
 
+	// Temporary token should succeed.
+	tmpToken, err := authenticateURL(credentials.AccessKey, credentials.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	code, bodyContent = test(tmpToken)
+
+	if code != http.StatusOK {
+		t.Fatalf("Expected the response status to be 200, but instead found `%d`", code)
+	}
+
+	if !bytes.Equal(bodyContent, content) {
+		t.Fatalf("The downloaded file is corrupted")
+	}
+
+	// Old token should fail.
+	code, bodyContent = test("eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE1MDAzMzIwOTUsImlhdCI6MTUwMDMzMjAzNSwic3ViIjoiRFlLSU01VlRZNDBJMVZQSE5VMTkifQ.tXQ45GJc8eOFet_a4VWVyeqJEOPWybotQYNr2zVxBpEOICkGbu_YWGhd9TkLLe1E65oeeiLHPdXSN8CzcbPoRA")
+	if code != http.StatusForbidden {
+		t.Fatalf("Expected the response status to be 403, but instead found `%d`", code)
+	}
+
+	if !bytes.Equal(bodyContent, bytes.NewBufferString("Authentication failed, check your access credentials").Bytes()) {
+		t.Fatalf("Expected authentication error message, got %v", bodyContent)
+	}
+
 	// Unauthenticated download should fail.
 	code, _ = test("")
 	if code != http.StatusForbidden {
@@ -848,7 +913,7 @@ func testWebHandlerDownloadZip(obj ObjectLayer, instanceType string, t TestErrHa
 	apiRouter := initTestWebRPCEndPoint(obj)
 	credentials := serverConfig.GetCredential()
 
-	authorization, err := getWebRPCToken(apiRouter, credentials.AccessKey, credentials.SecretKey)
+	authorization, err := authenticateURL(credentials.AccessKey, credentials.SecretKey)
 	if err != nil {
 		t.Fatal("Cannot authenticate")
 	}
@@ -1529,8 +1594,9 @@ func TestWebObjectLayerFaultyDisks(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed %v", err)
 	}
-	if storageInfoReply.StorageInfo.Total != -1 || storageInfoReply.StorageInfo.Free != -1 {
-		t.Fatalf("Should get negative values of Total and Free since disks are faulty ")
+	// if Total size is 0 it indicates faulty disk.
+	if storageInfoReply.StorageInfo.Total != 0 {
+		t.Fatalf("Should get zero Total size since disks are faulty ")
 	}
 
 	// Test authorization of Web.Download

cmd/web-router.go

@@ -83,6 +83,9 @@ func registerWebRouter(mux *router.Router) error {
 	// RPC handler at URI - /minio/webrpc
 	webBrowserRouter.Methods("POST").Path("/webrpc").Handler(webRPC)
 	webBrowserRouter.Methods("PUT").Path("/upload/{bucket}/{object:.+}").HandlerFunc(web.Upload)
+
+	// These methods use short-expiry tokens in the URLs. These tokens may unintentionally
+	// be logged, so a new one must be generated for each request.
 	webBrowserRouter.Methods("GET").Path("/download/{bucket}/{object:.+}").Queries("token", "{token:.*}").HandlerFunc(web.Download)
 	webBrowserRouter.Methods("POST").Path("/zip").Queries("token", "{token:.*}").HandlerFunc(web.DownloadZip)
 

cmd/xl-v1.go

@@ -231,10 +231,11 @@ func getStorageInfo(disks []StorageAPI) StorageInfo {
 
 	// Sort so that the first element is the smallest.
 	validDisksInfo := sortValidDisksInfo(disksInfo)
+	// If there are no valid disks, set total and free disks to 0
 	if len(validDisksInfo) == 0 {
 		return StorageInfo{
-			Total: -1,
-			Free:  -1,
+			Total: 0,
+			Free:  0,
 		}
 	}
 
@@ -242,8 +243,8 @@ func getStorageInfo(disks []StorageAPI) StorageInfo {
 	// Free as the total aggregated values. Total capacity is always
 	// the multiple of smallest disk among the disk list.
 	storageInfo := StorageInfo{
-		Total: validDisksInfo[0].Total * int64(onlineDisks) / 2,
-		Free:  validDisksInfo[0].Free * int64(onlineDisks) / 2,
+		Total: validDisksInfo[0].Total * uint64(onlineDisks) / 2,
+		Free:  validDisksInfo[0].Free * uint64(onlineDisks) / 2,
 	}
 
 	storageInfo.Backend.Type = Erasure

docs/config/README.md

@@ -34,8 +34,8 @@ $ tree ~/.minio
 |Field|Type|Description|
 |:---|:---|:---|
 |``credential``| | Auth credential for object storage and web access.|
-|``credential.accessKey`` | _string_ | Access key of 5 to 20 characters in length. You may override this field with `MINIO_ACCESS_KEY` environment variable.|
-|``credential.secretKey`` | _string_ | Secret key of 8 to 40 characters in length. You may override this field with `MINIO_SECRET_KEY` environment variable.|
+|``credential.accessKey`` | _string_ | Access key of minimum 5 characters in length. You may override this field with `MINIO_ACCESS_KEY` environment variable.|
+|``credential.secretKey`` | _string_ | Secret key of minimum 8 characters in length. You may override this field with `MINIO_SECRET_KEY` environment variable.|
 
 Example:
 

docs/deployment/kernel-tuning/README.md

@@ -57,13 +57,13 @@ Proper scheduler configuration makes sure Minio process gets adequate CPU time.
 - *`sched_min_granularity_ns`*: This parameter decides the minimum time a task will be be allowed to run on CPU before being pre-empted out. We recommend setting it to 10ms.
 
 ```sh
-sysctl -w kernel.sched_min_granularity_ns=10
+sysctl -w kernel.sched_min_granularity_ns=10000000
 ```
 
 - *`sched_wakeup_granularity_ns`*: Lowering this parameter improves wake-up latency and throughput for latency critical tasks, particularly when a short duty cycle load component must compete with CPU bound components.
 
 ```sh
-sysctl -w kernel.sched_wakeup_granularity_ns=15
+sysctl -w kernel.sched_wakeup_granularity_ns=15000000
 ```
 
 ## Tuning Disks

docs/erasure/README.md

@@ -26,12 +26,27 @@ Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quicks
 
 ### 2. Run Minio Server with Erasure Code.
 
-Example: Start Minio server in a 12 drives setup.
+Example: Start Minio server in a 12 drives setup, using Minio binary.
 
 ```sh
 minio server /mnt/export1/backend /mnt/export2/backend /mnt/export3/backend /mnt/export4/backend /mnt/export5/backend /mnt/export6/backend /mnt/export7/backend /mnt/export8/backend /mnt/export9/backend /mnt/export10/backend /mnt/export11/backend /mnt/export12/backend
 ```
 
+Example: Start Minio server in a 8 drives setup, using Minio Docker image. 
+
+```sh
+docker run -p 9000:9000 --name minio \
+  -v /mnt/export1/backend:/export1 \
+  -v /mnt/export2/backend:/export2 \
+  -v /mnt/export3/backend:/export3 \
+  -v /mnt/export4/backend:/export4 \
+  -v /mnt/export5/backend:/export5 \
+  -v /mnt/export6/backend:/export6 \
+  -v /mnt/export7/backend:/export7 \
+  -v /mnt/export8/backend:/export8 \
+  minio/minio server /export1 /export2 /export3 /export4 /export5 /export6 /export7 /export8
+```
+
 ### 3. Test your setup
 
 You may unplug drives randomly and continue to perform I/O on the system.

pkg/disk/disk.go

@@ -19,12 +19,14 @@ package disk
 // Info stat fs struct is container which holds following values
 // Total - total size of the volume / disk
 // Free - free size of the volume / disk
-// Type - file system type string
+// Files - total inodes available
+// Ffree - free inodes available
+// FSType - file system type
 type Info struct {
-	Total  int64
-	Free   int64
-	Files  int64
-	Ffree  int64
+	Total  uint64
+	Free   uint64
+	Files  uint64
+	Ffree  uint64
 	FSType string
 }
 

pkg/disk/stat_bsd.go

@@ -30,10 +30,10 @@ func GetInfo(path string) (info Info, err error) {
 		return Info{}, err
 	}
 	info = Info{}
-	info.Total = int64(s.Bsize) * int64(s.Blocks)
-	info.Free = int64(s.Bsize) * int64(s.Bavail)
-	info.Files = int64(s.Files)
-	info.Ffree = int64(s.Ffree)
+	info.Total = uint64(s.Bsize) * uint64(s.Blocks)
+	info.Free = uint64(s.Bsize) * uint64(s.Bavail)
+	info.Files = uint64(s.Files)
+	info.Ffree = uint64(s.Ffree)
 	info.FSType = getFSType(s.Fstypename)
 	return info, nil
 }

pkg/disk/stat_linux.go

@@ -30,10 +30,10 @@ func GetInfo(path string) (info Info, err error) {
 		return Info{}, err
 	}
 	info = Info{}
-	info.Total = int64(s.Bsize) * int64(s.Blocks)
-	info.Free = int64(s.Bsize) * int64(s.Bavail)
-	info.Files = int64(s.Files)
-	info.Ffree = int64(s.Ffree)
+	info.Total = uint64(s.Bsize) * uint64(s.Blocks)
+	info.Free = uint64(s.Bsize) * uint64(s.Bavail)
+	info.Files = uint64(s.Files)
+	info.Ffree = uint64(s.Ffree)
 	info.FSType = getFSType(int64(s.Type))
 	return info, nil
 }

pkg/disk/stat_openbsd.go

@@ -30,10 +30,10 @@ func GetInfo(path string) (info Info, err error) {
 		return Info{}, err
 	}
 	info = Info{}
-	info.Total = int64(s.F_bsize) * int64(s.F_blocks)
-	info.Free = int64(s.F_bsize) * int64(s.F_bavail)
-	info.Files = int64(s.F_files)
-	info.Ffree = int64(s.F_ffree)
+	info.Total = uint64(s.F_bsize) * uint64(s.F_blocks)
+	info.Free = uint64(s.F_bsize) * uint64(s.F_bavail)
+	info.Files = uint64(s.F_files)
+	info.Ffree = uint64(s.F_ffree)
 	info.FSType = getFSType(s.F_fstypename)
 	return info, nil
 }

pkg/disk/stat_windows.go

@@ -64,8 +64,8 @@ func GetInfo(path string) (info Info, err error) {
 		uintptr(unsafe.Pointer(&lpTotalNumberOfBytes)),
 		uintptr(unsafe.Pointer(&lpTotalNumberOfFreeBytes)))
 	info = Info{}
-	info.Total = int64(lpTotalNumberOfBytes)
-	info.Free = int64(lpFreeBytesAvailable)
+	info.Total = uint64(lpTotalNumberOfBytes)
+	info.Free = uint64(lpFreeBytesAvailable)
 	info.FSType = getFSType(path)
 
 	// Return values of GetDiskFreeSpace()
@@ -88,8 +88,8 @@ func GetInfo(path string) (info Info, err error) {
 		uintptr(unsafe.Pointer(&lpNumberOfFreeClusters)),
 		uintptr(unsafe.Pointer(&lpTotalNumberOfClusters)))
 
-	info.Files = int64(lpTotalNumberOfClusters)
-	info.Ffree = int64(lpNumberOfFreeClusters)
+	info.Files = uint64(lpTotalNumberOfClusters)
+	info.Ffree = uint64(lpNumberOfFreeClusters)
 
 	return info, nil
 }

pkg/http/listener.go

@@ -20,6 +20,7 @@ import (
 	"crypto/tls"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"strings"
@@ -88,6 +89,16 @@ type httpListener struct {
 	errorLogFunc           func(error, string, ...interface{}) // function to be called on errors.
 }
 
+// ignoreErr returns true if error is due to a network timeout or
+// io.EOF and false otherwise
+func ignoreErr(err error) bool {
+	if nErr, ok := err.(*net.OpError); ok {
+		return nErr.Timeout()
+	}
+
+	return err == io.EOF
+}
+
 // start - starts separate goroutine for each TCP listener.  A valid insecure/TLS HTTP new connection is passed to httpListener.acceptCh.
 func (listener *httpListener) start() {
 	listener.acceptCh = make(chan acceptResult)
@@ -122,9 +133,16 @@ func (listener *httpListener) start() {
 		data, err := bufconn.Peek(methodMaxLen)
 		if err != nil {
 			if listener.errorLogFunc != nil {
-				listener.errorLogFunc(err,
-					"Error in reading from new connection %s at server %s",
-					bufconn.RemoteAddr(), bufconn.LocalAddr())
+				// Peek could fail legitimately when clients abruptly close
+				// connection. E.g. Chrome browser opens connections speculatively to
+				// speed up loading of a web page. Peek may also fail due to network
+				// saturation on a transport with read timeout set. All other kind of
+				// errors should be logged for further investigation. Thanks @brendanashworth.
+				if !ignoreErr(err) {
+					listener.errorLogFunc(err,
+						"Error in reading from new connection %s at server %s",
+						bufconn.RemoteAddr(), bufconn.LocalAddr())
+				}
 			}
 			bufconn.Close()
 			return

pkg/http/listener_test.go

@@ -815,3 +815,42 @@ func TestHTTPListenerAcceptParallel(t *testing.T) {
 		listener.Close()
 	}
 }
+
+type myTimeoutErr struct {
+	timeout bool
+}
+
+func (m *myTimeoutErr) Error() string { return fmt.Sprintf("myTimeoutErr: %v", m.timeout) }
+func (m *myTimeoutErr) Timeout() bool { return m.timeout }
+
+// Test for ignoreErr helper function
+func TestIgnoreErr(t *testing.T) {
+	testCases := []struct {
+		err  error
+		want bool
+	}{
+		{
+			err:  io.EOF,
+			want: true,
+		},
+		{
+			err:  &net.OpError{Err: &myTimeoutErr{timeout: true}},
+			want: true,
+		},
+		{
+			err:  &net.OpError{Err: &myTimeoutErr{timeout: false}},
+			want: false,
+		},
+		{
+			err:  io.ErrUnexpectedEOF,
+			want: false,
+		},
+	}
+
+	for i, tc := range testCases {
+		if actual := ignoreErr(tc.err); actual != tc.want {
+			t.Errorf("Test case %d: Expected %v but got %v for %v", i+1,
+				tc.want, actual, tc.err)
+		}
+	}
+}

pkg/http/server.go

@@ -118,6 +118,10 @@ func (srv *Server) Start() (err error) {
 
 // Shutdown - shuts down HTTP server.
 func (srv *Server) Shutdown() error {
+	if srv.listener == nil {
+		return errors.New("server not initialized")
+	}
+
 	if atomic.AddUint32(&srv.inShutdown, 1) > 1 {
 		// shutdown in progress
 		return errors.New("http server already in shutdown")