Support 1GB disk size (#6137 )

Pivotal CF by default has 1GB disk option which causes minio to not start
fix object rebinding SSE-C security guarantee violation (#6121 )
2026-07-31 15:55:18 +03:00 · 2018-07-09 18:23:49 -07:00 · 2018-07-09 17:18:28 -07:00 · 2018-07-09 02:18:48 -07:00 · 2018-07-06 17:07:11 -07:00 · 2018-07-05 16:33:37 -07:00
51 changed files with 1292 additions and 837 deletions
@@ -8,27 +8,35 @@ dist: trusty

 language: go

-os:
- linux
-
-env:
- ARCH=x86_64
+matrix:
+  include:
+    - os: linux
+      env:
+        - ARCH=x86_64
+      go: 1.10.1
+      script:
+        - make
+        - diff -au <(gofmt -s -d cmd) <(printf "")
+        - diff -au <(gofmt -s -d pkg) <(printf "")
+        - make test GOFLAGS="-timeout 15m -race -v"
+        - make coverage
+        - node --version
+        - cd browser && yarn && yarn test && cd ..
+    - os: linux-ppc64le
+      env:
+        - ARCH=ppc64le
+      go: 1.10.1
+      script:
+        - make
+        - diff -au <(gofmt -s -d cmd) <(printf "")
+        - diff -au <(gofmt -s -d pkg) <(printf "")
+        - make test GOFLAGS="-timeout 15m -v"
+        - make coverage
+        - node --version
+        - cd browser && yarn && yarn test && cd ..

 before_install:
 - nvm install stable

-script:
-## Run all the tests
- make
- diff -au <(gofmt -s -d cmd) <(printf "")
- diff -au <(gofmt -s -d pkg) <(printf "")
- make test GOFLAGS="-timeout 15m -race -v"
- make coverage
- node --version
- cd browser && yarn && yarn test && cd ..
-
 after_success:
- bash <(curl -s https://codecov.io/bash)
-
-go:
- '1.10.1'
+- bash <(curl -s https://codecov.io/bash)
@@ -89,11 +89,11 @@ check_minimum_version() {

 assert_is_supported_arch() {
    case "${ARCH}" in
-        x86_64 | amd64 | aarch64 | arm* )
+        x86_64 | amd64 | aarch64 | ppc64le | arm* )
            return
            ;;
        *)
-            echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, arm*]"
+            echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*]"
            exit 1
    esac
 }
@@ -692,7 +692,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	ErrStorageFull: {
 		Code:           "XMinioStorageFull",
 		Description:    "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
-		HTTPStatusCode: http.StatusInternalServerError,
+		HTTPStatusCode: http.StatusInsufficientStorage,
 	},
 	ErrRequestBodyParse: {
 		Code:           "XMinioRequestBodyParse",
@@ -80,17 +80,9 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}

-	// In ListObjectsV2 'continuation-token' is the marker.
-	marker := token
-	// Check if 'continuation-token' is empty.
-	if token == "" {
-		// Then we need to use 'start-after' as marker instead.
-		marker = startAfter
-	}
-
 	// Validate the query params before beginning to serve the request.
 	// fetch-owner is not validated since it is a boolean
-	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
+	if s3Error := validateListObjectsArgs(prefix, token, delimiter, maxKeys); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -101,7 +93,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
 	// marshaled into S3 compatible XML header.
-	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, marker, delimiter, maxKeys, fetchOwner, startAfter)
+	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -34,14 +34,14 @@ import (

 	"github.com/gorilla/mux"

+	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/sync/errgroup"
-
-	"github.com/minio/minio-go/pkg/set"
 )

 // Check if there are buckets on server without corresponding entry in etcd backend and
@@ -375,7 +375,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,

 	// Get host and port from Request.RemoteAddr failing which
 	// fill them with empty strings.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -623,7 +623,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
 			}
-			reader, err = newEncryptReader(hashReader, key, metadata)
+			reader, err = newEncryptReader(hashReader, key, bucket, object, metadata)
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -648,7 +648,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	w.Header().Set("Location", location)

 	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -753,10 +753,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.

 	globalNotificationSys.RemoveNotification(bucket)
 	globalPolicySys.Remove(bucket)
-	for nerr := range globalNotificationSys.DeleteBucket(bucket) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.DeleteBucket(ctx, bucket)

 	if globalDNSConfig != nil {
 		if err := globalDNSConfig.Delete(bucket); err != nil {
@@ -146,10 +146,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,

 	rulesMap := config.ToRulesMap()
 	globalNotificationSys.AddRulesMap(bucketName, rulesMap)
-	for nerr := range globalNotificationSys.PutBucketNotification(bucketName, rulesMap) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.PutBucketNotification(ctx, bucketName, rulesMap)

 	writeSuccessResponseHeadersOnly(w)
 }
@@ -260,11 +257,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}

-	errCh := globalNotificationSys.ListenBucketNotification(bucketName, eventNames, pattern, target.ID(), *thisAddr)
-	for nerr := range errCh {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.ListenBucketNotification(ctx, bucketName, eventNames, pattern, target.ID(), *thisAddr)

 	<-target.DoneCh

@@ -91,10 +91,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}

 	globalPolicySys.Set(bucket, *bucketPolicy)
-	for nerr := range globalNotificationSys.SetBucketPolicy(bucket, bucketPolicy) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.SetBucketPolicy(ctx, bucket, bucketPolicy)

 	// Success.
 	writeSuccessNoContent(w)
@@ -130,10 +127,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	}

 	globalPolicySys.Remove(bucket)
-	for nerr := range globalNotificationSys.RemoveBucketPolicy(bucket) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.RemoveBucketPolicy(ctx, bucket)

 	// Success.
 	writeSuccessNoContent(w)
@@ -32,7 +32,7 @@
 //       Input: ClientKey, bucket, object, metadata, object_data
 //       -              IV := Random({0,1}²⁵⁶)
 //       -       ObjectKey := SHA256(ClientKey || Random({0,1}²⁵⁶))
-//       -       KeyEncKey := HMAC-SHA256(ClientKey, IV || bucket || object)
+//       -       KeyEncKey := HMAC-SHA256(ClientKey, IV || 'SSE-C' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
 //       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
 //       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
 //       -        metadata <- IV
@@ -43,7 +43,7 @@
 //       Input: ClientKey, bucket, object, metadata, enc_object_data
 //       -          IV <- metadata
 //       -   SealedKey <- metadata
-//       -   KeyEncKey := HMAC-SHA256(ClientKey, IV || bucket || object)
+//       -   KeyEncKey := HMAC-SHA256(ClientKey, IV || 'SSE-C' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
 //       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
 //       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
 //       Output: object_data
@@ -64,7 +64,7 @@
 //       Input: MasterKey, bucket, object, metadata, object_data
 //       -              IV := Random({0,1}²⁵⁶)
 //       -       ObjectKey := SHA256(MasterKey || Random({0,1}²⁵⁶))
-//       -       KeyEncKey := HMAC-SHA256(MasterKey, IV || bucket || object)
+//       -       KeyEncKey := HMAC-SHA256(MasterKey, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
 //       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
 //       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
 //       -        metadata <- IV
@@ -75,7 +75,7 @@
 //       Input: MasterKey, bucket, object, metadata, enc_object_data
 //       -          IV <- metadata
 //       -   SealedKey <- metadata
-//       -   KeyEncKey := HMAC-SHA256(MasterKey, IV || bucket || object)
+//       -   KeyEncKey := HMAC-SHA256(MasterKey, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
 //       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
 //       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
 //       Output: object_data
@@ -92,7 +92,7 @@
 //       -     Key, EncKey := Generate(KeyID)
 //       -              IV := Random({0,1}²⁵⁶)
 //       -       ObjectKey := SHA256(Key, Random({0,1}²⁵⁶))
-//       -       KeyEncKey := HMAC-SHA256(Key, IV || bucket || object)
+//       -       KeyEncKey := HMAC-SHA256(Key, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
 //       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
 //       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
 //       -        metadata <- IV
@@ -108,7 +108,7 @@
 //       -          IV <- metadata
 //       -   SealedKey <- metadata
 //       -         Key := Unseal(KeyID, EncKey)
-//       -   KeyEncKey := HMAC-SHA256(Key, IV || bucket || object)
+//       -   KeyEncKey := HMAC-SHA256(Key, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
 //       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
 //       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
 //       Output: object_data
@@ -19,6 +19,9 @@ package cmd
 import (
 	"encoding/json"
 	"path/filepath"
+	"strings"
+
+	"github.com/minio/minio/pkg/ellipses"
 )

 // CacheConfig represents cache config settings
@@ -52,12 +55,42 @@ func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {

 // Parses given cacheDrivesEnv and returns a list of cache drives.
 func parseCacheDrives(drives []string) ([]string, error) {
+	if len(drives) == 0 {
+		return drives, nil
+	}
+	var endpoints []string
 	for _, d := range drives {
+		if ellipses.HasEllipses(d) {
+			s, err := parseCacheDrivePaths(d)
+			if err != nil {
+				return nil, err
+			}
+			endpoints = append(endpoints, s...)
+		} else {
+			endpoints = append(endpoints, d)
+		}
+	}
+
+	for _, d := range endpoints {
 		if !filepath.IsAbs(d) {
 			return nil, uiErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
 		}
 	}
-	return drives, nil
+	return endpoints, nil
+}
+
+// Parses all arguments and returns a slice of drive paths following the ellipses pattern.
+func parseCacheDrivePaths(arg string) (ep []string, err error) {
+	patterns, perr := ellipses.FindEllipsesPatterns(arg)
+	if perr != nil {
+		return []string{}, uiErrInvalidCacheDrivesValue(nil).Msg(perr.Error())
+	}
+
+	for _, lbls := range patterns.Expand() {
+		ep = append(ep, strings.Join(lbls, ""))
+	}
+
+	return ep, nil
 }

 // Parses given cacheExcludesEnv and returns a list of cache exclude patterns.
@@ -41,12 +41,32 @@ func TestParseCacheDrives(t *testing.T) {
 			expectedPatterns []string
 			success          bool
 		}{"C:/home/drive1;C:/home/drive2;C:/home/drive3", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"C:/home/drive{1...3}", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"C:/home/drive{1..3}", []string{}, false})
 	} else {
 		testCases = append(testCases, struct {
 			driveStr         string
 			expectedPatterns []string
 			success          bool
 		}{"/home/drive1;/home/drive2;/home/drive3", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"/home/drive{1...3}", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"/home/drive{1..3}", []string{}, false})
 	}
 	for i, testCase := range testCases {
 		drives, err := parseCacheDrives(strings.Split(testCase.driveStr, cacheEnvDelimiter))
@@ -28,6 +28,7 @@ import (
 	"errors"
 	"io"
 	"net/http"
+	"path"
 	"strconv"

 	"github.com/minio/minio/cmd/logger"
@@ -144,9 +145,20 @@ const (
 	ServerSideEncryptionSealedKey = ReservedMetadataPrefix + "Server-Side-Encryption-Sealed-Key"
 )

-// SSESealAlgorithmDareSha256 specifies DARE as authenticated en/decryption scheme and SHA256 as cryptographic
-// hash function.
-const SSESealAlgorithmDareSha256 = "DARE-SHA256"
+const (
+	// SSESealAlgorithmDareSha256 specifies DARE as authenticated en/decryption scheme and SHA256 as cryptographic
+	// hash function. The key derivation of DARE-SHA256 is not optimal and does not include the object path.
+	// It is considered legacy and should not be used anymore.
+	SSESealAlgorithmDareSha256 = "DARE-SHA256"
+
+	// SSESealAlgorithmDareV2HmacSha256 specifies DAREv2 as authenticated en/decryption scheme and SHA256 as cryptographic
+	// hash function for the HMAC PRF.
+	SSESealAlgorithmDareV2HmacSha256 = "DAREv2-HMAC-SHA256"
+
+	// SSEDomain specifies the domain for the derived key - in this case the
+	// key should be used for SSE-C.
+	SSEDomain = "SSE-C"
+)

 // hasSSECustomerHeader returns true if the given HTTP header
 // contains server-side-encryption with customer provided key fields.
@@ -250,10 +262,11 @@ func ParseSSECustomerHeader(header http.Header) (key []byte, err error) {
 }

 // This function rotates old to new key.
-func rotateKey(oldKey []byte, newKey []byte, metadata map[string]string) error {
+func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map[string]string) error {
 	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident

-	if metadata[ServerSideEncryptionSealAlgorithm] != SSESealAlgorithmDareSha256 { // currently DARE-SHA256 is the only option
+	algorithm := metadata[ServerSideEncryptionSealAlgorithm]
+	if algorithm != SSESealAlgorithmDareSha256 && algorithm != SSESealAlgorithmDareV2HmacSha256 {
 		return errObjectTampered
 	}
 	iv, err := base64.StdEncoding.DecodeString(metadata[ServerSideEncryptionIV])
@@ -265,14 +278,33 @@ func rotateKey(oldKey []byte, newKey []byte, metadata map[string]string) error {
 		return errObjectTampered
 	}

-	sha := sha256.New() // derive key encryption key
-	sha.Write(oldKey)
-	sha.Write(iv)
-	keyEncryptionKey := sha.Sum(nil)
+	var (
+		minDAREVersion   byte
+		keyEncryptionKey [32]byte
+	)
+	switch algorithm {
+	default:
+		return errObjectTampered
+	case SSESealAlgorithmDareSha256: // legacy key-encryption-key derivation
+		minDAREVersion = sio.Version10
+		sha := sha256.New()
+		sha.Write(oldKey)
+		sha.Write(iv)
+		sha.Sum(keyEncryptionKey[:0])
+	case SSESealAlgorithmDareV2HmacSha256: // key-encryption-key derivation - See: crypto/doc.go
+		minDAREVersion = sio.Version20
+		mac := hmac.New(sha256.New, oldKey)
+		mac.Write(iv)
+		mac.Write([]byte(SSEDomain))
+		mac.Write([]byte(SSESealAlgorithmDareV2HmacSha256))
+		mac.Write([]byte(path.Join(bucket, object)))
+		mac.Sum(keyEncryptionKey[:0])
+	}

 	objectEncryptionKey := bytes.NewBuffer(nil) // decrypt object encryption key
 	n, err := sio.Decrypt(objectEncryptionKey, bytes.NewReader(sealedKey), sio.Config{
-		Key: keyEncryptionKey,
+		MinVersion: minDAREVersion,
+		Key:        keyEncryptionKey[:],
 	})
 	if n != 32 || err != nil { // Either the provided key does not match or the object was tampered.
 		if subtle.ConstantTimeCompare(oldKey, newKey) == 1 {
@@ -280,46 +312,34 @@ func rotateKey(oldKey []byte, newKey []byte, metadata map[string]string) error {
 		}
 		return errSSEKeyMismatch // To provide strict AWS S3 compatibility we return: access denied.
 	}
-	if subtle.ConstantTimeCompare(oldKey, newKey) == 1 {
-		return nil // we don't need to rotate keys if newKey == oldKey
+	if subtle.ConstantTimeCompare(oldKey, newKey) == 1 && algorithm != SSESealAlgorithmDareSha256 {
+		return nil // we don't need to rotate keys if newKey == oldKey but we may have to upgrade KDF algorithm
 	}

-	nonce := make([]byte, 32) // generate random values for key derivation
-	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
-		return err
-	}
-
-	niv := sha256.Sum256(nonce[:]) // derive key encryption key
-	sha = sha256.New()
-	sha.Write(newKey)
-	sha.Write(niv[:])
-	keyEncryptionKey = sha.Sum(nil)
+	mac := hmac.New(sha256.New, newKey) // key-encryption-key derivation - See: crypto/doc.go
+	mac.Write(iv)
+	mac.Write([]byte(SSEDomain))
+	mac.Write([]byte(SSESealAlgorithmDareV2HmacSha256))
+	mac.Write([]byte(path.Join(bucket, object)))
+	mac.Sum(keyEncryptionKey[:0])

 	sealedKeyW := bytes.NewBuffer(nil) // sealedKey := 16 byte header + 32 byte payload + 16 byte tag
 	n, err = sio.Encrypt(sealedKeyW, bytes.NewReader(objectEncryptionKey.Bytes()), sio.Config{
-		Key: keyEncryptionKey,
+		Key: keyEncryptionKey[:],
 	})
 	if n != 64 || err != nil {
 		return errors.New("failed to seal object encryption key") // if this happens there's a bug in the code (may panic ?)
 	}

-	metadata[ServerSideEncryptionIV] = base64.StdEncoding.EncodeToString(niv[:])
-	metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareSha256
+	metadata[ServerSideEncryptionIV] = base64.StdEncoding.EncodeToString(iv[:])
+	metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareV2HmacSha256
 	metadata[ServerSideEncryptionSealedKey] = base64.StdEncoding.EncodeToString(sealedKeyW.Bytes())
 	return nil
 }

-func newEncryptMetadata(key []byte, metadata map[string]string) ([]byte, error) {
+func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) {
 	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
-
-	// security notice:
-	//  - If the first 32 bytes of the random value are ever repeated under the same client-provided
-	//    key the encrypted object will not be tamper-proof. [ P(coll) ~= 1 / 2^(256 / 2)]
-	//  - If the last 32 bytes of the random value are ever repeated under the same client-provided
-	//    key an adversary may be able to extract the object encryption key. This depends on the
-	//    authenticated en/decryption scheme. The DARE format will generate an 8 byte nonce which must
-	//    be repeated in addition to reveal the object encryption key.
-	//    [ P(coll) ~= 1 / 2^((256 + 64) / 2) ]
+	// See crypto/doc.go for detailed description
 	nonce := make([]byte, 32+SSEIVSize) // generate random values for key derivation
 	if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
 		return nil, err
@@ -329,11 +349,13 @@ func newEncryptMetadata(key []byte, metadata map[string]string) ([]byte, error)
 	sha.Write(nonce[:32])
 	objectEncryptionKey := sha.Sum(nil)

-	iv := sha256.Sum256(nonce[32:]) // derive key encryption key
-	sha = sha256.New()
-	sha.Write(key)
-	sha.Write(iv[:])
-	keyEncryptionKey := sha.Sum(nil)
+	iv := sha256.Sum256(nonce[32:]) // key-encryption-key derivation - See: crypto/doc.go
+	mac := hmac.New(sha256.New, key)
+	mac.Write(iv[:])
+	mac.Write([]byte(SSEDomain))
+	mac.Write([]byte(SSESealAlgorithmDareV2HmacSha256))
+	mac.Write([]byte(path.Join(bucket, object)))
+	keyEncryptionKey := mac.Sum(nil)

 	sealedKey := bytes.NewBuffer(nil) // sealedKey := 16 byte header + 32 byte payload + 16 byte tag
 	n, err := sio.Encrypt(sealedKey, bytes.NewReader(objectEncryptionKey), sio.Config{
@@ -344,14 +366,14 @@ func newEncryptMetadata(key []byte, metadata map[string]string) ([]byte, error)
 	}

 	metadata[ServerSideEncryptionIV] = base64.StdEncoding.EncodeToString(iv[:])
-	metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareSha256
+	metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareV2HmacSha256
 	metadata[ServerSideEncryptionSealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Bytes())

 	return objectEncryptionKey, nil
 }

-func newEncryptReader(content io.Reader, key []byte, metadata map[string]string) (io.Reader, error) {
-	objectEncryptionKey, err := newEncryptMetadata(key, metadata)
+func newEncryptReader(content io.Reader, key []byte, bucket, object string, metadata map[string]string) (io.Reader, error) {
+	objectEncryptionKey, err := newEncryptMetadata(key, bucket, object, metadata)
 	if err != nil {
 		return nil, err
 	}
@@ -367,29 +389,26 @@ func newEncryptReader(content io.Reader, key []byte, metadata map[string]string)
 // EncryptRequest takes the client provided content and encrypts the data
 // with the client provided key. It also marks the object as client-side-encrypted
 // and sets the correct headers.
-func EncryptRequest(content io.Reader, r *http.Request, metadata map[string]string) (io.Reader, error) {
+func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, metadata map[string]string) (io.Reader, error) {
 	key, err := ParseSSECustomerRequest(r)
 	if err != nil {
 		return nil, err
 	}
-	return newEncryptReader(content, key, metadata)
+	return newEncryptReader(content, key, bucket, object, metadata)
 }

 // DecryptCopyRequest decrypts the object with the client provided key. It also removes
 // the client-side-encryption metadata from the object and sets the correct headers.
-func DecryptCopyRequest(client io.Writer, r *http.Request, metadata map[string]string) (io.WriteCloser, error) {
+func DecryptCopyRequest(client io.Writer, r *http.Request, bucket, object string, metadata map[string]string) (io.WriteCloser, error) {
 	key, err := ParseSSECopyCustomerRequest(r)
 	if err != nil {
 		return nil, err
 	}
 	delete(metadata, SSECopyCustomerKey) // make sure we do not save the key by accident
-	return newDecryptWriter(client, key, 0, metadata)
+	return newDecryptWriter(client, key, bucket, object, 0, metadata)
 }

-func decryptObjectInfo(key []byte, metadata map[string]string) ([]byte, error) {
-	if metadata[ServerSideEncryptionSealAlgorithm] != SSESealAlgorithmDareSha256 { // currently DARE-SHA256 is the only option
-		return nil, errObjectTampered
-	}
+func decryptObjectInfo(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) {
 	iv, err := base64.StdEncoding.DecodeString(metadata[ServerSideEncryptionIV])
 	if err != nil || len(iv) != SSEIVSize {
 		return nil, errObjectTampered
@@ -399,14 +418,33 @@ func decryptObjectInfo(key []byte, metadata map[string]string) ([]byte, error) {
 		return nil, errObjectTampered
 	}

-	sha := sha256.New() // derive key encryption key
-	sha.Write(key)
-	sha.Write(iv)
-	keyEncryptionKey := sha.Sum(nil)
+	var (
+		minDAREVersion   byte
+		keyEncryptionKey [32]byte
+	)
+	switch algorithm := metadata[ServerSideEncryptionSealAlgorithm]; algorithm {
+	default:
+		return nil, errObjectTampered
+	case SSESealAlgorithmDareSha256: // legacy key-encryption-key derivation
+		minDAREVersion = sio.Version10
+		sha := sha256.New()
+		sha.Write(key)
+		sha.Write(iv)
+		sha.Sum(keyEncryptionKey[:0])
+	case SSESealAlgorithmDareV2HmacSha256: // key-encryption-key derivation - See: crypto/doc.go
+		minDAREVersion = sio.Version20
+		mac := hmac.New(sha256.New, key)
+		mac.Write(iv)
+		mac.Write([]byte(SSEDomain))
+		mac.Write([]byte(SSESealAlgorithmDareV2HmacSha256))
+		mac.Write([]byte(path.Join(bucket, object)))
+		mac.Sum(keyEncryptionKey[:0])
+	}

 	objectEncryptionKey := bytes.NewBuffer(nil) // decrypt object encryption key
 	n, err := sio.Decrypt(objectEncryptionKey, bytes.NewReader(sealedKey), sio.Config{
-		Key: keyEncryptionKey,
+		MinVersion: minDAREVersion,
+		Key:        keyEncryptionKey[:],
 	})
 	if n != 32 || err != nil {
 		// Either the provided key does not match or the object was tampered.
@@ -416,11 +454,10 @@ func decryptObjectInfo(key []byte, metadata map[string]string) ([]byte, error) {
 	return objectEncryptionKey.Bytes(), nil
 }

-func newDecryptWriter(client io.Writer, key []byte, seqNumber uint32, metadata map[string]string) (io.WriteCloser, error) {
-	objectEncryptionKey, err := decryptObjectInfo(key, metadata)
+func newDecryptWriter(client io.Writer, key []byte, bucket, object string, seqNumber uint32, metadata map[string]string) (io.WriteCloser, error) {
+	objectEncryptionKey, err := decryptObjectInfo(key, bucket, object, metadata)
 	if err != nil {
 		return nil, err
-
 	}
 	return newDecryptWriterWithObjectKey(client, objectEncryptionKey, seqNumber, metadata)
 }
@@ -443,19 +480,19 @@ func newDecryptWriterWithObjectKey(client io.Writer, objectEncryptionKey []byte,

 // DecryptRequestWithSequenceNumber decrypts the object with the client provided key. It also removes
 // the client-side-encryption metadata from the object and sets the correct headers.
-func DecryptRequestWithSequenceNumber(client io.Writer, r *http.Request, seqNumber uint32, metadata map[string]string) (io.WriteCloser, error) {
+func DecryptRequestWithSequenceNumber(client io.Writer, r *http.Request, bucket, object string, seqNumber uint32, metadata map[string]string) (io.WriteCloser, error) {
 	key, err := ParseSSECustomerRequest(r)
 	if err != nil {
 		return nil, err
 	}
 	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
-	return newDecryptWriter(client, key, seqNumber, metadata)
+	return newDecryptWriter(client, key, bucket, object, seqNumber, metadata)
 }

 // DecryptRequest decrypts the object with the client provided key. It also removes
 // the client-side-encryption metadata from the object and sets the correct headers.
-func DecryptRequest(client io.Writer, r *http.Request, metadata map[string]string) (io.WriteCloser, error) {
-	return DecryptRequestWithSequenceNumber(client, r, 0, metadata)
+func DecryptRequest(client io.Writer, r *http.Request, bucket, object string, metadata map[string]string) (io.WriteCloser, error) {
+	return DecryptRequestWithSequenceNumber(client, r, bucket, object, 0, metadata)
 }

 // DecryptBlocksWriter - decrypts multipart parts, while implementing a io.Writer compatible interface.
@@ -469,9 +506,10 @@ type DecryptBlocksWriter struct {
 	// Current part index
 	partIndex int
 	// Parts information
-	parts    []objectPartInfo
-	req      *http.Request
-	metadata map[string]string
+	parts          []objectPartInfo
+	req            *http.Request
+	bucket, object string
+	metadata       map[string]string

 	partEncRelOffset int64

@@ -499,7 +537,7 @@ func (w *DecryptBlocksWriter) buildDecrypter(partID int) error {
 		return err
 	}

-	objectEncryptionKey, err := decryptObjectInfo(key, m)
+	objectEncryptionKey, err := decryptObjectInfo(key, w.bucket, w.object, m)
 	if err != nil {
 		return err
 	}
@@ -594,14 +632,14 @@ func (w *DecryptBlocksWriter) Close() error {
 // DecryptAllBlocksCopyRequest - setup a struct which can decrypt many concatenated encrypted data
 // parts information helps to know the boundaries of each encrypted data block, this function decrypts
 // all parts starting from part-1.
-func DecryptAllBlocksCopyRequest(client io.Writer, r *http.Request, objInfo ObjectInfo) (io.WriteCloser, int64, error) {
-	w, _, size, err := DecryptBlocksRequest(client, r, 0, objInfo.Size, objInfo, true)
+func DecryptAllBlocksCopyRequest(client io.Writer, r *http.Request, bucket, object string, objInfo ObjectInfo) (io.WriteCloser, int64, error) {
+	w, _, size, err := DecryptBlocksRequest(client, r, bucket, object, 0, objInfo.Size, objInfo, true)
 	return w, size, err
 }

 // DecryptBlocksRequest - setup a struct which can decrypt many concatenated encrypted data
 // parts information helps to know the boundaries of each encrypted data block.
-func DecryptBlocksRequest(client io.Writer, r *http.Request, startOffset, length int64, objInfo ObjectInfo, copySource bool) (io.WriteCloser, int64, int64, error) {
+func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object string, startOffset, length int64, objInfo ObjectInfo, copySource bool) (io.WriteCloser, int64, int64, error) {
 	seqNumber, encStartOffset, encLength := getEncryptedStartOffset(startOffset, length)

 	// Encryption length cannot be bigger than the file size, if it is
@@ -614,9 +652,9 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, startOffset, length
 		var writer io.WriteCloser
 		var err error
 		if copySource {
-			writer, err = DecryptCopyRequest(client, r, objInfo.UserDefined)
+			writer, err = DecryptCopyRequest(client, r, bucket, object, objInfo.UserDefined)
 		} else {
-			writer, err = DecryptRequestWithSequenceNumber(client, r, seqNumber, objInfo.UserDefined)
+			writer, err = DecryptRequestWithSequenceNumber(client, r, bucket, object, seqNumber, objInfo.UserDefined)
 		}
 		if err != nil {
 			return nil, 0, 0, err
@@ -656,6 +694,8 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, startOffset, length
 		parts:             objInfo.Parts,
 		partIndex:         partStartIndex,
 		req:               r,
+		bucket:            bucket,
+		object:            object,
 		customerKeyHeader: r.Header.Get(SSECustomerKey),
 		copySource:        copySource,
 	}
@@ -308,7 +308,7 @@ func TestEncryptRequest(t *testing.T) {
 		for k, v := range test.header {
 			req.Header.Set(k, v)
 		}
-		_, err := EncryptRequest(content, req, test.metadata)
+		_, err := EncryptRequest(content, req, "bucket", "object", test.metadata)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
 		}
@@ -328,11 +328,14 @@ func TestEncryptRequest(t *testing.T) {
 }

 var decryptRequestTests = []struct {
-	header     map[string]string
-	metadata   map[string]string
-	shouldFail bool
+	bucket, object string
+	header         map[string]string
+	metadata       map[string]string
+	shouldFail     bool
 }{
 	{
+		bucket: "bucket",
+		object: "object",
 		header: map[string]string{
 			SSECustomerAlgorithm: "AES256",
 			SSECustomerKey:       "MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=",
@@ -346,6 +349,23 @@ var decryptRequestTests = []struct {
 		shouldFail: false,
 	},
 	{
+		bucket: "bucket",
+		object: "object",
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=",
+			SSECustomerKeyMD5:    "7PpPLAK26ONlVUGOWlusfg==",
+		},
+		metadata: map[string]string{
+			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareV2HmacSha256,
+			ServerSideEncryptionIV:            "qEqmsONcorqlcZXJxaw32H04eyXyXwUgjHzlhkaIYrU=",
+			ServerSideEncryptionSealedKey:     "IAAfAIM14ugTGcM/dIrn4iQMrkl1sjKyeBQ8FBEvRebYj8vWvxG+0cJRpC6NXRU1wJN50JaUOATjO7kz0wZ2mA==",
+		},
+		shouldFail: false,
+	},
+	{
+		bucket: "bucket",
+		object: "object",
 		header: map[string]string{
 			SSECustomerAlgorithm: "AES256",
 			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
@@ -359,6 +379,8 @@ var decryptRequestTests = []struct {
 		shouldFail: true,
 	},
 	{
+		bucket: "bucket",
+		object: "object",
 		header: map[string]string{
 			SSECustomerAlgorithm: "AES256",
 			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
@@ -372,6 +394,8 @@ var decryptRequestTests = []struct {
 		shouldFail: true,
 	},
 	{
+		bucket: "bucket",
+		object: "object",
 		header: map[string]string{
 			SSECustomerAlgorithm: "AES256",
 			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
@@ -384,21 +408,39 @@ var decryptRequestTests = []struct {
 		},
 		shouldFail: true,
 	},
+	{
+		bucket: "bucket",
+		object: "object-2",
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=",
+			SSECustomerKeyMD5:    "7PpPLAK26ONlVUGOWlusfg==",
+		},
+		metadata: map[string]string{
+			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareV2HmacSha256,
+			ServerSideEncryptionIV:            "qEqmsONcorqlcZXJxaw32H04eyXyXwUgjHzlhkaIYrU=",
+			ServerSideEncryptionSealedKey:     "IAAfAIM14ugTGcM/dIrn4iQMrkl1sjKyeBQ8FBEvRebYj8vWvxG+0cJRpC6NXRU1wJN50JaUOATjO7kz0wZ2mA==",
+		},
+		shouldFail: true,
+	},
 }

 func TestDecryptRequest(t *testing.T) {
 	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
 	globalIsSSL = true
-	for i, test := range decryptRequestTests {
+	for i, test := range decryptRequestTests[1:] {
 		client := bytes.NewBuffer(nil)
 		req := &http.Request{Header: http.Header{}}
 		for k, v := range test.header {
 			req.Header.Set(k, v)
 		}
-		_, err := DecryptRequest(client, req, test.metadata)
+		_, err := DecryptRequest(client, req, test.bucket, test.object, test.metadata)
 		if err != nil && !test.shouldFail {
 			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
 		}
+		if err == nil && test.shouldFail {
+			t.Fatalf("Test %d: should fail but passed", i)
+		}
 		if key, ok := test.metadata[SSECustomerKey]; ok {
 			t.Errorf("Test %d: Client provided key survived in metadata - key: %s", i, key)
 		}
@@ -1250,7 +1250,12 @@ func (fs *FSObjects) DeleteBucketPolicy(ctx context.Context, bucket string) erro

 // ListObjectsV2 lists all blobs in bucket filtered by prefix
 func (fs *FSObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
-	loi, err := fs.ListObjects(ctx, bucket, prefix, continuationToken, delimiter, maxKeys)
+	marker := continuationToken
+	if marker == "" {
+		marker = startAfter
+	}
+
+	loi, err := fs.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
 		return result, err
 	}
@@ -483,7 +483,6 @@ func (a *azureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi min
 			} // else continue
 		}
 	}
-	logger.LogIf(ctx, minio.BucketNotFound{Bucket: bucket})
 	return bi, minio.BucketNotFound{Bucket: bucket}
 }

@@ -611,7 +610,7 @@ func (a *azureObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
 // ListObjectsV2 - list all blobs in Azure bucket filtered by prefix
 func (a *azureObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result minio.ListObjectsV2Info, err error) {
 	marker := continuationToken
-	if startAfter != "" {
+	if marker == "" {
 		marker = startAfter
 	}

@@ -1131,7 +1130,6 @@ func (a *azureObjects) GetBucketPolicy(ctx context.Context, bucket string) (*pol
 	}

 	if perm.AccessType == storage.ContainerAccessTypePrivate {
-		logger.LogIf(ctx, minio.BucketPolicyNotFound{Bucket: bucket})
 		return nil, minio.BucketPolicyNotFound{Bucket: bucket}
 	} else if perm.AccessType != storage.ContainerAccessTypeContainer {
 		logger.LogIf(ctx, minio.NotImplemented{})
@@ -278,7 +278,6 @@ func (l *b2Objects) Bucket(ctx context.Context, bucket string) (*b2.Bucket, erro
 			return bkt, nil
 		}
 	}
-	logger.LogIf(ctx, minio.BucketNotFound{Bucket: bucket})
 	return nil, minio.BucketNotFound{Bucket: bucket}
 }

@@ -355,12 +354,20 @@ func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix strin
 // ListObjectsV2 lists all objects in B2 bucket filtered by prefix, returns upto max 1000 entries at a time.
 func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int,
 	fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
-	// fetchOwner, startAfter are not supported and unused.
+	// fetchOwner is not supported and unused.
+	marker := continuationToken
+	if marker == "" {
+		// B2's continuation token is an object name to "start at" rather than "start after"
+		// startAfter plus the lowest character B2 supports is used so that the startAfter
+		// object isn't included in the results
+		marker = startAfter + " "
+	}
+
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
 		return loi, err
 	}
-	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, continuationToken, prefix, delimiter)
+	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
 	if lerr != nil {
 		logger.LogIf(ctx, lerr)
 		return loi, b2ToObjectError(lerr, bucket)
@@ -772,7 +779,6 @@ func (l *b2Objects) GetBucketPolicy(ctx context.Context, bucket string) (*policy
 	// just return back as policy not found for all cases.
 	// CreateBucket always sets the value to allPrivate by default.
 	if bkt.Type != bucketTypeReadOnly {
-		logger.LogIf(ctx, minio.BucketPolicyNotFound{Bucket: bucket})
 		return nil, minio.BucketPolicyNotFound{Bucket: bucket}
 	}

@@ -658,6 +658,10 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri

 // ListObjectsV2 - lists all blobs in GCS bucket filtered by prefix
 func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (minio.ListObjectsV2Info, error) {
+	if continuationToken == "" && startAfter != "" {
+		continuationToken = toGCSPageToken(startAfter)
+	}
+
 	it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
 		Delimiter: delimiter,
 		Prefix:    prefix,
@@ -1011,12 +1015,8 @@ func (l *gcsGateway) AbortMultipartUpload(ctx context.Context, bucket string, ke

 // CompleteMultipartUpload completes ongoing multipart upload and finalizes object
 // Note that there is a limit (currently 32) to the number of components that can
-// be composed in a single operation. There is a limit (currently 1024) to the total
-// number of components for a given composite object. This means you can append to
-// each object at most 1023 times. There is a per-project rate limit (currently 200)
-// to the number of components you can compose per second. This rate counts both the
-// components being appended to a composite object as well as the components being
-// copied when the composite object of which they are a part is copied.
+// be composed in a single operation. There is a per-project rate limit (currently 200)
+// to the number of source objects you can compose per second.
 func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string, key string, uploadID string, uploadedParts []minio.CompletePart) (minio.ObjectInfo, error) {
 	meta := gcsMultipartMetaName(uploadID)
 	object := l.client.Bucket(bucket).Object(meta)
@@ -1136,7 +1136,6 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
 func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
 	policyInfo, err := minio.PolicyToBucketAccessPolicy(bucketPolicy)
 	if err != nil {
-		// This should not happen.
 		logger.LogIf(ctx, err)
 		return gcsToObjectError(err, bucket)
 	}
@@ -1192,7 +1191,6 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketP
 func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
 	rules, err := l.client.Bucket(bucket).ACL().List(l.ctx)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		return nil, gcsToObjectError(err, bucket)
 	}

@@ -1227,7 +1225,6 @@ func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*polic

 	// Return NoSuchBucketPolicy error, when policy is not set
 	if len(actionSet) == 0 {
-		logger.LogIf(ctx, minio.BucketPolicyNotFound{})
 		return nil, gcsToObjectError(minio.BucketPolicyNotFound{}, bucket)
 	}

@@ -1252,7 +1249,6 @@ func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*polic
 func (l *gcsGateway) DeleteBucketPolicy(ctx context.Context, bucket string) error {
 	// This only removes the storage.AllUsers policies
 	if err := l.client.Bucket(bucket).ACL().Delete(l.ctx, storage.AllUsers); err != nil {
-		logger.LogIf(ctx, err)
 		return gcsToObjectError(err, bucket)
 	}

@@ -347,6 +347,13 @@ func (t *tritonObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
 		dirName = path.Join(mantaRoot, bucket, pathDir)
 	}

+	if marker != "" {
+		// Manta uses the marker as the key to start at rather than start after
+		// A space is appended to the marker so that the corresponding object is not
+		// included in the results
+		marker += " "
+	}
+
 	input = &storage.ListDirectoryInput{
 		DirectoryName: dirName,
 		Limit:         uint64(maxKeys),
@@ -419,6 +426,18 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti
 		pathBase = path.Base(prefix)
 	)

+	marker := continuationToken
+	if marker == "" {
+		marker = startAfter
+	}
+
+	if marker != "" {
+		// Manta uses the marker as the key to start at rather than start after.
+		// A space is appended to the marker so that the corresponding object is not
+		// included in the results
+		marker += " "
+	}
+
 	if pathDir := path.Dir(prefix); pathDir == "." {
 		dirName = path.Join(mantaRoot, bucket)
 	} else {
@@ -428,7 +447,7 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti
 	input = &storage.ListDirectoryInput{
 		DirectoryName: dirName,
 		Limit:         uint64(maxKeys),
-		Marker:        continuationToken,
+		Marker:        marker,
 	}
 	objs, err = t.client.Dir().List(ctx, input)
 	if err != nil {
@@ -479,8 +479,11 @@ func ossListObjects(ctx context.Context, client *oss.Client, bucket, prefix, mar
 // ossListObjectsV2 lists all blobs in OSS bucket filtered by prefix.
 func ossListObjectsV2(ctx context.Context, client *oss.Client, bucket, prefix, continuationToken, delimiter string, maxKeys int,
 	fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
-	// fetchOwner and startAfter are not supported and unused.
+	// fetchOwner is not supported and unused.
 	marker := continuationToken
+	if marker == "" {
+		marker = startAfter
+	}

 	resultV1, err := ossListObjects(ctx, client, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
@@ -1018,7 +1021,6 @@ func (l *ossObjects) GetBucketPolicy(ctx context.Context, bucket string) (*polic
 	switch result.ACL {
 	case string(oss.ACLPrivate):
 		// By default, all buckets starts with a "private" policy.
-		logger.LogIf(ctx, minio.BucketPolicyNotFound{})
 		return nil, ossToObjectError(minio.BucketPolicyNotFound{}, bucket)
 	case string(oss.ACLPublicRead):
 		readOnly = true
@@ -227,7 +227,6 @@ func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.
 		}, nil
 	}

-	logger.LogIf(ctx, minio.BucketNotFound{Bucket: bucket})
 	return bi, minio.BucketNotFound{Bucket: bucket}
 }

@@ -464,7 +463,6 @@ func (l *s3Objects) SetBucketPolicy(ctx context.Context, bucket string, bucketPo
 func (l *s3Objects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
 	data, err := l.Client.GetBucketPolicy(bucket)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		return nil, minio.ErrorRespToObjectError(err, bucket)
 	}

@@ -26,6 +26,7 @@ import (
 	"strings"

 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/handlers"
 	httptracer "github.com/minio/minio/pkg/handlers"
 )

@@ -167,7 +168,7 @@ func extractReqParams(r *http.Request) map[string]string {

 	// Success.
 	return map[string]string{
-		"sourceIPAddress": r.RemoteAddr,
+		"sourceIPAddress": handlers.GetSourceIP(r),
 		// Add more fields here.
 	}
 }
@@ -68,28 +68,21 @@ type NotificationPeerErr struct {
 }

 // DeleteBucket - calls DeleteBucket RPC call on all peers.
-func (sys *NotificationSys) DeleteBucket(bucketName string) <-chan NotificationPeerErr {
-	errCh := make(chan NotificationPeerErr)
+func (sys *NotificationSys) DeleteBucket(ctx context.Context, bucketName string) {
 	go func() {
-		defer close(errCh)
-
 		var wg sync.WaitGroup
 		for addr, client := range sys.peerRPCClientMap {
 			wg.Add(1)
 			go func(addr xnet.Host, client *PeerRPCClient) {
 				defer wg.Done()
 				if err := client.DeleteBucket(bucketName); err != nil {
-					errCh <- NotificationPeerErr{
-						Host: addr,
-						Err:  err,
-					}
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
+					logger.LogIf(ctx, err)
 				}
 			}(addr, client)
 		}
 		wg.Wait()
 	}()
-
-	return errCh
 }

 // SetCredentials - calls SetCredentials RPC call on all peers.
@@ -120,104 +113,76 @@ func (sys *NotificationSys) SetCredentials(credentials auth.Credentials) map[xne
 }

 // SetBucketPolicy - calls SetBucketPolicy RPC call on all peers.
-func (sys *NotificationSys) SetBucketPolicy(bucketName string, bucketPolicy *policy.Policy) <-chan NotificationPeerErr {
-	errCh := make(chan NotificationPeerErr)
+func (sys *NotificationSys) SetBucketPolicy(ctx context.Context, bucketName string, bucketPolicy *policy.Policy) {
 	go func() {
-		defer close(errCh)
-
 		var wg sync.WaitGroup
 		for addr, client := range sys.peerRPCClientMap {
 			wg.Add(1)
 			go func(addr xnet.Host, client *PeerRPCClient) {
 				defer wg.Done()
 				if err := client.SetBucketPolicy(bucketName, bucketPolicy); err != nil {
-					errCh <- NotificationPeerErr{
-						Host: addr,
-						Err:  err,
-					}
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
+					logger.LogIf(ctx, err)
 				}
 			}(addr, client)
 		}
 		wg.Wait()
 	}()
-
-	return errCh
 }

 // RemoveBucketPolicy - calls RemoveBucketPolicy RPC call on all peers.
-func (sys *NotificationSys) RemoveBucketPolicy(bucketName string) <-chan NotificationPeerErr {
-	errCh := make(chan NotificationPeerErr)
+func (sys *NotificationSys) RemoveBucketPolicy(ctx context.Context, bucketName string) {
 	go func() {
-		defer close(errCh)
-
 		var wg sync.WaitGroup
 		for addr, client := range sys.peerRPCClientMap {
 			wg.Add(1)
 			go func(addr xnet.Host, client *PeerRPCClient) {
 				defer wg.Done()
 				if err := client.RemoveBucketPolicy(bucketName); err != nil {
-					errCh <- NotificationPeerErr{
-						Host: addr,
-						Err:  err,
-					}
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
+					logger.LogIf(ctx, err)
 				}
 			}(addr, client)
 		}
 		wg.Wait()
 	}()
-
-	return errCh
 }

 // PutBucketNotification - calls PutBucketNotification RPC call on all peers.
-func (sys *NotificationSys) PutBucketNotification(bucketName string, rulesMap event.RulesMap) <-chan NotificationPeerErr {
-	errCh := make(chan NotificationPeerErr)
+func (sys *NotificationSys) PutBucketNotification(ctx context.Context, bucketName string, rulesMap event.RulesMap) {
 	go func() {
-		defer close(errCh)
-
 		var wg sync.WaitGroup
 		for addr, client := range sys.peerRPCClientMap {
 			wg.Add(1)
 			go func(addr xnet.Host, client *PeerRPCClient, rulesMap event.RulesMap) {
 				defer wg.Done()
 				if err := client.PutBucketNotification(bucketName, rulesMap); err != nil {
-					errCh <- NotificationPeerErr{
-						Host: addr,
-						Err:  err,
-					}
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
+					logger.LogIf(ctx, err)
 				}
 			}(addr, client, rulesMap.Clone())
 		}
 		wg.Wait()
 	}()
-
-	return errCh
 }

 // ListenBucketNotification - calls ListenBucketNotification RPC call on all peers.
-func (sys *NotificationSys) ListenBucketNotification(bucketName string, eventNames []event.Name, pattern string,
-	targetID event.TargetID, localPeer xnet.Host) <-chan NotificationPeerErr {
-	errCh := make(chan NotificationPeerErr)
+func (sys *NotificationSys) ListenBucketNotification(ctx context.Context, bucketName string, eventNames []event.Name, pattern string,
+	targetID event.TargetID, localPeer xnet.Host) {
 	go func() {
-		defer close(errCh)
-
 		var wg sync.WaitGroup
 		for addr, client := range sys.peerRPCClientMap {
 			wg.Add(1)
 			go func(addr xnet.Host, client *PeerRPCClient) {
 				defer wg.Done()
 				if err := client.ListenBucketNotification(bucketName, eventNames, pattern, targetID, localPeer); err != nil {
-					errCh <- NotificationPeerErr{
-						Host: addr,
-						Err:  err,
-					}
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
+					logger.LogIf(ctx, err)
 				}
 			}(addr, client)
 		}
 		wg.Wait()
 	}()
-
-	return errCh
 }

 // AddRemoteTarget - adds event rules map, HTTP/PeerRPC client target to bucket name.
@@ -556,13 +521,16 @@ func sendEvent(args eventArgs) {
 		return
 	}

-	for _, err := range globalNotificationSys.Send(args) {
-		reqInfo := &logger.ReqInfo{BucketName: args.BucketName, ObjectName: args.Object.Name}
-		reqInfo.AppendTags("EventName", args.EventName.String())
-		reqInfo.AppendTags("targetID", err.ID.Name)
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogOnceIf(ctx, err.Err, err.ID)
-	}
+	notifyCh := globalNotificationSys.Send(args)
+	go func() {
+		for _, err := range notifyCh {
+			reqInfo := &logger.ReqInfo{BucketName: args.BucketName, ObjectName: args.Object.Name}
+			reqInfo.AppendTags("EventName", args.EventName.String())
+			reqInfo.AppendTags("targetID", err.ID.Name)
+			ctx := logger.SetReqInfo(context.Background(), reqInfo)
+			logger.LogOnceIf(ctx, err.Err, err.ID)
+		}
+	}()
 }

 func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error {
@@ -24,6 +24,7 @@ import (
 	"time"

 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/handlers"
 )

 // Validates the preconditions for CopyObjectPart, returns true if CopyObjectPart
@@ -243,7 +244,7 @@ func deleteObject(ctx context.Context, obj ObjectLayer, cache CacheObjectLayer,
 	}

 	// Get host and port from Request.RemoteAddr.
-	host, port, _ := net.SplitHostPort(r.RemoteAddr)
+	host, port, _ := net.SplitHostPort(handlers.GetSourceIP(r))

 	// Notify object deleted event.
 	sendEvent(eventArgs{
@@ -31,11 +31,13 @@ import (
 	"sort"
 	"strconv"

+	etcd "github.com/coreos/etcd/client"
 	"github.com/gorilla/mux"
 	miniogo "github.com/minio/minio-go"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/ioutil"
 	"github.com/minio/minio/pkg/policy"
@@ -159,7 +161,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 			// additionally also skipping mod(offset)64KiB boundaries.
 			writer = ioutil.LimitedWriter(writer, startOffset%(64*1024), length)

-			writer, startOffset, length, err = DecryptBlocksRequest(writer, r, startOffset, length, objInfo, false)
+			writer, startOffset, length, err = DecryptBlocksRequest(writer, r, bucket, object, startOffset, length, objInfo, false)
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -196,7 +198,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 	}

 	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -268,7 +270,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 			writeErrorResponse(w, apiErr, r.URL)
 			return
 		} else if encrypted {
-			if _, err = DecryptRequest(w, r, objInfo.UserDefined); err != nil {
+			if _, err = DecryptRequest(w, r, bucket, object, objInfo.UserDefined); err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
 			}
@@ -292,7 +294,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 	w.WriteHeader(http.StatusOK)

 	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -461,7 +463,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			for k, v := range srcInfo.UserDefined {
 				encMetadata[k] = v
 			}
-			if err = rotateKey(oldKey, newKey, encMetadata); err != nil {
+			if err = rotateKey(oldKey, newKey, srcBucket, srcObject, encMetadata); err != nil {
 				pipeWriter.CloseWithError(err)
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -473,7 +475,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			if sseCopyC {
 				// Source is encrypted make sure to save the encrypted size.
 				writer = ioutil.LimitedWriter(writer, 0, srcInfo.Size)
-				writer, srcInfo.Size, err = DecryptAllBlocksCopyRequest(writer, r, srcInfo)
+				writer, srcInfo.Size, err = DecryptAllBlocksCopyRequest(writer, r, srcBucket, srcObject, srcInfo)
 				if err != nil {
 					pipeWriter.CloseWithError(err)
 					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -488,7 +490,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 				}
 			}
 			if sseC {
-				reader, err = newEncryptReader(pipeReader, newKey, encMetadata)
+				reader, err = newEncryptReader(reader, newKey, dstBucket, dstObject, encMetadata)
 				if err != nil {
 					pipeWriter.CloseWithError(err)
 					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -616,7 +618,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	writeSuccessResponseXML(w, encodedSuccessResponse)

 	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -787,7 +789,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req

 	if objectAPI.IsEncryptionSupported() {
 		if hasSSECustomerHeader(r.Header) && !hasSuffix(object, slashSeparator) { // handle SSE-C requests
-			reader, err = EncryptRequest(hashReader, r, metadata)
+			reader, err = EncryptRequest(hashReader, r, bucket, object, metadata)
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -822,7 +824,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseHeadersOnly(w)

 	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -886,7 +888,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
 			}
-			_, err = newEncryptMetadata(key, encMetadata)
+			_, err = newEncryptMetadata(key, bucket, object, encMetadata)
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -1054,7 +1056,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			// Response writer should be limited early on for decryption upto required length,
 			// additionally also skipping mod(offset)64KiB boundaries.
 			writer = ioutil.LimitedWriter(writer, startOffset%(64*1024), length)
-			writer, startOffset, length, err = DecryptBlocksRequest(pipeWriter, r, startOffset, length, srcInfo, true)
+			writer, startOffset, length, err = DecryptBlocksRequest(writer, r, srcBucket, srcObject, startOffset, length, srcInfo, true)
 			if err != nil {
 				pipeWriter.CloseWithError(err)
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -1076,14 +1078,14 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt

 			// Calculating object encryption key
 			var objectEncryptionKey []byte
-			objectEncryptionKey, err = decryptObjectInfo(key, li.UserDefined)
+			objectEncryptionKey, err = decryptObjectInfo(key, dstBucket, dstObject, li.UserDefined)
 			if err != nil {
 				pipeWriter.CloseWithError(err)
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
 			}

-			reader, err = sio.EncryptReader(pipeReader, sio.Config{Key: objectEncryptionKey})
+			reader, err = sio.EncryptReader(reader, sio.Config{Key: objectEncryptionKey})
 			if err != nil {
 				pipeWriter.CloseWithError(err)
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -1279,7 +1281,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http

 			// Calculating object encryption key
 			var objectEncryptionKey []byte
-			objectEncryptionKey, err = decryptObjectInfo(key, li.UserDefined)
+			objectEncryptionKey, err = decryptObjectInfo(key, bucket, object, li.UserDefined)
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -1493,7 +1495,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	writeSuccessResponseXML(w, encodedSuccessResponse)

 	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -1539,6 +1541,27 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		return
 	}

+	if globalDNSConfig != nil {
+		_, err := globalDNSConfig.Get(bucket)
+		if err != nil {
+			if etcd.IsKeyNotFound(err) || err == dns.ErrNoEntriesFound {
+				writeErrorResponse(w, ErrNoSuchBucket, r.URL)
+			} else {
+				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			}
+			return
+		}
+	} else {
+		getBucketInfo := objectAPI.GetBucketInfo
+		if api.CacheAPI() != nil {
+			getBucketInfo = api.CacheAPI().GetBucketInfo
+		}
+		if _, err := getBucketInfo(ctx, bucket); err != nil {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
+	}
+
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
 	// Ignore delete object errors while replying to client, since we are
 	// suppposed to reply only 204. Additionally log the error for
@@ -38,8 +38,8 @@ import (
 )

 const (
-	diskMinFreeSpace  = 1 * humanize.GiByte // Min 1GiB free space.
-	diskMinTotalSpace = diskMinFreeSpace    // Min 1GiB total space.
+	diskMinFreeSpace  = 900 * humanize.MiByte // Min 900MiB free space.
+	diskMinTotalSpace = diskMinFreeSpace      // Min 900MiB total space.
 	maxAllowedIOError = 5
 )

@@ -34,6 +34,7 @@ import (
 	"time"

 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/handlers"

 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
@@ -335,7 +336,13 @@ func newContext(r *http.Request, api string) context.Context {
 	if prefix != "" {
 		object = prefix
 	}
-	reqInfo := &logger.ReqInfo{RemoteHost: r.RemoteAddr, UserAgent: r.Header.Get("user-agent"), API: api, BucketName: bucket, ObjectName: object}
+	reqInfo := &logger.ReqInfo{
+		RemoteHost: handlers.GetSourceIP(r),
+		UserAgent:  r.Header.Get("user-agent"),
+		API:        api,
+		BucketName: bucket,
+		ObjectName: object,
+	}
 	return logger.SetReqInfo(context.Background(), reqInfo)
 }

@@ -193,10 +193,7 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs,

 	globalNotificationSys.RemoveNotification(args.BucketName)
 	globalPolicySys.Remove(args.BucketName)
-	for nerr := range globalNotificationSys.DeleteBucket(args.BucketName) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.DeleteBucket(ctx, args.BucketName)

 	if globalDNSConfig != nil {
 		if err := globalDNSConfig.Delete(args.BucketName); err != nil {
@@ -979,10 +976,7 @@ func (web *webAPIHandlers) SetBucketPolicy(r *http.Request, args *SetBucketPolic
 	}

 	globalPolicySys.Set(args.BucketName, *bucketPolicy)
-	for nerr := range globalNotificationSys.SetBucketPolicy(args.BucketName, bucketPolicy) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.SetBucketPolicy(ctx, args.BucketName, bucketPolicy)

 	return nil
 }
@@ -454,7 +454,12 @@ func (s *xlSets) GetBucketInfo(ctx context.Context, bucket string) (bucketInfo B

 // ListObjectsV2 lists all objects in bucket filtered by prefix
 func (s *xlSets) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
-	loi, err := s.ListObjects(ctx, bucket, prefix, continuationToken, delimiter, maxKeys)
+	marker := continuationToken
+	if marker == "" {
+		marker = startAfter
+	}
+
+	loi, err := s.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
 		return result, err
 	}
@@ -855,7 +855,12 @@ func (xl xlObjects) DeleteObject(ctx context.Context, bucket, object string) (er

 // ListObjectsV2 lists all blobs in bucket filtered by prefix
 func (xl xlObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
-	loi, err := xl.ListObjects(ctx, bucket, prefix, continuationToken, delimiter, maxKeys)
+	marker := continuationToken
+	if marker == "" {
+		marker = startAfter
+	}
+
+	loi, err := xl.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
 		return result, err
 	}
@@ -110,6 +110,7 @@ By default, parity for objects with standard storage class is set to `N/2`, and
 |``drives``| _[]string_ | List of mounted file system drives with [`atime`](http://kerolasa.github.io/filetimes.html) support enabled|
 |``exclude`` | _[]string_ | List of wildcard patterns for prefixes to exclude from cache |
 |``expiry`` | _int_ | Days to cache expiry |
+|``maxuse`` | _int_ | Percentage of disk available to cache |

 #### Notify
 |Field|Type|Description|
@@ -1,11 +1,12 @@
 {
-    "version": "24",
+    "version": "26",
    "credential": {
        "accessKey": "USWUXHGYZQYFYFFIT3RE",
        "secretKey": "MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03"
    },
    "region": "us-east-1",
    "browser": "on",
+    "worm": "off",
    "domain": "",
    "storageclass": {
        "standard": "",
@@ -14,11 +15,12 @@
    "cache": {
        "drives": [],
        "expiry": 90,
-        "exclude": []
+        "exclude": [],
+        "maxuse": 80
    },
    "usage": {
        "interval": "3h"
-    }
+    },
    "notify": {
        "amqp": {
            "1": {
@@ -124,3 +126,4 @@
        }
    }
 }
+
@@ -1,6 +1,6 @@
 #!/bin/bash

-## Minio Cloud Storage, (C) 2017 Minio, Inc.
+## Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
 ##
 ## Licensed under the Apache License, Version 2.0 (the "License");
 ## you may not use this file except in compliance with the License.
@@ -16,9 +16,9 @@

 # This script changes protected files, and must be run as root

-for i in $(ls -d /sys/block/*/queue/iosched 2>/dev/null); do
-    iosched_dir=$(echo $i | awk '/iosched/ {print $1}')
-    [ -z $iosched_dir ] && {
+for i in $(echo /sys/block/*/queue/iosched 2>/dev/null); do
+    iosched_dir=$(echo "${i}" | awk '/iosched/ {print $1}')
+    [ -z "${iosched_dir}" ] && {
        continue
    }
    ## Change each disk ioscheduler to be "deadline"
@@ -29,22 +29,22 @@ for i in $(ls -d /sys/block/*/queue/iosched 2>/dev/null); do
    ## see whether write requests have been starved for too
    ## long, and then decides whether to start a new batch
    ## of reads or writes
-    path=$(dirname $iosched_dir)
-    [ -f $path/scheduler ] && {
-        echo "deadline" > $path/scheduler
+    path=$(dirname "${iosched_dir}")
+    [ -f "${path}/scheduler" ] && {
+        echo "deadline" > "${path}/scheduler" 2>/dev/null || true
    }
    ## This controls how many requests may be allocated
    ## in the block layer for read or write requests.
    ## Note that the total allocated number may be twice
    ## this amount, since it applies only to reads or
    ## writes (not the accumulate sum).
-    [ -f $path/nr_requests ] && {
-        echo "256" > $path/nr_requests
+    [ -f "${path}/nr_requests" ] && {
+        echo "256" > "${path}/nr_requests" 2>/dev/null || true
    }
    ## This is the maximum number of kilobytes
    ## supported in a single data transfer at
    ## block layer.
-    [ -f $path/max_sectors_kb ] && {
-        echo "1024" > $path/max_sectors_kb || true
+    [ -f "${path}/max_sectors_kb" ] && {
+        echo "1024" > "${path}/max_sectors_kb" 2>/dev/null || true
    }
 done
@@ -16,10 +16,10 @@ minio server -h
 ...
 ...

-  7. Start minio server with edge caching enabled on '/mnt/drive1', '/mnt/drive2' and '/mnt/drive3',
+  7. Start minio server with edge caching enabled on '/mnt/drive1', '/mnt/drive2' and '/mnt/export1 ... /mnt/export24',
     exclude all objects under 'mybucket', exclude all objects with '.pdf' as extension
     with expiry upto 40 days.
-     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3"
+     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/export{1..24}"
     $ export MINIO_CACHE_EXCLUDE="mybucket/*;*.pdf"
     $ export MINIO_CACHE_EXPIRY=40
     $ export MINIO_CACHE_MAXUSE=80
@@ -17,14 +17,15 @@ Disk caching can be enabled by updating the `cache` config settings for Minio se
 "cache": {
 	"drives": ["/mnt/drive1", "/mnt/drive2", "/mnt/drive3"],
 	"expiry": 90,
-	"exclude": ["*.pdf","mybucket/*"]
+	"exclude": ["*.pdf","mybucket/*"],
+	"maxuse" : 70,
 },
 ```

-The cache settings may also be set through environment variables. When set, environment variables override any `cache` config settings for Minio server. Following example uses `/mnt/drive1`, `/mnt/drive2` and `/mnt/drive3` for caching, with expiry upto 90 days while excluding all objects under bucket `mybucket` and all objects with '.pdf' as extension while starting a standalone erasure coded setup.
+The cache settings may also be set through environment variables. When set, environment variables override any `cache` config settings for Minio server. Following example uses `/mnt/drive1`, `/mnt/drive2` ,`/mnt/cache1` ... `/mnt/cache3` for caching, with expiry upto 90 days while excluding all objects under bucket `mybucket` and all objects with '.pdf' as extension while starting a standalone erasure coded setup. Cache max usage is restricted to 80% of disk capacity in this example.

 ```bash
-export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3"
+export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/cache{1...3}"
 export MINIO_CACHE_EXPIRY=90
 export MINIO_CACHE_EXCLUDE="*.pdf;mybucket/*"
 export MINIO_CACHE_MAXUSE=80
@@ -55,7 +55,6 @@ mc ls mygcs
 ### Known limitations
 Gateway inherits the following GCS limitations:

- Maximum number of multipart parts per upload is 1024.
 - Only read-only or write-only bucket policy supported at bucket level, all other variations will return API Notimplemented error.
 - _List Multipart Uploads_ and _List Object parts_ always returns empty list. i.e Client will need to remember all the parts that it has uploaded and use it for _Complete Multipart Upload_

@@ -5,7 +5,7 @@ version: '2'
 # 9001 through 9004.
 services:
 minio1:
-  image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+  image: minio/minio:RELEASE.2018-06-29T02-11-29Z
  volumes:
   - data1:/data
  ports:
@@ -15,7 +15,7 @@ services:
   MINIO_SECRET_KEY: minio123
  command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
 minio2:
-  image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+  image: minio/minio:RELEASE.2018-06-29T02-11-29Z
  volumes:
   - data2:/data
  ports:
@@ -25,7 +25,7 @@ services:
   MINIO_SECRET_KEY: minio123
  command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
 minio3:
-  image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+  image: minio/minio:RELEASE.2018-06-29T02-11-29Z
  volumes:
   - data3:/data
  ports:
@@ -35,7 +35,7 @@ services:
   MINIO_SECRET_KEY: minio123
  command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
 minio4:
-  image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+  image: minio/minio:RELEASE.2018-06-29T02-11-29Z
  volumes:
   - data4:/data
  ports:
@@ -2,7 +2,7 @@ version: '3.1'

 services:
  minio1:
-    image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+    image: minio/minio:RELEASE.2018-06-29T02-11-29Z
    volumes:
      - minio1-data:/export
    ports:
@@ -20,7 +20,7 @@ services:
      - access_key

  minio2:
-    image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+    image: minio/minio:RELEASE.2018-06-29T02-11-29Z
    volumes:
      - minio2-data:/export
    ports:
@@ -38,7 +38,7 @@ services:
      - access_key

  minio3:
-    image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+    image: minio/minio:RELEASE.2018-06-29T02-11-29Z
    volumes:
      - minio3-data:/export
    ports:
@@ -56,7 +56,7 @@ services:
      - access_key

  minio4:
-    image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+    image: minio/minio:RELEASE.2018-06-29T02-11-29Z
    volumes:
      - minio4-data:/export
    ports:
@@ -2,7 +2,7 @@ version: '3'

 services:
  minio1:
-    image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+    image: minio/minio:RELEASE.2018-06-29T02-11-29Z
    volumes:
      - minio1-data:/export
    ports:
@@ -20,7 +20,7 @@ services:
    command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export

  minio2:
-    image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+    image: minio/minio:RELEASE.2018-06-29T02-11-29Z
    volumes:
      - minio2-data:/export
    ports:
@@ -38,7 +38,7 @@ services:
    command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export

  minio3:
-    image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+    image: minio/minio:RELEASE.2018-06-29T02-11-29Z
    volumes:
      - minio3-data:/export
    ports:
@@ -56,7 +56,7 @@ services:
    command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export

  minio4:
-    image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+    image: minio/minio:RELEASE.2018-06-29T02-11-29Z
    volumes:
      - minio4-data:/export
    ports:
@@ -124,7 +124,7 @@ spec:
        - name: data 
          mountPath: "/data"
        # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+        image: minio/minio:RELEASE.2018-06-29T02-11-29Z
        args:
        - server
        - /data
@@ -303,7 +303,7 @@ spec:
          value: "minio"
        - name: MINIO_SECRET_KEY
          value: "minio123"
-        image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+        image: minio/minio:RELEASE.2018-06-29T02-11-29Z
        args:
        - server
        - http://minio-0.minio.default.svc.cluster.local/data
@@ -514,7 +514,7 @@ spec:
      containers:
      - name: minio
        # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+        image: minio/minio:RELEASE.2018-06-29T02-11-29Z
        args:
        - gateway
        - gcs
@@ -21,7 +21,7 @@ spec:
          value: "minio"
        - name: MINIO_SECRET_KEY
          value: "minio123"
-        image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+        image: minio/minio:RELEASE.2018-06-29T02-11-29Z
        args:
        - server
        - http://minio-0.minio.default.svc.cluster.local/data
@@ -21,7 +21,7 @@ spec:
      containers:
      - name: minio
        # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+        image: minio/minio:RELEASE.2018-06-29T02-11-29Z
        args:
        - gateway
        - gcs
@@ -29,7 +29,7 @@ spec:
        - name: data 
          mountPath: "/data"
        # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+        image: minio/minio:RELEASE.2018-06-29T02-11-29Z
        args:
        - server
        - /data
@@ -30,7 +30,7 @@ Above command deploys Minio on the Kubernetes cluster in the default configurati
 | Parameter                  | Description                         | Default                                                 |
 |----------------------------|-------------------------------------|---------------------------------------------------------|
 | `image`                    | Minio image name                    | `minio/minio`                                           |
-| `imageTag`                 | Minio image tag. Possible values listed [here](https://hub.docker.com/r/minio/minio/tags/).| `RELEASE.2018-06-22T23-48-46Z`|
+| `imageTag`                 | Minio image tag. Possible values listed [here](https://hub.docker.com/r/minio/minio/tags/).| `RELEASE.2018-06-29T02-11-29Z`|
 | `imagePullPolicy`          | Image pull policy                   | `Always`                                                |
 | `mode`                     | Minio server mode (`standalone`, `shared` or `distributed`)| `standalone`                     |
 | `numberOfNodes`            | Number of nodes (applicable only for Minio distributed mode). Should be 4 <= x <= 16 | `4`    |
@@ -54,7 +54,7 @@ spec:
          value: "minio"
        - name: MINIO_SECRET_KEY
          value: "minio123"
-        image: minio/minio:RELEASE.2018-06-22T23-48-46Z
+        image: minio/minio:RELEASE.2018-06-29T02-11-29Z
        imagePullPolicy: IfNotPresent
        args: ["server", "http://minio-0.minio.default.svc.cluster.local/data", "http://minio-1.minio.default.svc.cluster.local/data", "http://minio-2.minio.default.svc.cluster.local/data", "http://minio-3.minio.default.svc.cluster.local/data"]
        ports:
@@ -18,6 +18,7 @@ package certs

 import (
 	"crypto/tls"
+	"path/filepath"
 	"sync"

 	"github.com/rjeczalik/notify"
@@ -74,11 +75,14 @@ func (c *Certs) watch() (err error) {
 		}
 	}()

-	if err = notify.Watch(c.certFile, c.e, eventWrite...); err != nil {
+	// Windows doesn't allow for watching file changes but instead allows
+	// for directory changes only, while we can still watch for changes
+	// on files on other platforms. Watch parent directory on all platforms
+	// for simplicity.
+	if err = notify.Watch(filepath.Dir(c.certFile), c.e, eventWrite...); err != nil {
 		return err
 	}
-
-	if err = notify.Watch(c.keyFile, c.e, eventWrite...); err != nil {
+	if err = notify.Watch(filepath.Dir(c.keyFile), c.e, eventWrite...); err != nil {
 		return err
 	}
 	c.Lock()
@@ -93,16 +97,21 @@ func (c *Certs) watch() (err error) {

 func (c *Certs) run() {
 	for event := range c.e {
+		base := filepath.Base(event.Path())
 		if isWriteEvent(event.Event()) {
-			cert, err := c.loadCert(c.certFile, c.keyFile)
-			if err != nil {
-				// ignore the error continue to use
-				// old certificates.
-				continue
+			certChanged := base == filepath.Base(c.certFile)
+			keyChanged := base == filepath.Base(c.keyFile)
+			if certChanged || keyChanged {
+				cert, err := c.loadCert(c.certFile, c.keyFile)
+				if err != nil {
+					// ignore the error continue to use
+					// old certificates.
+					continue
+				}
+				c.Lock()
+				c.cert = cert
+				c.Unlock()
 			}
-			c.Lock()
-			c.cert = cert
-			c.Unlock()
 		}
 	}
 }
@@ -19,10 +19,13 @@ package madmin

 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
+
+	"github.com/minio/minio/pkg/quick"
 )

 // NodeSummary - represents the result of an operation part of
@@ -65,15 +68,41 @@ func (adm *AdminClient) GetConfig() ([]byte, error) {

 // SetConfig - set config supplied as config.json for the setup.
 func (adm *AdminClient) SetConfig(config io.Reader) (r SetConfigResult, err error) {
+	const maxConfigJSONSize = 256 * 1024 // 256KiB
+
 	if !adm.secure { // No TLS?
 		return r, fmt.Errorf("credentials/configuration cannot be updated over an insecure connection")
 	}

-	// Read config bytes to calculate MD5, SHA256 and content length.
-	configBytes, err := ioutil.ReadAll(config)
-	if err != nil {
+	// Read configuration bytes
+	configBuf := make([]byte, maxConfigJSONSize+1)
+	n, err := io.ReadFull(config, configBuf)
+	if err == nil {
+		return r, fmt.Errorf("too large file")
+	}
+	if err != io.ErrUnexpectedEOF {
 		return r, err
 	}
+	configBytes := configBuf[:n]
+
+	type configVersion struct {
+		Version string `json:"version,omitempty"`
+	}
+	var cfg configVersion
+
+	// Check if read data is in json format
+	if err = json.Unmarshal(configBytes, &cfg); err != nil {
+		return r, errors.New("Invalid JSON format: " + err.Error())
+	}
+
+	// Check if the provided json file has "version" key set
+	if cfg.Version == "" {
+		return r, errors.New("Missing or unset \"version\" key in json file")
+	}
+	// Validate there are no duplicate keys in the JSON
+	if err = quick.CheckDuplicateKeys(string(configBytes)); err != nil {
+		return r, errors.New("Duplicate key in json file: " + err.Error())
+	}

 	reqData := requestData{
 		relPath: "/v1/config",
@@ -162,6 +162,16 @@ func (functions *Functions) UnmarshalJSON(data []byte) error {
 	return nil
 }

+// GobEncode - encodes Functions to gob data.
+func (functions Functions) GobEncode() ([]byte, error) {
+	return functions.MarshalJSON()
+}
+
+// GobDecode - decodes gob data to Functions.
+func (functions *Functions) GobDecode(data []byte) error {
+	return functions.UnmarshalJSON(data)
+}
+
 // NewFunctions - returns new Functions with given function list.
 func NewFunctions(functions ...Function) Functions {
 	return Functions(functions)
@@ -3,15 +3,19 @@
    src="logo.png" 
    width="240" height="78" border="0" alt="GJSON">
 <br>
-<a href="https://travis-ci.org/tidwall/gjson"><img src="https://img.shields.io/travis/tidwall/gjson.svg?style=flat-square" alt="Build Status"></a><!--
-<a href="http://gocover.io/github.com/tidwall/gjson"><img src="https://img.shields.io/badge/coverage-97%25-brightgreen.svg?style=flat-square" alt="Code Coverage"></a>
-->
+<a href="https://travis-ci.org/tidwall/gjson"><img src="https://img.shields.io/travis/tidwall/gjson.svg?style=flat-square" alt="Build Status"></a>
 <a href="https://godoc.org/github.com/tidwall/gjson"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
+<a href="http://tidwall.com/gjson-play"><img src="https://img.shields.io/badge/%F0%9F%8F%90-playground-9900cc.svg?style=flat-square" alt="GJSON Playground"></a>
 </p>

-<p align="center">get a json value quickly</a></p>

-GJSON is a Go package that provides a [very fast](#performance) and simple way to get a value from a json document. The purpose for this library it to give efficient json indexing for the [BuntDB](https://github.com/tidwall/buntdb) project. 
+
+<p align="center">get json values quickly</a></p>
+
+GJSON is a Go package that provides a [fast](#performance) and [simple](#get-a-value) way to get values from a json document.
+It has features such as [one line retrieval](#get-a-value), [dot notation paths](#path-syntax), [iteration](#iterate-through-an-object-or-array), and [parsing json lines](#json-lines).
+
+Also check out [SJSON](https://github.com/tidwall/sjson) for modifying json, and the [JJ](https://github.com/tidwall/jj) command line tool.

 Getting Started
 ===============
@@ -27,7 +31,7 @@ $ go get -u github.com/tidwall/gjson
 This will retrieve the library.

 ## Get a value
-Get searches json for the specified path. A path is in dot syntax, such as "name.last" or "age". This function expects that the json is well-formed and validates. Invalid json will not panic, but it may return back unexpected results. When the value is found it's returned immediately. 
+Get searches json for the specified path. A path is in dot syntax, such as "name.last" or "age". When the value is found it's returned immediately. 

 ```go
 package main
@@ -55,7 +59,7 @@ A path is a series of keys separated by a dot.
 A key may contain special wildcard characters '\*' and '?'.
 To access an array value use the index as the key.
 To get the number of elements in an array or to access a child path, use the '#' character.
-The dot and wildcard characters can be escaped with '\'.
+The dot and wildcard characters can be escaped with '\\'.

 ```json
 {
@@ -93,6 +97,36 @@ friends.#[age>45]#.last            >> ["Craig","Murphy"]
 friends.#[first%"D*"].last         >> "Murphy"
 ```

+## JSON Lines
+
+There's support for [JSON Lines](http://jsonlines.org/) using the `..` prefix, which treats a multilined document as an array. 
+
+For example:
+
+```
+{"name": "Gilbert", "age": 61}
+{"name": "Alexa", "age": 34}
+{"name": "May", "age": 57}
+{"name": "Deloise", "age": 44}
+```
+
+```
+..#                   >> 4
+..1                   >> {"name": "Alexa", "age": 34}
+..3                   >> {"name": "Deloise", "age": 44}
+..#.name              >> ["Gilbert","Alexa","May","Deloise"]
+..#[name="May"].age   >> 57
+```
+
+The `ForEachLines` function will iterate through JSON lines.
+
+```go
+gjson.ForEachLine(json, func(line gjson.Result) bool{
+    println(line.String())
+    return true
+})
+```
+
 ## Result Type

 GJSON supports the json types `string`, `number`, `bool`, and `null`. 
@@ -120,12 +154,14 @@ result.Index   // index of raw value in original json, zero means index unknown
 There are a variety of handy functions that work on a result:

 ```go
+result.Exists() bool
 result.Value() interface{}
 result.Int() int64
 result.Uint() uint64
 result.Float() float64
 result.String() string
 result.Bool() bool
+result.Time() time.Time
 result.Array() []gjson.Result
 result.Map() map[string]gjson.Result
 result.Get(path string) Result
@@ -135,8 +171,6 @@ result.Less(token Result, caseSensitive bool) bool

 The `result.Value()` function returns an `interface{}` which requires type assertion and is one of the following Go types:

-
-
 The `result.Array()` function returns back an array of values.
 If the result represents a non-existent value, then an empty array will be returned.
 If the result is not a JSON array, the return value will be an array containing one result.
@@ -150,6 +184,15 @@ array   >> []interface{}
 object  >> map[string]interface{}
 ```

+### 64-bit integers
+
+The `result.Int()` and `result.Uint()` calls are capable of reading all 64 bits, allowing for large JSON integers.
+
+```go
+result.Int() int64    // -9223372036854775808 to 9223372036854775807
+result.Uint() int64   // 0 to 18446744073709551615
+```
+
 ## Get nested array values

 Suppose you want all the last names from the following json:
@@ -168,14 +211,14 @@ Suppose you want all the last names from the following json:
      "lastName": "Harold", 
    }
  ]
-}`
+}
 ```

 You would use the path "programmers.#.lastName" like such:

 ```go
 result := gjson.Get(json, "programmers.#.lastName")
-for _,name := range result.Array() {
+for _, name := range result.Array() {
 	println(name.String())
 }
 ```
@@ -196,7 +239,7 @@ Returning `false` from an iterator will stop iteration.

 ```go
 result := gjson.Get(json, "programmers")
-result.ForEach(func(key, value gjson.Result) bool{
+result.ForEach(func(key, value gjson.Result) bool {
 	println(value.String()) 
 	return true // keep iterating
 })
@@ -227,18 +270,31 @@ if !value.Exists() {
 }

 // Or as one step
-if gjson.Get(json, "name.last").Exists(){
+if gjson.Get(json, "name.last").Exists() {
 	println("has a last name")
 }
 ```

+## Validate JSON
+
+The `Get*` and `Parse*` functions expects that the json is well-formed. Bad json will not panic, but it may return back unexpected results.
+
+If you are consuming JSON from an unpredictable source then you may want to validate prior to using GJSON.
+
+```go
+if !gjson.Valid(json) {
+	return errors.New("invalid json")
+}
+value := gjson.Get(json, "name.last")
+```
+
 ## Unmarshal to a map

 To unmarshal to a `map[string]interface{}`:

 ```go
 m, ok := gjson.Parse(json).Value().(map[string]interface{})
-if !ok{
+if !ok {
 	// not a map
 }
 ```
@@ -269,7 +325,7 @@ This is a best-effort no allocation sub slice of the original json. This method

 ## Get multiple values at once

-The `GetMany` function can be used to get multiple values at the same time, and is optimized to scan over a JSON payload once.
+The `GetMany` function can be used to get multiple values at the same time.

 ```go
 results := gjson.GetMany(json, "name.first", "name.last", "age")
@@ -282,28 +338,18 @@ The return value is a `[]Result`, which will always contain exactly the same num
 Benchmarks of GJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/), 
 [ffjson](https://github.com/pquerna/ffjson), 
 [EasyJSON](https://github.com/mailru/easyjson),
-and [jsonparser](https://github.com/buger/jsonparser)
+[jsonparser](https://github.com/buger/jsonparser),
+and [json-iterator](https://github.com/json-iterator/go)

 ```
-BenchmarkGJSONGet-8              	15000000	       333 ns/op	       0 B/op	       0 allocs/op
-BenchmarkGJSONUnmarshalMap-8     	  900000	      4188 ns/op	    1920 B/op	      26 allocs/op
-BenchmarkJSONUnmarshalMap-8      	  600000	      8908 ns/op	    3048 B/op	      69 allocs/op
-BenchmarkJSONUnmarshalStruct-8   	  600000	      9026 ns/op	    1832 B/op	      69 allocs/op
-BenchmarkJSONDecoder-8           	  300000	     14339 ns/op	    4224 B/op	     184 allocs/op
-BenchmarkFFJSONLexer-8           	 1500000	      3156 ns/op	     896 B/op	       8 allocs/op
-BenchmarkEasyJSONLexer-8         	 3000000	       938 ns/op	     613 B/op	       6 allocs/op
-BenchmarkJSONParserGet-8         	 3000000	       442 ns/op	      21 B/op	       0 allocs/op
-```
-
-Benchmarks for the `GetMany` function:
-
-```
-BenchmarkGJSONGetMany4Paths-8     	 4000000	       319 ns/op	     112 B/op	       0 allocs/op
-BenchmarkGJSONGetMany8Paths-8     	 8000000	       218 ns/op	      56 B/op	       0 allocs/op
-BenchmarkGJSONGetMany16Paths-8    	16000000	       160 ns/op	      56 B/op	       0 allocs/op
-BenchmarkGJSONGetMany32Paths-8    	32000000	       130 ns/op	      64 B/op	       0 allocs/op
-BenchmarkGJSONGetMany64Paths-8    	64000000	       117 ns/op	      64 B/op	       0 allocs/op
-BenchmarkGJSONGetMany128Paths-8    128000000	       109 ns/op	      64 B/op	       0 allocs/op
+BenchmarkGJSONGet-8                  3000000        372 ns/op          0 B/op         0 allocs/op
+BenchmarkGJSONUnmarshalMap-8          900000       4154 ns/op       1920 B/op        26 allocs/op
+BenchmarkJSONUnmarshalMap-8           600000       9019 ns/op       3048 B/op        69 allocs/op
+BenchmarkJSONDecoder-8                300000      14120 ns/op       4224 B/op       184 allocs/op
+BenchmarkFFJSONLexer-8               1500000       3111 ns/op        896 B/op         8 allocs/op
+BenchmarkEasyJSONLexer-8             3000000        887 ns/op        613 B/op         6 allocs/op
+BenchmarkJSONParserGet-8             3000000        499 ns/op         21 B/op         0 allocs/op
+BenchmarkJSONIterator-8              3000000        812 ns/op        544 B/op         9 allocs/op
 ```

 JSON document used:
@@ -344,22 +390,8 @@ widget.image.hOffset
 widget.text.onMouseUp
 ```

-For the `GetMany` benchmarks these paths are used:
+*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.8 and can be be found [here](https://github.com/tidwall/gjson-benchmarks).*

-```
-widget.window.name
-widget.image.hOffset
-widget.text.onMouseUp
-widget.window.title
-widget.image.alignment
-widget.text.style
-widget.window.height
-widget.image.src
-widget.text.data
-widget.text.size
-```
-
-*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.7.*

 ## Contact
 Josh Baker [@tidwall](http://twitter.com/tidwall)
@@ -0,0 +1,10 @@
+//+build appengine
+
+package gjson
+
+func getBytes(json []byte, path string) Result {
+	return Get(string(json), path)
+}
+func fillIndex(json string, c *parseContext) {
+	// noop. Use zero for the Index value.
+}
@@ -0,0 +1,73 @@
+//+build !appengine
+
+package gjson
+
+import (
+	"reflect"
+	"unsafe"
+)
+
+// getBytes casts the input json bytes to a string and safely returns the
+// results as uniquely allocated data. This operation is intended to minimize
+// copies and allocations for the large json string->[]byte.
+func getBytes(json []byte, path string) Result {
+	var result Result
+	if json != nil {
+		// unsafe cast to string
+		result = Get(*(*string)(unsafe.Pointer(&json)), path)
+		result = fromBytesGet(result)
+	}
+	return result
+}
+
+func fromBytesGet(result Result) Result {
+	// safely get the string headers
+	rawhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Raw))
+	strhi := *(*reflect.StringHeader)(unsafe.Pointer(&result.Str))
+	// create byte slice headers
+	rawh := reflect.SliceHeader{Data: rawhi.Data, Len: rawhi.Len}
+	strh := reflect.SliceHeader{Data: strhi.Data, Len: strhi.Len}
+	if strh.Data == 0 {
+		// str is nil
+		if rawh.Data == 0 {
+			// raw is nil
+			result.Raw = ""
+		} else {
+			// raw has data, safely copy the slice header to a string
+			result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
+		}
+		result.Str = ""
+	} else if rawh.Data == 0 {
+		// raw is nil
+		result.Raw = ""
+		// str has data, safely copy the slice header to a string
+		result.Str = string(*(*[]byte)(unsafe.Pointer(&strh)))
+	} else if strh.Data >= rawh.Data &&
+		int(strh.Data)+strh.Len <= int(rawh.Data)+rawh.Len {
+		// Str is a substring of Raw.
+		start := int(strh.Data - rawh.Data)
+		// safely copy the raw slice header
+		result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
+		// substring the raw
+		result.Str = result.Raw[start : start+strh.Len]
+	} else {
+		// safely copy both the raw and str slice headers to strings
+		result.Raw = string(*(*[]byte)(unsafe.Pointer(&rawh)))
+		result.Str = string(*(*[]byte)(unsafe.Pointer(&strh)))
+	}
+	return result
+}
+
+// fillIndex finds the position of Raw data and assigns it to the Index field
+// of the resulting value. If the position cannot be found then Index zero is
+// used instead.
+func fillIndex(json string, c *parseContext) {
+	if len(c.value.Raw) > 0 && !c.calcd {
+		jhdr := *(*reflect.StringHeader)(unsafe.Pointer(&json))
+		rhdr := *(*reflect.StringHeader)(unsafe.Pointer(&(c.value.Raw)))
+		c.value.Index = int(rhdr.Data - jhdr.Data)
+		if c.value.Index < 0 || c.value.Index >= len(json) {
+			c.value.Index = 0
+		}
+	}
+}
@@ -689,10 +689,10 @@
 			"revisionTime": "2016-03-11T21:55:03Z"
 		},
 		{
-			"checksumSHA1": "k/Xh0p5L7+tBCXAL2dOCwUf9J3Y=",
+			"checksumSHA1": "d+4iuDj/qQ+2LiWEbQ+AWjxEJOc=",
 			"path": "github.com/tidwall/gjson",
-			"revision": "09d1c5c5bc64e094394dfe2150220d906c55ac37",
-			"revisionTime": "2017-02-05T16:10:42Z"
+			"revision": "f123b340873a0084cb27267eddd8ff615115fbff",
+			"revisionTime": "2018-06-21T18:09:58Z"
 		},
 		{
 			"checksumSHA1": "qmePMXEDYGwkAfT9QvtMC58JN/E=",
Author	SHA1	Message	Date
Krishna Srinivas	40ed0d1f5d	Support 1GB disk size (#6137 ) Pivotal CF by default has 1GB disk option which causes minio to not start	2018-07-09 18:23:49 -07:00
Andreas Auernhammer	b181a693fb	fix object rebinding SSE-C security guarantee violation (#6121 ) This commit fixes a weakness of the key-encryption-key derivation for SSE-C encrypted objects. Before this change the key-encryption-key was not bound to / didn't depend on the object path. This allows an attacker to repalce objects - encrypted with the same client-key - with each other. This change fixes this issue by updating the key-encryption-key derivation to include: - the domain (in this case SSE-C) - a canonical object path representation - the encryption & key derivation algorithm Changing the object path now causes the KDF to derive a different key-encryption-key such that the object-key unsealing fails. Including the domain (SSE-C) and encryption & key derivation algorithm is not directly neccessary for this fix. However, both will be included for the SSE-S3 KDF. So they are included here to avoid updating the KDF again when we add SSE-S3. The leagcy KDF 'DARE-SHA256' is only used for existing objects and never for new objects / key rotation.	2018-07-09 17:18:28 -07:00
Anis Elleuch	4ddc222f46	fix: Propagate bucket policy update in a distributed setup (#6135 ) Commit `0d52126023` caused a regression in setting a new bucket policy in a distributed setup. The reason is that gob is not able to encode fields declared as interfaces unless we provide GobEncode() and GobDecode() This PR adds them by using json marshaller and unmarshaller that are already implemented for Functions interface.	2018-07-09 02:18:48 -07:00
Mike Scarlett	c310cbbe89	Update comments regarding GCS component count (#6131 )	2018-07-06 17:07:11 -07:00
Harshavardhana	0ef0d7e685	pkg/certs: On windows watch for directory changes to load certs (#6128 ) This PR fixes an issue when configuring Minio TLS on windows	2018-07-05 16:33:37 -07:00
wksw	c62813c887	completed sample config and fix wrong json format (#6096 ) missing worm configuration and wrong json format	2018-07-05 13:40:07 +05:30
poornas	1da362538b	cache: allow ellipse style entries for MINIO_CACHE_DRIVES (#6088 ) Fixes #5863	2018-07-03 16:54:10 -07:00
Krishna Srinivas	e40a5e05e1	Do notification in background to not block S3 client REST calls (#6005 )	2018-07-03 11:09:36 -07:00
ebozduman	b0b0fb4c8d	Validate Minio config.json file on the client side (#6067 )	2018-07-03 11:07:46 -07:00
kannappanr	726e75611e	Do not log BucketNotFound errors on minio console (#6114 )	2018-07-03 11:04:55 -07:00
Rafael Peria de Sene	317e648c0d	Add support for ppc64le (#6116 ) Signed-off-by: Rafael Peria de Sene <rpsene@br.ibm.com>	2018-07-03 09:42:19 -07:00
Harshavardhana	80b3e9cb03	use appropriate HTTP status for storage is full (#6117 )	2018-07-03 09:40:14 -07:00
Harshavardhana	6c85706c24	Use GetSourceIP for source ip as request params (#6109 ) Fixes #6108	2018-07-02 14:40:18 -07:00
Harshavardhana	d7ced9a8b5	Upgrade gjson to bring in new performance improvements (#6106 )	2018-07-02 13:31:11 -07:00
Praveen raj Mani	360f3f9335	Checking the existence of the bucket in DeleteObjectHandler (#6085 ) Fixes #6077	2018-06-30 22:35:43 -07:00
wd256	25f9b0bc3b	Handle ListObjectsV2 start-after parameter in ObjectLayer (#6078 )	2018-07-01 09:52:45 +05:30
Harshavardhana	a5453c307f	Fix kernel tuning script to ignore write failures (#6107 ) Certain SCSI drivers do not allow certain tuning parameters like nr_requests, max_sectors_kb to be changed, ignore these errors silently as this script is simply a best effort. Fixes #6103	2018-06-30 14:55:21 -07:00
Harshavardhana	92a6676a2f	Avoid unnecessary logging for policy not found errors (#6104 )	2018-06-29 06:30:10 -07:00
Minio Trusted	f53d511798	Update yaml files to latest version RELEASE.2018-06-29T02-11-29Z	2018-06-29 02:14:58 +00:00