Log errors only once for event notification errors (#5905)

This PR fixes another situation where a crash occurs
thanks to @krishnasrinivas for reproducing this

Fixes #5897

Dockerfile

@@ -1,6 +1,6 @@
 FROM golang:1.10.1-alpine3.7
 
-MAINTAINER Minio Inc <dev@minio.io>
+LABEL maintainer="Minio Inc <dev@minio.io>"
 
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin

Dockerfile.dev

@@ -1,6 +1,6 @@
 FROM golang:1.10.1-alpine3.7
 
-MAINTAINER Minio Inc <dev@minio.io>
+LABEL maintainer="Minio Inc <dev@minio.io>"
 
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin

Dockerfile.release

@@ -1,7 +1,7 @@
 FROM alpine:3.7
 
-MAINTAINER Minio Inc <dev@minio.io>
- 
+LABEL maintainer="Minio Inc <dev@minio.io>"
+
 COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
 
 ENV MINIO_UPDATE off
@@ -27,4 +27,3 @@ HEALTHCHECK --interval=30s --timeout=5s \
     CMD /usr/bin/healthcheck.sh
 
 CMD ["minio"]
-

cmd/admin-handlers.go

@@ -356,6 +356,13 @@ func (a adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "ClearLocks")
 
+	// Get object layer instance.
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
@@ -387,7 +394,7 @@ func (a adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	newObjectLayerFn().ClearLocks(ctx, volLocks)
+	objLayer.ClearLocks(ctx, volLocks)
 
 	// Reply with list of locks cleared, as json.
 	writeSuccessResponseJSON(w, jsonBytes)
@@ -493,6 +500,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	// after 10s unless a response item comes in
 	keepConnLive := func(w http.ResponseWriter, respCh chan healResp) {
 		ticker := time.NewTicker(time.Second * 10)
+		defer ticker.Stop()
 		started := false
 	forLoop:
 		for {
@@ -521,7 +529,6 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 				break forLoop
 			}
 		}
-		ticker.Stop()
 	}
 
 	// find number of disks in the setup
@@ -561,7 +568,6 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 // GetConfigHandler - GET /minio/admin/v1/config
 // Get config.json of this minio setup.
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
-
 	// Validate request signature.
 	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
 	if adminAPIErr != ErrNone {
@@ -569,12 +575,6 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	// check if objectLayer is initialized, if not return.
-	if newObjectLayerFn() == nil {
-		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
-		return
-	}
-
 	// Take a read lock on minio/config.json. NB minio is a
 	// reserved bucket name and wouldn't conflict with normal
 	// object operations.

cmd/admin-handlers_test.go

@@ -31,7 +31,7 @@ import (
 	"testing"
 	"time"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/madmin"
 )
@@ -140,7 +140,7 @@ type adminXLTestBed struct {
 	configPath string
 	xlDirs     []string
 	objLayer   ObjectLayer
-	mux        *router.Router
+	router     *mux.Router
 }
 
 // prepareAdminXLTestBed - helper function that setups a single-node
@@ -181,15 +181,18 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 		return nil, err
 	}
 
+	// Create new policy system.
+	globalPolicySys = NewPolicySys()
+
 	// Setup admin mgmt REST API handlers.
-	adminRouter := router.NewRouter()
+	adminRouter := mux.NewRouter()
 	registerAdminRouter(adminRouter)
 
 	return &adminXLTestBed{
 		configPath: rootPath,
 		xlDirs:     xlDirs,
 		objLayer:   objLayer,
-		mux:        adminRouter,
+		router:     adminRouter,
 	}, nil
 }
 
@@ -299,7 +302,7 @@ func TestAdminVersionHandler(t *testing.T) {
 	}
 
 	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
+	adminTestBed.router.ServeHTTP(rec, req)
 	if http.StatusOK != rec.Code {
 		t.Errorf("Unexpected status code - got %d but expected %d",
 			rec.Code, http.StatusOK)
@@ -440,7 +443,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	globalMinioAddr = "127.0.0.1:9000"
 	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
-	// Setting up a go routine to simulate ServerMux's
+	// Setting up a go routine to simulate ServerRouter's
 	// handleServiceSignals for stop and restart commands.
 	if cmd == restartCmd {
 		go testServiceSignalReceiver(cmd, t)
@@ -459,7 +462,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	}
 
 	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
+	adminTestBed.router.ServeHTTP(rec, req)
 
 	if cmd == statusCmd {
 		expectedInfo := madmin.ServiceStatus{
@@ -543,7 +546,7 @@ func TestServiceSetCreds(t *testing.T) {
 		rec := httptest.NewRecorder()
 
 		// Execute request
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 
 		// Check if the http code response is expected
 		if rec.Code != testCase.ExpectedStatusCode {
@@ -636,7 +639,7 @@ func TestListLocksHandler(t *testing.T) {
 			t.Fatalf("Test %d - Failed to sign list locks request - %v", i+1, err)
 		}
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		if test.expectedStatus != rec.Code {
 			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
 		}
@@ -703,7 +706,7 @@ func TestClearLocksHandler(t *testing.T) {
 			t.Fatalf("Test %d - Failed to sign clear locks request - %v", i+1, err)
 		}
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		if test.expectedStatus != rec.Code {
 			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.expectedStatus, rec.Code)
 		}
@@ -795,7 +798,7 @@ func TestGetConfigHandler(t *testing.T) {
 	}
 
 	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
+	adminTestBed.router.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
 		t.Errorf("Expected to succeed but failed with %d", rec.Code)
 	}
@@ -829,7 +832,7 @@ func TestSetConfigHandler(t *testing.T) {
 	}
 
 	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
+	adminTestBed.router.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
 		t.Errorf("Expected to succeed but failed with %d", rec.Code)
 	}
@@ -855,7 +858,7 @@ func TestSetConfigHandler(t *testing.T) {
 		}
 
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		respBody := string(rec.Body.Bytes())
 		if rec.Code != http.StatusBadRequest ||
 			!strings.Contains(respBody, "Configuration data provided exceeds the allowed maximum of") {
@@ -874,7 +877,7 @@ func TestSetConfigHandler(t *testing.T) {
 		}
 
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		respBody := string(rec.Body.Bytes())
 		if rec.Code != http.StatusBadRequest ||
 			!strings.Contains(respBody, "JSON configuration provided has objects with duplicate keys") {
@@ -904,7 +907,7 @@ func TestAdminServerInfo(t *testing.T) {
 	}
 
 	rec := httptest.NewRecorder()
-	adminTestBed.mux.ServeHTTP(rec, req)
+	adminTestBed.router.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
 		t.Errorf("Expected to succeed but failed with %d", rec.Code)
 	}
@@ -1105,7 +1108,7 @@ func collectHealResults(t *testing.T, adminTestBed *adminXLTestBed, bucket,
 		}
 		req := mkHealStatusReq(t, bucket, prefix, clientToken)
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		if http.StatusOK != rec.Code {
 			t.Errorf("Unexpected status code - got %d but expected %d",
 				rec.Code, http.StatusOK)
@@ -1151,7 +1154,7 @@ func TestHealStartNStatusHandler(t *testing.T) {
 	{
 		req := mkHealStartReq(t, bucketName, objName, healOpts)
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		if http.StatusOK != rec.Code {
 			t.Errorf("Unexpected status code - got %d but expected %d",
 				rec.Code, http.StatusOK)
@@ -1171,7 +1174,7 @@ func TestHealStartNStatusHandler(t *testing.T) {
 		// test with an invalid client token
 		req := mkHealStatusReq(t, bucketName, objName, hss.ClientToken+hss.ClientToken)
 		rec := httptest.NewRecorder()
-		adminTestBed.mux.ServeHTTP(rec, req)
+		adminTestBed.router.ServeHTTP(rec, req)
 		if rec.Code != http.StatusBadRequest {
 			t.Errorf("Unexpected status code")
 		}

cmd/admin-router.go

@@ -19,7 +19,7 @@ package cmd
 import (
 	"net/http"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 )
 
 const (
@@ -31,11 +31,11 @@ type adminAPIHandlers struct {
 }
 
 // registerAdminRouter - Add handler functions for each service REST API routes.
-func registerAdminRouter(mux *router.Router) {
+func registerAdminRouter(router *mux.Router) {
 
 	adminAPI := adminAPIHandlers{}
 	// Admin router
-	adminRouter := mux.NewRoute().PathPrefix(adminAPIPathPrefix).Subrouter()
+	adminRouter := router.PathPrefix(adminAPIPathPrefix).Subrouter()
 
 	// Version handler
 	adminRouter.Methods(http.MethodGet).Path("/version").HandlerFunc(adminAPI.VersionHandler)

cmd/admin-rpc-client.go

@@ -89,7 +89,12 @@ func (lc localAdminClient) ReInitFormat(dryRun bool) error {
 
 // ListLocks - Fetches lock information from local lock instrumentation.
 func (lc localAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	return listLocksInfo(bucket, prefix, duration), nil
+	// check if objectLayer is initialized, if not return.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return nil, errServerNotInitialized
+	}
+	return objectAPI.ListLocks(context.Background(), bucket, prefix, duration)
 }
 
 func (rc remoteAdminClient) SignalService(s serviceSignal) (err error) {

cmd/admin-rpc-server.go

@@ -25,7 +25,7 @@ import (
 	"path/filepath"
 	"time"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
 )
 
@@ -102,7 +102,14 @@ func (s *adminCmd) ListLocks(query *ListLocksQuery, reply *ListLocksReply) error
 	if err := query.IsAuthenticated(); err != nil {
 		return err
 	}
-	volLocks := listLocksInfo(query.Bucket, query.Prefix, query.Duration)
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+	volLocks, err := objectAPI.ListLocks(context.Background(), query.Bucket, query.Prefix, query.Duration)
+	if err != nil {
+		return err
+	}
 	*reply = ListLocksReply{VolLocks: volLocks}
 	return nil
 }
@@ -219,7 +226,7 @@ func (s *adminCmd) CommitConfig(cArgs *CommitConfigArgs, cReply *CommitConfigRep
 
 // registerAdminRPCRouter - registers RPC methods for service status,
 // stop and restart commands.
-func registerAdminRPCRouter(mux *router.Router) error {
+func registerAdminRPCRouter(router *mux.Router) error {
 	adminRPCHandler := &adminCmd{}
 	adminRPCServer := newRPCServer()
 	err := adminRPCServer.RegisterName("Admin", adminRPCHandler)
@@ -227,7 +234,7 @@ func registerAdminRPCRouter(mux *router.Router) error {
 		logger.LogIf(context.Background(), err)
 		return err
 	}
-	adminRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
+	adminRouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
 	adminRouter.Path(adminPath).Handler(adminRPCServer)
 	return nil
 }

cmd/api-errors.go

@@ -953,8 +953,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrEntityTooSmall
 	case NotImplemented:
 		apiErr = ErrNotImplemented
-	case PolicyNotFound:
-		apiErr = ErrNoSuchBucketPolicy
 	case PartTooBig:
 		apiErr = ErrEntityTooLarge
 	case UnsupportedMetadata:

cmd/api-router.go

@@ -19,7 +19,7 @@ package cmd
 import (
 	"net/http"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
 )
 
@@ -30,7 +30,7 @@ type objectAPIHandlers struct {
 }
 
 // registerAPIRouter - registers S3 compatible APIs.
-func registerAPIRouter(mux *router.Router) {
+func registerAPIRouter(router *mux.Router) {
 	var err error
 	var cacheConfig = globalServerConfig.GetCacheConfig()
 	if len(cacheConfig.Drives) > 0 {
@@ -46,8 +46,8 @@ func registerAPIRouter(mux *router.Router) {
 	}
 
 	// API Router
-	apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
-	var routers []*router.Router
+	apiRouter := router.PathPrefix("/").Subrouter()
+	var routers []*mux.Router
 	if globalDomainName != "" {
 		routers = append(routers, apiRouter.Host("{bucket:.+}."+globalDomainName).Subrouter())
 	}

cmd/auth-handler.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015-2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,12 +22,13 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"errors"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"strings"
 
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/handlers"
+	"github.com/minio/minio/pkg/policy"
 )
 
 // Verify if request has JWT.
@@ -123,28 +124,66 @@ func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
 	return s3Err
 }
 
-func checkRequestAuthType(ctx context.Context, r *http.Request, bucket, policyAction, region string) APIErrorCode {
-	reqAuthType := getRequestAuthType(r)
+func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) APIErrorCode {
+	isOwner := true
+	accountName := globalServerConfig.GetCredential().AccessKey
 
-	switch reqAuthType {
+	switch getRequestAuthType(r) {
+	case authTypeUnknown:
+		return ErrAccessDenied
 	case authTypePresignedV2, authTypeSignedV2:
-		// Signature V2 validation.
-		return isReqAuthenticatedV2(r)
-	case authTypeSigned, authTypePresigned:
-		return isReqAuthenticated(r, region)
-	}
-
-	if reqAuthType == authTypeAnonymous && policyAction != "" {
-		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
-		if err != nil {
-			return ErrInternalError
+		if errorCode := isReqAuthenticatedV2(r); errorCode != ErrNone {
+			return errorCode
 		}
-		return enforceBucketPolicy(ctx, bucket, policyAction, resource,
-			r.Referer(), handlers.GetSourceIP(r), r.URL.Query())
+	case authTypeSigned, authTypePresigned:
+		region := globalServerConfig.GetRegion()
+		switch action {
+		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
+			region = ""
+		}
+
+		if errorCode := isReqAuthenticated(r, region); errorCode != ErrNone {
+			return errorCode
+		}
+	default:
+		isOwner = false
+		accountName = ""
+	}
+
+	// LocationConstraint is valid only for CreateBucketAction.
+	var locationConstraint string
+	if action == policy.CreateBucketAction {
+		// To extract region from XML in request body, get copy of request body.
+		payload, err := ioutil.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return ErrMalformedXML
+		}
+
+		// Populate payload to extract location constraint.
+		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+
+		var s3Error APIErrorCode
+		locationConstraint, s3Error = parseLocationConstraint(r)
+		if s3Error != ErrNone {
+			return s3Error
+		}
+
+		// Populate payload again to handle it in HTTP handler.
+		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+	}
+
+	if globalPolicySys.IsAllowed(policy.Args{
+		AccountName:     accountName,
+		Action:          action,
+		BucketName:      bucketName,
+		ConditionValues: getConditionValues(r, locationConstraint),
+		IsOwner:         isOwner,
+		ObjectName:      objectName,
+	}) {
+		return ErrNone
 	}
 
-	// By default return ErrAccessDenied
 	return ErrAccessDenied
 }
 

cmd/browser-rpc-router.go

@@ -19,7 +19,7 @@ package cmd
 import (
 	"context"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
 )
 
@@ -37,7 +37,7 @@ type browserPeerAPIHandlers struct {
 }
 
 // Register RPC router
-func registerBrowserPeerRPCRouter(mux *router.Router) error {
+func registerBrowserPeerRPCRouter(router *mux.Router) error {
 	bpHandlers := &browserPeerAPIHandlers{AuthRPCServer{}}
 
 	bpRPCServer := newRPCServer()
@@ -47,7 +47,7 @@ func registerBrowserPeerRPCRouter(mux *router.Router) error {
 		return err
 	}
 
-	bpRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
+	bpRouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
 	bpRouter.Path(browserPeerPath).Handler(bpRPCServer)
 	return nil
 }

cmd/bucket-handlers-listobjects.go

@@ -20,6 +20,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/policy"
 )
 
 // Validate all the ListObjects query arguments, returns an APIErrorCode
@@ -64,7 +65,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -134,7 +135,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}

cmd/bucket-handlers.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"encoding/base64"
 	"encoding/xml"
 	"io"
@@ -26,86 +25,16 @@ import (
 	"net/url"
 	"path"
 	"path/filepath"
-	"reflect"
 	"strings"
 	"sync"
 
-	mux "github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/policy"
 )
 
-// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-// Enforces bucket policies for a bucket for a given tatusaction.
-func enforceBucketPolicy(ctx context.Context, bucket, action, resource, referer, sourceIP string, queryParams url.Values) (s3Error APIErrorCode) {
-	// Verify if bucket actually exists
-	objAPI := newObjectLayerFn()
-	if err := checkBucketExist(ctx, bucket, objAPI); err != nil {
-		switch err.(type) {
-		case BucketNameInvalid:
-			// Return error for invalid bucket name.
-			return ErrInvalidBucketName
-		case BucketNotFound:
-			// For no bucket found we return NoSuchBucket instead.
-			return ErrNoSuchBucket
-		}
-		// Return internal error for any other errors so that we can investigate.
-		return ErrInternalError
-	}
-
-	// Fetch bucket policy, if policy is not set return access denied.
-	p, err := objAPI.GetBucketPolicy(ctx, bucket)
-	if err != nil {
-		return ErrAccessDenied
-	}
-	if reflect.DeepEqual(p, emptyBucketPolicy) {
-		return ErrAccessDenied
-	}
-
-	// Construct resource in 'arn:aws:s3:::examplebucket/object' format.
-	arn := bucketARNPrefix + strings.TrimSuffix(strings.TrimPrefix(resource, "/"), "/")
-
-	// Get conditions for policy verification.
-	conditionKeyMap := make(policy.ConditionKeyMap)
-	for queryParam := range queryParams {
-		conditionKeyMap[queryParam] = set.CreateStringSet(queryParams.Get(queryParam))
-	}
-
-	// Add request referer to conditionKeyMap if present.
-	if referer != "" {
-		conditionKeyMap["referer"] = set.CreateStringSet(referer)
-	}
-	// Add request source Ip to conditionKeyMap.
-	conditionKeyMap["ip"] = set.CreateStringSet(sourceIP)
-
-	// Validate action, resource and conditions with current policy statements.
-	if !bucketPolicyEvalStatements(action, arn, conditionKeyMap, p.Statements) {
-		return ErrAccessDenied
-	}
-	return ErrNone
-}
-
-// Check if the action is allowed on the bucket/prefix.
-func isBucketActionAllowed(action, bucket, prefix string, objectAPI ObjectLayer) bool {
-	reqInfo := &logger.ReqInfo{BucketName: bucket}
-	reqInfo.AppendTags("prefix", prefix)
-	ctx := logger.SetReqInfo(context.Background(), reqInfo)
-	bp, err := objectAPI.GetBucketPolicy(ctx, bucket)
-	if err != nil {
-		return false
-	}
-	if reflect.DeepEqual(bp, emptyBucketPolicy) {
-		return false
-	}
-	resource := bucketARNPrefix + path.Join(bucket, prefix)
-	var conditionKeyMap map[string]set.StringSet
-	// Validate action, resource and conditions with current policy statements.
-	return bucketPolicyEvalStatements(action, resource, conditionKeyMap, bp.Statements)
-}
-
 // GetBucketLocationHandler - GET Bucket location.
 // -------------------------
 // This operation returns bucket location.
@@ -121,12 +50,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	s3Error := checkRequestAuthType(ctx, r, bucket, "s3:GetBucketLocation", globalMinioDefaultRegion)
-	if s3Error == ErrInvalidRegion {
-		// Clients like boto3 send getBucketLocation() call signed with region that is configured.
-		s3Error = checkRequestAuthType(ctx, r, "", "s3:GetBucketLocation", globalServerConfig.GetRegion())
-	}
-	if s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLocationAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -180,7 +104,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:ListBucketMultipartUploads", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketMultipartUploadsAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -228,16 +152,12 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	if api.CacheAPI() != nil {
 		listBuckets = api.CacheAPI().ListBuckets
 	}
-	// ListBuckets does not have any bucket action.
-	s3Error := checkRequestAuthType(ctx, r, "", "", globalMinioDefaultRegion)
-	if s3Error == ErrInvalidRegion {
-		// Clients like boto3 send listBuckets() call signed with region that is configured.
-		s3Error = checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion())
-	}
-	if s3Error != ErrNone {
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListAllMyBucketsAction, "", ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
+
 	// Invoke the list buckets.
 	bucketsInfo, err := listBuckets(ctx)
 	if err != nil {
@@ -266,12 +186,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	var authError APIErrorCode
-	if authError = checkRequestAuthType(ctx, r, bucket, "s3:DeleteObject", globalServerConfig.GetRegion()); authError != ErrNone {
+	var s3Error APIErrorCode
+	if s3Error = checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, ""); s3Error != ErrNone {
 		// In the event access is denied, a 200 response should still be returned
 		// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
-		if authError != ErrAccessDenied {
-			writeErrorResponse(w, authError, r.URL)
+		if s3Error != ErrAccessDenied {
+			writeErrorResponse(w, s3Error, r.URL)
 			return
 		}
 	}
@@ -326,7 +246,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			defer wg.Done()
 			// If the request is denied access, each item
 			// should be marked as 'AccessDenied'
-			if authError == ErrAccessDenied {
+			if s3Error == ErrAccessDenied {
 				dErrs[i] = PrefixAccessDenied{
 					Bucket: bucket,
 					Object: obj.ObjectName,
@@ -411,16 +331,14 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	// PutBucket does not have any bucket action.
-	s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion())
-	if s3Error != ErrNone {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
 	// Parse incoming location constraint.
 	location, s3Error := parseLocationConstraint(r)
 	if s3Error != ErrNone {
@@ -690,10 +608,11 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponseHeadersOnly(w, s3Error)
 		return
 	}
+
 	getBucketInfo := objectAPI.GetBucketInfo
 	if api.CacheAPI() != nil {
 		getBucketInfo = api.CacheAPI().GetBucketInfo
@@ -710,20 +629,20 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "DeleteBucket")
 
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
 
-	// DeleteBucket does not have any bucket action.
-	if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
 	deleteBucket := objectAPI.DeleteBucket
 	if api.CacheAPI() != nil {
 		deleteBucket = api.CacheAPI().DeleteBucket
@@ -734,16 +653,11 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
-	// Notify all peers (including self) to update in-memory state
-	for addr, err := range globalNotificationSys.UpdateBucketPolicy(bucket) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
-		logger.LogIf(ctx, err)
-	}
-
 	globalNotificationSys.RemoveNotification(bucket)
-	for addr, err := range globalNotificationSys.DeleteBucket(bucket) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
-		logger.LogIf(ctx, err)
+	globalPolicySys.Remove(bucket)
+	for nerr := range globalNotificationSys.DeleteBucket(bucket) {
+		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
+		logger.LogIf(ctx, nerr.Err)
 	}
 
 	// Write success response.

cmd/bucket-handlers_test.go

@@ -155,7 +155,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getReadOnlyBucketStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestGetBucketLocationHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyBucketStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestGetBucketLocationHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonReadOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -260,7 +260,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getReadOnlyBucketStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestHeadBucketHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyBucketStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestHeadBucketHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonReadOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -313,7 +313,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 			maxUploads:         "0",
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
-			expectedRespStatus: http.StatusBadRequest,
+			expectedRespStatus: http.StatusNotFound,
 			shouldPass:         false,
 		},
 		// Test case - 2.
@@ -494,7 +494,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyBucketStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestListMultipartUploadsHandler", bucketName, "", instanceType, apiRouter, anonReq, getWriteOnlyBucketStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestListMultipartUploadsHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -592,7 +592,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "ListBucketsHandler", "", "", instanceType, apiRouter, anonReq, getWriteOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "ListBucketsHandler", "", "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy("*"))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -793,31 +793,3 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	// `ExecObjectLayerAPINilTest` manages the operation.
 	ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq)
 }
-
-func TestIsBucketActionAllowed(t *testing.T) {
-	ExecObjectLayerAPITest(t, testIsBucketActionAllowedHandler, []string{"BucketLocation"})
-}
-
-func testIsBucketActionAllowedHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
-
-	testCases := []struct {
-		// input.
-		action              string
-		bucket              string
-		prefix              string
-		isGlobalPoliciesNil bool
-		// flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		{"s3:GetBucketLocation", "mybucket", "abc", true, false},
-		{"s3:ListObject", "mybucket", "abc", false, false},
-	}
-	for i, testCase := range testCases {
-		isAllowed := isBucketActionAllowed(testCase.action, testCase.bucket, testCase.prefix, obj)
-		if isAllowed != testCase.shouldPass {
-			t.Errorf("Case %d: Expected the response status to be `%t`, but instead found `%t`", i+1, testCase.shouldPass, isAllowed)
-		}
-
-	}
-}

cmd/bucket-notification-handlers.go

@@ -27,6 +27,7 @@ import (
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
 	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/policy"
 )
 
 const (
@@ -43,6 +44,9 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
 func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "GetBucketNotification")
 
+	vars := mux.Vars(r)
+	bucketName := vars["bucket"]
+
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -53,14 +57,12 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		writeErrorResponse(w, ErrNotImplemented, r.URL)
 		return
 	}
-	if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucketName := vars["bucket"]
-
 	_, err := objAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -104,14 +106,15 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		writeErrorResponse(w, ErrNotImplemented, r.URL)
 		return
 	}
-	if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
-		return
-	}
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+
 	_, err := objectAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -143,9 +146,9 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 
 	rulesMap := config.ToRulesMap()
 	globalNotificationSys.AddRulesMap(bucketName, rulesMap)
-	for addr, err := range globalNotificationSys.PutBucketNotification(bucketName, rulesMap) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
-		logger.LogIf(ctx, err)
+	for nerr := range globalNotificationSys.PutBucketNotification(bucketName, rulesMap) {
+		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
+		logger.LogIf(ctx, nerr.Err)
 	}
 
 	writeSuccessResponseHeadersOnly(w)
@@ -166,14 +169,15 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		writeErrorResponse(w, ErrNotImplemented, r.URL)
 		return
 	}
-	if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
-		return
-	}
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListenBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+
 	values := r.URL.Query()
 
 	var prefix string
@@ -220,8 +224,12 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
-	host := xnet.MustParseHost(r.RemoteAddr)
-	target := target.NewHTTPClientTarget(*host, w)
+	host, e := xnet.ParseHost(r.RemoteAddr)
+	logger.CriticalIf(ctx, e)
+
+	target, e := target.NewHTTPClientTarget(*host, w)
+	logger.CriticalIf(ctx, e)
+
 	rulesMap := event.NewRulesMap(eventNames, pattern, target.ID())
 
 	if err := globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
@@ -233,7 +241,9 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
 	defer globalNotificationSys.RemoveRulesMap(bucketName, rulesMap)
 
-	thisAddr := xnet.MustParseHost(GetLocalPeer(globalEndpoints))
+	thisAddr, e := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	logger.CriticalIf(ctx, e)
+
 	if err := SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
 		logger.LogIf(ctx, err)
@@ -241,10 +251,10 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
-	errors := globalNotificationSys.ListenBucketNotification(bucketName, eventNames, pattern, target.ID(), *thisAddr)
-	for addr, err := range errors {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
-		logger.LogIf(ctx, err)
+	errCh := globalNotificationSys.ListenBucketNotification(bucketName, eventNames, pattern, target.ID(), *thisAddr)
+	for nerr := range errCh {
+		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
+		logger.LogIf(ctx, nerr.Err)
 	}
 
 	<-target.DoneCh

cmd/bucket-policy-handlers.go

@@ -19,206 +19,24 @@ package cmd
 import (
 	"encoding/json"
 	"io"
-	"io/ioutil"
-	"net"
 	"net/http"
-	"runtime"
-	"strings"
 
 	humanize "github.com/dustin/go-humanize"
-	mux "github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/policy"
+	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/wildcard"
+	"github.com/minio/minio/pkg/policy"
 )
 
-// maximum supported access policy size.
-const maxAccessPolicySize = 20 * humanize.KiByte
+const (
+	// As per AWS S3 specification, 20KiB policy JSON data is allowed.
+	maxBucketPolicySize = 20 * humanize.KiByte
 
-// Verify if a given action is valid for the url path based on the
-// existing bucket access policy.
-func bucketPolicyEvalStatements(action string, resource string, conditions policy.ConditionKeyMap,
-	statements []policy.Statement) bool {
-	for _, statement := range statements {
-		if bucketPolicyMatchStatement(action, resource, conditions, statement) {
-			if statement.Effect == "Allow" {
-				return true
-			}
-			// Do not uncomment kept here for readability.
-			// else statement.Effect == "Deny"
-			return false
-		}
-	}
-	// None match so deny.
-	return false
-}
+	// Policy configuration file.
+	bucketPolicyConfig = "policy.json"
+)
 
-// Verify if action, resource and conditions match input policy statement.
-func bucketPolicyMatchStatement(action string, resource string, conditions policy.ConditionKeyMap,
-	statement policy.Statement) bool {
-	// Verify if action, resource and condition match in given statement.
-	return (bucketPolicyActionMatch(action, statement) &&
-		bucketPolicyResourceMatch(resource, statement) &&
-		bucketPolicyConditionMatch(conditions, statement))
-}
-
-// Verify if given action matches with policy statement.
-func bucketPolicyActionMatch(action string, statement policy.Statement) bool {
-	return !statement.Actions.FuncMatch(actionMatch, action).IsEmpty()
-}
-
-// Match function matches wild cards in 'pattern' for resource.
-func resourceMatch(pattern, resource string) bool {
-	if runtime.GOOS == "windows" {
-		// For windows specifically make sure we are case insensitive.
-		return wildcard.Match(strings.ToLower(pattern), strings.ToLower(resource))
-	}
-	return wildcard.Match(pattern, resource)
-}
-
-// Match function matches wild cards in 'pattern' for action.
-func actionMatch(pattern, action string) bool {
-	return wildcard.MatchSimple(pattern, action)
-}
-
-func refererMatch(pattern, referer string) bool {
-	return wildcard.MatchSimple(pattern, referer)
-}
-
-// isIPInCIDR - checks if a given a IP address is a member of the given subnet.
-func isIPInCIDR(cidr, ip string) bool {
-	// AWS S3 spec says IPs must use standard CIDR notation.
-	// http://docs.aws.amazon.com/AmazonS3/latest/dev/example-bucket-policies.html#example-bucket-policies-use-case-3.
-	_, cidrNet, err := net.ParseCIDR(cidr)
-	if err != nil {
-		return false // If provided CIDR can't be parsed no IP will be in the subnet.
-	}
-	addr := net.ParseIP(ip)
-	return cidrNet.Contains(addr)
-}
-
-// Verify if given resource matches with policy statement.
-func bucketPolicyResourceMatch(resource string, statement policy.Statement) bool {
-	// the resource rule for object could contain "*" wild card.
-	// the requested object can be given access based on the already set bucket policy if
-	// the match is successful.
-	// More info: http://docs.aws.amazon.com/AmazonS3/latest/dev/s3-arn-format.html.
-	return !statement.Resources.FuncMatch(resourceMatch, resource).IsEmpty()
-}
-
-// Verify if given condition matches with policy statement.
-func bucketPolicyConditionMatch(conditions policy.ConditionKeyMap, statement policy.Statement) bool {
-	// Supports following conditions.
-	// - StringEquals
-	// - StringNotEquals
-	// - StringLike
-	// - StringNotLike
-	// - IpAddress
-	// - NotIpAddress
-	//
-	// Supported applicable condition keys for each conditions.
-	// - s3:prefix
-	// - s3:max-keys
-	// - s3:aws-Referer
-	// - s3:aws-SourceIp
-
-	// The following loop evaluates the logical AND of all the
-	// conditions in the statement. Note: we can break out of the
-	// loop if and only if a condition evaluates to false.
-	for condition, conditionKeyVal := range statement.Conditions {
-		prefixConditon := conditionKeyVal["s3:prefix"]
-		maxKeyCondition := conditionKeyVal["s3:max-keys"]
-		if condition == "StringEquals" {
-			// If there is no condition with "s3:prefix" or "s3:max-keys" condition key
-			// then there is nothing to check condition against.
-			if !prefixConditon.IsEmpty() && !prefixConditon.Equals(conditions["prefix"]) {
-				return false
-			}
-			if !maxKeyCondition.IsEmpty() && !maxKeyCondition.Equals(conditions["max-keys"]) {
-				return false
-			}
-		} else if condition == "StringNotEquals" {
-			// If there is no condition with "s3:prefix" or "s3:max-keys" condition key
-			// then there is nothing to check condition against.
-			if !prefixConditon.IsEmpty() && prefixConditon.Equals(conditions["prefix"]) {
-				return false
-			}
-			if !maxKeyCondition.IsEmpty() && maxKeyCondition.Equals(conditions["max-keys"]) {
-				return false
-			}
-		} else if condition == "StringLike" {
-			awsReferers := conditionKeyVal["aws:Referer"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsReferers.IsEmpty() {
-				continue
-			}
-			// wildcard match of referer in statement was not empty.
-			// StringLike has a match, i.e, condition evaluates to true.
-			refererFound := false
-			for referer := range conditions["referer"] {
-				if !awsReferers.FuncMatch(refererMatch, referer).IsEmpty() {
-					refererFound = true
-					break
-				}
-			}
-			// No matching referer found, so the condition is false.
-			if !refererFound {
-				return false
-			}
-		} else if condition == "StringNotLike" {
-			awsReferers := conditionKeyVal["aws:Referer"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsReferers.IsEmpty() {
-				continue
-			}
-			// wildcard match of referer in statement was not empty.
-			// StringNotLike has a match, i.e, condition evaluates to false.
-			for referer := range conditions["referer"] {
-				if !awsReferers.FuncMatch(refererMatch, referer).IsEmpty() {
-					return false
-				}
-			}
-		} else if condition == "IpAddress" {
-			awsIps := conditionKeyVal["aws:SourceIp"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsIps.IsEmpty() {
-				continue
-			}
-			// wildcard match of ip if statement was not empty.
-			// Find a valid ip.
-			ipFound := false
-			for ip := range conditions["ip"] {
-				if !awsIps.FuncMatch(isIPInCIDR, ip).IsEmpty() {
-					ipFound = true
-					break
-				}
-			}
-			if !ipFound {
-				return false
-			}
-		} else if condition == "NotIpAddress" {
-			awsIps := conditionKeyVal["aws:SourceIp"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsIps.IsEmpty() {
-				continue
-			}
-			// wildcard match of ip if statement was not empty.
-			// Find if nothing matches.
-			for ip := range conditions["ip"] {
-				if !awsIps.FuncMatch(isIPInCIDR, ip).IsEmpty() {
-					return false
-				}
-			}
-		}
-	}
-
-	return true
-}
-
-// PutBucketPolicyHandler - PUT Bucket policy
-// -----------------
-// This implementation of the PUT operation uses the policy
-// subresource to add to or replace a policy on a bucket
+// PutBucketPolicyHandler - This HTTP handler stores given bucket policy configuration as per
+// https://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "PutBucketPolicy")
 
@@ -228,77 +46,61 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
-	if err != nil {
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	// If Content-Length is unknown or zero, deny the
-	// request. PutBucketPolicy always needs a Content-Length.
-	if r.ContentLength == -1 || r.ContentLength == 0 {
+	// Error out if Content-Length is missing.
+	// PutBucketPolicy always needs Content-Length.
+	if r.ContentLength <= 0 {
 		writeErrorResponse(w, ErrMissingContentLength, r.URL)
 		return
 	}
-	// If Content-Length is greater than maximum allowed policy size.
-	if r.ContentLength > maxAccessPolicySize {
+
+	// Error out if Content-Length is beyond allowed size.
+	if r.ContentLength > maxBucketPolicySize {
 		writeErrorResponse(w, ErrEntityTooLarge, r.URL)
 		return
 	}
 
-	// Read access policy up to maxAccessPolicySize.
-	// http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
-	// bucket policies are limited to 20KB in size, using a limit reader.
-	policyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
+	bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		writeErrorResponse(w, ErrMalformedPolicy, r.URL)
+		return
+	}
+
+	// Version in policy must not be empty
+	if bucketPolicy.Version == "" {
+		writeErrorResponse(w, ErrMalformedPolicy, r.URL)
+		return
+	}
+
+	if err = objAPI.SetBucketPolicy(ctx, bucket, bucketPolicy); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
-	policyInfo := policy.BucketAccessPolicy{}
-	if err = json.Unmarshal(policyBytes, &policyInfo); err != nil {
-		writeErrorResponse(w, ErrInvalidPolicyDocument, r.URL)
-		return
-	}
-	// Parse check bucket policy.
-	if s3Error := checkBucketPolicyResources(bucket, policyInfo); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
-		return
-	}
 
-	if err = objAPI.SetBucketPolicy(ctx, bucket, policyInfo); err != nil {
-		switch err.(type) {
-		case NotImplemented:
-			// Return error for invalid bucket name.
-			writeErrorResponse(w, ErrNotImplemented, r.URL)
-		default:
-			writeErrorResponse(w, ErrMalformedPolicy, r.URL)
-		}
-		return
-	}
-
-	for addr, err := range globalNotificationSys.UpdateBucketPolicy(bucket) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
-		logger.LogIf(ctx, err)
+	globalPolicySys.Set(bucket, *bucketPolicy)
+	for nerr := range globalNotificationSys.SetBucketPolicy(bucket, bucketPolicy) {
+		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
+		logger.LogIf(ctx, nerr.Err)
 	}
 
 	// Success.
 	writeSuccessNoContent(w)
 }
 
-// DeleteBucketPolicyHandler - DELETE Bucket policy
-// -----------------
-// This implementation of the DELETE operation uses the policy
-// subresource to add to remove a policy on a bucket.
+// DeleteBucketPolicyHandler - This HTTP handler removes bucket policy configuration.
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "DeleteBucketPolicy")
 
@@ -308,41 +110,36 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
-	if err != nil {
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	// Delete bucket access policy, by passing an empty policy
-	// struct.
 	if err := objAPI.DeleteBucketPolicy(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	for addr, err := range globalNotificationSys.UpdateBucketPolicy(bucket) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
-		logger.LogIf(ctx, err)
+	globalPolicySys.Remove(bucket)
+	for nerr := range globalNotificationSys.RemoveBucketPolicy(bucket) {
+		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
+		logger.LogIf(ctx, nerr.Err)
 	}
 
 	// Success.
 	writeSuccessNoContent(w)
 }
 
-// GetBucketPolicyHandler - GET Bucket policy
-// -----------------
-// This operation uses the policy
-// subresource to return the policy of a specified bucket.
+// GetBucketPolicyHandler - This HTTP handler returns bucket policy configuration.
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, "GetBucketPolicy")
 
@@ -352,29 +149,28 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	// Before proceeding validate if bucket exists.
-	_, err := objAPI.GetBucketInfo(ctx, bucket)
-	if err != nil {
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Read bucket access policy.
-	policy, err := objAPI.GetBucketPolicy(ctx, bucket)
+	bucketPolicy, err := objAPI.GetBucketPolicy(ctx, bucket)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	policyBytes, err := json.Marshal(&policy)
+	policyData, err := json.Marshal(bucketPolicy)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -382,5 +178,5 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Write to client.
-	w.Write(policyBytes)
+	w.Write(policyData)
 }

cmd/bucket-policy-handlers_test.go

@@ -24,222 +24,72 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"reflect"
+	"strings"
 	"testing"
 
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/policy/condition"
 )
 
-// Tests validate Bucket policy resource matcher.
-func TestBucketPolicyResourceMatch(t *testing.T) {
-
-	// generates statement with given resource..
-	generateStatement := func(resource string) policy.Statement {
-		statement := policy.Statement{}
-		statement.Resources = set.CreateStringSet([]string{resource}...)
-		return statement
-	}
-
-	// generates resource prefix.
-	generateResource := func(bucketName, objectName string) string {
-		return bucketARNPrefix + bucketName + "/" + objectName
-	}
-
-	testCases := []struct {
-		resourceToMatch       string
-		statement             policy.Statement
-		expectedResourceMatch bool
-	}{
-		// Test case 1-4.
-		// Policy with resource ending with bucket/* allows access to all objects inside the given bucket.
-		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
-		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
-		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
-		{generateResource("minio-bucket", ""), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/*")), true},
-		// Test case - 5.
-		// Policy with resource ending with bucket/oo* should not allow access to bucket/output.txt.
-		{generateResource("minio-bucket", "output.txt"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/oo*")), false},
-		// Test case - 6.
-		// Policy with resource ending with bucket/oo* should allow access to bucket/ootput.txt.
-		{generateResource("minio-bucket", "ootput.txt"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/oo*")), true},
-		// Test case - 7.
-		// Policy with resource ending with bucket/oo* allows access to all sub-dirs starting with "oo" inside given bucket.
-		{generateResource("minio-bucket", "oop-bucket/my-file"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/oo*")), true},
-		// Test case - 8.
-		{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/Asia/Japan/*")), false},
-		// Test case - 9.
-		{generateResource("minio-bucket", "Asia/India/1.pjg"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/Asia/Japan/*")), false},
-		// Test case - 10.
-		// Proves that the name space is flat.
-		{generateResource("minio-bucket", "Africa/Bihar/India/design_info.doc/Bihar"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix,
-			"minio-bucket"+"/*/India/*/Bihar")), true},
-		// Test case - 11.
-		// Proves that the name space is flat.
-		{generateResource("minio-bucket", "Asia/China/India/States/Bihar/output.txt"), generateStatement(fmt.Sprintf("%s%s", bucketARNPrefix,
-			"minio-bucket"+"/*/India/*/Bihar/*")), true},
-	}
-	for i, testCase := range testCases {
-		actualResourceMatch := bucketPolicyResourceMatch(testCase.resourceToMatch, testCase.statement)
-		if testCase.expectedResourceMatch != actualResourceMatch {
-			t.Errorf("Test %d: Expected Resource match to be `%v`, but instead found it to be `%v`", i+1, testCase.expectedResourceMatch, actualResourceMatch)
-		}
+func getAnonReadOnlyBucketPolicy(bucketName string) *policy.Policy {
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{policy.NewStatement(
+			policy.Allow,
+			policy.NewPrincipal("*"),
+			policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
+			policy.NewResourceSet(policy.NewResource(bucketName, "")),
+			condition.NewFunctions(),
+		)},
 	}
 }
 
-// TestBucketPolicyActionMatch - Test validates whether given action on the
-// bucket/object matches the allowed actions in policy.Statement.
-// This test preserves the allowed actions for all 3 sets of policies, that is read-write,read-only, write-only.
-// The intention of the test is to catch any changes made to allowed action for on eof the above 3 major policy groups mentioned.
-func TestBucketPolicyActionMatch(t *testing.T) {
-	bucketName := getRandomBucketName()
-	objectPrefix := "test-object"
-
-	testCases := []struct {
-		action         string
-		statement      policy.Statement
-		expectedResult bool
-	}{
-		// s3:GetBucketLocation is the action necessary to be present in the bucket policy to allow
-		// fetching of bucket location on an Anonymous/unsigned request.
-
-		//r ead-write bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 1).
-		{"s3:GetBucketLocation", getReadWriteBucketStatement(bucketName, objectPrefix), true},
-		//	write-only bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 2).
-		{"s3:GetBucketLocation", getWriteOnlyBucketStatement(bucketName, objectPrefix), true},
-		//	read-only bucket policy is expected to allow GetBucketLocation operation on an anonymous request (Test case - 3).
-		{"s3:GetBucketLocation", getReadOnlyBucketStatement(bucketName, objectPrefix), true},
-
-		// Any of the Object level access permissions shouldn't allow for GetBucketLocation operation  on an Anonymous/unsigned request (Test cases 4-6).
-		{"s3:GetBucketLocation", getReadWriteObjectStatement(bucketName, objectPrefix), false},
-		{"s3:GetBucketLocation", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
-		{"s3:GetBucketLocation", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-
-		// s3:ListBucketMultipartUploads is the action necessary to be present in the bucket policy to allow
-		// Listing of multipart uploads in a given bucket for an Anonymous/unsigned request.
-
-		//read-write bucket policy is expected to allow  ListBucketMultipartUploads operation on an anonymous request (Test case 7).
-		{"s3:ListBucketMultipartUploads", getReadWriteBucketStatement(bucketName, objectPrefix), true},
-		//	write-only bucket policy is expected to allow  ListBucketMultipartUploads operation on an anonymous request (Test case 8).
-		{"s3:ListBucketMultipartUploads", getWriteOnlyBucketStatement(bucketName, objectPrefix), true},
-		// read-only bucket policy is expected to not allow ListBucketMultipartUploads operation on an anonymous request (Test case 9).
-		// the allowed actions in read-only bucket statement are  "s3:GetBucketLocation","s3:ListBucket",
-		// this should not allow for ListBucketMultipartUploads operations.
-		{"s3:ListBucketMultipartUploads", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Any of the object level policy will not allow for s3:ListBucketMultipartUploads (Test cases 10-12).
-		{"s3:ListBucketMultipartUploads", getReadWriteObjectStatement(bucketName, objectPrefix), false},
-		{"s3:ListBucketMultipartUploads", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
-		{"s3:ListBucketMultipartUploads", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-
-		// s3:ListBucket is the action necessary to be present in the bucket policy to allow
-		// listing of all objects inside a given bucket on an Anonymous/unsigned request.
-
-		// Cases for testing ListBucket access for different Bucket level access permissions.
-		// read-only bucket policy is expected to allow ListBucket operation on an anonymous request (Test case 13).
-		{"s3:ListBucket", getReadOnlyBucketStatement(bucketName, objectPrefix), true},
-		// read-write bucket policy is expected to allow ListBucket operation on an anonymous request (Test case 14).
-		{"s3:ListBucket", getReadWriteBucketStatement(bucketName, objectPrefix), true},
-		// write-only  bucket policy is expected to not allow ListBucket operation on an anonymous request (Test case 15).
-		// the allowed actions in write-only  bucket statement are "s3:GetBucketLocation",	"s3:ListBucketMultipartUploads",
-		// this should not allow for ListBucket operations.
-		{"s3:ListBucket", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Cases for testing ListBucket access for different Object level access permissions (Test cases 16-18).
-		// Any of the Object level access permissions shouldn't allow for ListBucket operation  on an Anonymous/unsigned request.
-		{"s3:ListBucket", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-		{"s3:ListBucket", getReadWriteObjectStatement(bucketName, objectPrefix), false},
-		{"s3:ListBucket", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
-
-		// s3:DeleteObject is the action necessary to be present in the bucket policy to allow
-		// deleting/removal of  objects inside a given bucket for an Anonymous/unsigned request.
-
-		// Cases for testing DeleteObject access for different Bucket level access permissions (Test cases 19-21).
-		// Any of the Bucket level access permissions shouldn't allow for DeleteObject operation  on an Anonymous/unsigned request.
-		{"s3:DeleteObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-		{"s3:DeleteObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
-		{"s3:DeleteObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Cases for testing DeleteObject access for different Object level access permissions (Test cases 22).
-		// read-only bucket policy is expected to not allow Delete Object operation on an anonymous request.
-		{"s3:DeleteObject", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-		// read-write bucket policy is expected to allow Delete Bucket operation on an anonymous request (Test cases 23).
-		{"s3:DeleteObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
-		// write-only  bucket policy is expected to allow Delete Object operation on an anonymous request (Test cases 24).
-		{"s3:DeleteObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
-
-		// s3:AbortMultipartUpload is the action necessary to be present in the bucket policy to allow
-		// cancelling or abortion of an already initiated multipart upload operation for an Anonymous/unsigned request.
-
-		// Cases for testing AbortMultipartUpload access for different Bucket level access permissions (Test cases 25-27).
-		// Any of the Bucket level access permissions shouldn't allow for AbortMultipartUpload operation on an Anonymous/unsigned request.
-		{"s3:AbortMultipartUpload", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-		{"s3:AbortMultipartUpload", getReadWriteBucketStatement(bucketName, objectPrefix), false},
-		{"s3:AbortMultipartUpload", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Cases for testing AbortMultipartUpload access for different Object level access permissions.
-		// read-only object policy is expected to not allow AbortMultipartUpload operation on an anonymous request (Test case 28).
-		{"s3:AbortMultipartUpload", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-		// read-write object policy is expected to allow AbortMultipartUpload operation on an anonymous request (Test case 29).
-		{"s3:AbortMultipartUpload", getReadWriteObjectStatement(bucketName, objectPrefix), true},
-		// write-only object policy is expected to allow AbortMultipartUpload operation on an anonymous request (Test case 30).
-		{"s3:AbortMultipartUpload", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
-
-		// s3:PutObject is the action necessary to be present in the bucket policy to allow
-		// uploading of an object for an Anonymous/unsigned request.
-
-		// Cases for testing PutObject access for different Bucket level access permissions (Test cases 31-33).
-		// Any of the Bucket level access permissions shouldn't allow for PutObject operation on an Anonymous/unsigned request.
-		{"s3:PutObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-		{"s3:PutObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
-		{"s3:PutObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Cases for testing PutObject access for different Object level access permissions.
-		// read-only object policy is expected to not allow PutObject operation on an anonymous request (Test case 34).
-		{"s3:PutObject", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-		// read-write object policy is expected to allow PutObject operation on an anonymous request (Test case 35).
-		{"s3:PutObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
-		// write-only  object policy is expected to allow PutObject operation on an anonymous request (Test case 36).
-		{"s3:PutObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
-
-		// s3:GetObject is the action necessary to be present in the bucket policy to allow
-		// downloading of an object for an Anonymous/unsigned request.
-
-		// Cases for testing GetObject access for different Bucket level access permissions (Test cases 37-39).
-		// Any of the Bucket level access permissions shouldn't allow for GetObject operation on an Anonymous/unsigned request.
-		{"s3:GetObject", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-		{"s3:GetObject", getReadWriteBucketStatement(bucketName, objectPrefix), false},
-		{"s3:GetObject", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// Cases for testing GetObject access for different Object level access permissions.
-		// read-only bucket policy is expected to allow  downloading of an Object on an anonymous request (Test case 40).
-		{"s3:GetObject", getReadOnlyObjectStatement(bucketName, objectPrefix), true},
-		// read-write bucket policy is expected to allow  downloading of an Object on an anonymous request (Test case 41).
-		{"s3:GetObject", getReadWriteObjectStatement(bucketName, objectPrefix), true},
-		// write-only  bucket policy is expected to not allow  downloading of an Object on an anonymous request (Test case 42).
-		{"s3:GetObject", getWriteOnlyObjectStatement(bucketName, objectPrefix), false},
-
-		// s3:ListMultipartUploadParts is the action necessary to be present in the bucket policy to allow
-		// Listing of uploaded parts for an Anonymous/unsigned request.
-
-		// Any of the Bucket level access permissions shouldn't allow for ListMultipartUploadParts operation on an Anonymous/unsigned request.
-		// read-only bucket policy is expected to not allow ListMultipartUploadParts operation on an anonymous request (Test cases 43-45).
-		{"s3:ListMultipartUploadParts", getReadOnlyBucketStatement(bucketName, objectPrefix), false},
-		{"s3:ListMultipartUploadParts", getReadWriteBucketStatement(bucketName, objectPrefix), false},
-		{"s3:ListMultipartUploadParts", getWriteOnlyBucketStatement(bucketName, objectPrefix), false},
-
-		// read-only object policy is expected to not allow ListMultipartUploadParts operation on an anonymous request (Test case 46).
-		{"s3:ListMultipartUploadParts", getReadOnlyObjectStatement(bucketName, objectPrefix), false},
-		// read-write object policy is expected to allow ListMultipartUploadParts operation on an anonymous request (Test case 47).
-		{"s3:ListMultipartUploadParts", getReadWriteObjectStatement(bucketName, objectPrefix), true},
-		// write-only  object policy is expected to allow ListMultipartUploadParts operation on an anonymous request (Test case 48).
-		{"s3:ListMultipartUploadParts", getWriteOnlyObjectStatement(bucketName, objectPrefix), true},
+func getAnonWriteOnlyBucketPolicy(bucketName string) *policy.Policy {
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{policy.NewStatement(
+			policy.Allow,
+			policy.NewPrincipal("*"),
+			policy.NewActionSet(
+				policy.GetBucketLocationAction,
+				policy.ListBucketMultipartUploadsAction,
+			),
+			policy.NewResourceSet(policy.NewResource(bucketName, "")),
+			condition.NewFunctions(),
+		)},
 	}
-	for i, testCase := range testCases {
-		actualResult := bucketPolicyActionMatch(testCase.action, testCase.statement)
-		if testCase.expectedResult != actualResult {
-			t.Errorf("Test %d: Expected the result to be `%v`, but instead found it to be `%v`", i+1, testCase.expectedResult, actualResult)
-		}
+}
+
+func getAnonReadOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{policy.NewStatement(
+			policy.Allow,
+			policy.NewPrincipal("*"),
+			policy.NewActionSet(policy.GetObjectAction),
+			policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
+			condition.NewFunctions(),
+		)},
+	}
+}
+
+func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{policy.NewStatement(
+			policy.Allow,
+			policy.NewPrincipal("*"),
+			policy.NewActionSet(
+				policy.AbortMultipartUploadAction,
+				policy.DeleteObjectAction,
+				policy.ListMultipartUploadPartsAction,
+				policy.PutObjectAction,
+			),
+			policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
+			condition.NewFunctions(),
+		)},
 	}
 }
 
@@ -260,6 +110,8 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	// template for constructing HTTP request body for PUT bucket policy.
 	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/this*"]}]}`
 
+	bucketPolicyTemplateWithoutVersion := `{"Version":"","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/this*"]}]}`
+
 	// test cases with sample input and expected output.
 	testCases := []struct {
 		bucketName string
@@ -290,7 +142,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			bucketName:         bucketName,
 			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName))),
 
-			policyLen:          maxAccessPolicySize + 1,
+			policyLen:          maxBucketPolicySize + 1,
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
 			expectedRespStatus: http.StatusBadRequest,
@@ -357,7 +209,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		// Test case - 8.
 		// non-existent bucket is used.
 		// writing BucketPolicy should fail.
-		// should result is 500 InternalServerError.
+		// should result in 404 StatusNotFound
 		{
 			bucketName:         "non-existent-bucket",
 			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, "non-existent-bucket", "non-existent-bucket"))),
@@ -368,9 +220,9 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			expectedRespStatus: http.StatusNotFound,
 		},
 		// Test case - 9.
-		// invalid bucket name is used.
+		// non-existent bucket is used (with invalid bucket name)
 		// writing BucketPolicy should fail.
-		// should result is 400 StatusBadRequest.
+		// should result in 404 StatusNotFound
 		{
 			bucketName:         ".invalid-bucket",
 			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplate, ".invalid-bucket", ".invalid-bucket"))),
@@ -378,6 +230,19 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			policyLen:          len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)),
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
+			expectedRespStatus: http.StatusNotFound,
+		},
+		// Test case - 10.
+		// Existent bucket with policy with Version field empty.
+		// writing BucketPolicy should fail.
+		// should result in 400 StatusBadRequest.
+		{
+			bucketName:         bucketName,
+			bucketPolicyReader: bytes.NewReader([]byte(fmt.Sprintf(bucketPolicyTemplateWithoutVersion, bucketName, bucketName))),
+
+			policyLen:          len(fmt.Sprintf(bucketPolicyTemplateWithoutVersion, bucketName, bucketName)),
+			accessKey:          credentials.AccessKey,
+			secretKey:          credentials.SecretKey,
 			expectedRespStatus: http.StatusBadRequest,
 		},
 	}
@@ -430,7 +295,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "PutBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getWriteOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "PutBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -459,7 +324,7 @@ func TestGetBucketPolicyHandler(t *testing.T) {
 func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
 	credentials auth.Credentials, t *testing.T) {
 	// template for constructing HTTP request body for PUT bucket policy.
-	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s/this*"],"Sid":""}]}`
+	bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s"]},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s/this*"]}]}`
 
 	// Writing bucket policy before running test on GetBucketPolicy.
 	putTestPolicies := []struct {
@@ -535,13 +400,13 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			expectedRespStatus:   http.StatusNotFound,
 		},
 		// Test case - 3.
-		// Case with invalid bucket name.
+		// Case with non-existent bucket name.
 		{
 			bucketName:           ".invalid-bucket-name",
 			accessKey:            credentials.AccessKey,
 			secretKey:            credentials.SecretKey,
 			expectedBucketPolicy: "",
-			expectedRespStatus:   http.StatusBadRequest,
+			expectedRespStatus:   http.StatusNotFound,
 		},
 	}
 	// Iterating over the cases, fetching the policy and validating the response.
@@ -572,7 +437,16 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 
 		if recV4.Code != testCase.expectedRespStatus {
 			// Verify whether the bucket policy fetched is same as the one inserted.
-			if expectedBucketPolicyStr != string(bucketPolicyReadBuf) {
+			expectedPolicy, err := policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
+			if err != nil {
+				t.Fatalf("unexpected error. %v", err)
+			}
+			gotPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
+			if err != nil {
+				t.Fatalf("unexpected error. %v", err)
+			}
+
+			if !reflect.DeepEqual(expectedPolicy, gotPolicy) {
 				t.Errorf("Test %d: %s: Bucket policy differs from expected value.", i+1, instanceType)
 			}
 		}
@@ -598,7 +472,16 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		}
 		if recV2.Code == http.StatusOK {
 			// Verify whether the bucket policy fetched is same as the one inserted.
-			if expectedBucketPolicyStr != string(bucketPolicyReadBuf) {
+			expectedPolicy, err := policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
+			if err != nil {
+				t.Fatalf("unexpected error. %v", err)
+			}
+			gotPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
+			if err != nil {
+				t.Fatalf("unexpected error. %v", err)
+			}
+
+			if !reflect.DeepEqual(expectedPolicy, gotPolicy) {
 				t.Errorf("Test %d: %s: Bucket policy differs from expected value.", i+1, instanceType)
 			}
 		}
@@ -617,7 +500,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "GetBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "GetBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonReadOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -742,12 +625,12 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 			expectedRespStatus: http.StatusNotFound,
 		},
 		// Test case - 3.
-		// Case with invalid bucket name.
+		// Case with non-existent-bucket.
 		{
 			bucketName:         ".invalid-bucket-name",
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
-			expectedRespStatus: http.StatusBadRequest,
+			expectedRespStatus: http.StatusNotFound,
 		},
 	}
 	// Iterating over the cases and deleting the bucket policy and then asserting response.
@@ -820,7 +703,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "DeleteBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getReadOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "DeleteBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy(bucketName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -838,174 +721,3 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 	// `ExecObjectLayerAPINilTest` manages the operation.
 	ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq)
 }
-
-// TestBucketPolicyConditionMatch - Tests to validate whether bucket policy conditions match.
-func TestBucketPolicyConditionMatch(t *testing.T) {
-	// obtain the inner map[string]set.StringSet for policy.Statement.Conditions.
-	getInnerMap := func(key2, value string) map[string]set.StringSet {
-		innerMap := make(map[string]set.StringSet)
-		innerMap[key2] = set.CreateStringSet(value)
-		return innerMap
-	}
-
-	// obtain policy.Statement with Conditions set.
-	getStatementWithCondition := func(key1, key2, value string) policy.Statement {
-		innerMap := getInnerMap(key2, value)
-		// to set policyStatment.Conditions .
-		conditions := make(policy.ConditionMap)
-		conditions[key1] = innerMap
-		// new policy statement.
-		statement := policy.Statement{}
-		// set the condition.
-		statement.Conditions = conditions
-		return statement
-	}
-
-	testCases := []struct {
-		statementCondition policy.Statement
-		condition          map[string]set.StringSet
-
-		expectedMatch bool
-	}{
-
-		// Test case - 1.
-		// StringEquals condition matches.
-		{
-
-			statementCondition: getStatementWithCondition("StringEquals", "s3:prefix", "Asia/"),
-			condition:          getInnerMap("prefix", "Asia/"),
-
-			expectedMatch: true,
-		},
-		// Test case - 2.
-		// StringEquals condition doesn't match.
-		{
-
-			statementCondition: getStatementWithCondition("StringEquals", "s3:prefix", "Asia/"),
-			condition:          getInnerMap("prefix", "Africa/"),
-
-			expectedMatch: false,
-		},
-		// Test case - 3.
-		// StringEquals condition matches.
-		{
-
-			statementCondition: getStatementWithCondition("StringEquals", "s3:max-keys", "Asia/"),
-			condition:          getInnerMap("max-keys", "Asia/"),
-
-			expectedMatch: true,
-		},
-		// Test case - 4.
-		// StringEquals condition doesn't match.
-		{
-
-			statementCondition: getStatementWithCondition("StringEquals", "s3:max-keys", "Asia/"),
-			condition:          getInnerMap("max-keys", "Africa/"),
-
-			expectedMatch: false,
-		},
-		// Test case - 5.
-		// StringNotEquals condition matches.
-		{
-
-			statementCondition: getStatementWithCondition("StringNotEquals", "s3:prefix", "Asia/"),
-			condition:          getInnerMap("prefix", "Asia/"),
-
-			expectedMatch: false,
-		},
-		// Test case - 6.
-		// StringNotEquals condition doesn't match.
-		{
-
-			statementCondition: getStatementWithCondition("StringNotEquals", "s3:prefix", "Asia/"),
-			condition:          getInnerMap("prefix", "Africa/"),
-
-			expectedMatch: true,
-		},
-		// Test case - 7.
-		// StringNotEquals condition matches.
-		{
-
-			statementCondition: getStatementWithCondition("StringNotEquals", "s3:max-keys", "Asia/"),
-			condition:          getInnerMap("max-keys", "Asia/"),
-
-			expectedMatch: false,
-		},
-		// Test case - 8.
-		// StringNotEquals condition doesn't match.
-		{
-
-			statementCondition: getStatementWithCondition("StringNotEquals", "s3:max-keys", "Asia/"),
-			condition:          getInnerMap("max-keys", "Africa/"),
-
-			expectedMatch: true,
-		},
-		// Test case - 9.
-		// StringLike condition matches.
-		{
-			statementCondition: getStatementWithCondition("StringLike", "aws:Referer", "http://www.example.com/"),
-			condition:          getInnerMap("referer", "http://www.example.com/"),
-			expectedMatch:      true,
-		},
-		// Test case - 10.
-		// StringLike condition doesn't match.
-		{
-			statementCondition: getStatementWithCondition("StringLike", "aws:Referer", "http://www.example.com/"),
-			condition:          getInnerMap("referer", "www.somethingelse.com"),
-			expectedMatch:      false,
-		},
-		// Test case - 11.
-		// StringNotLike condition evaluates to false.
-		{
-			statementCondition: getStatementWithCondition("StringNotLike", "aws:Referer", "http://www.example.com/"),
-			condition:          getInnerMap("referer", "http://www.example.com/"),
-			expectedMatch:      false,
-		},
-		// Test case - 12.
-		// StringNotLike condition evaluates to true.
-		{
-			statementCondition: getStatementWithCondition("StringNotLike", "aws:Referer", "http://www.example.com/"),
-			condition:          getInnerMap("referer", "http://somethingelse.com/"),
-			expectedMatch:      true,
-		},
-		// Test case 13.
-		// IpAddress condition evaluates to true.
-		{
-			statementCondition: getStatementWithCondition("IpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "54.240.143.2"),
-			expectedMatch:      true,
-		},
-		// Test case 14.
-		// IpAddress condition evaluates to false.
-		{
-			statementCondition: getStatementWithCondition("IpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "127.240.143.224"),
-			expectedMatch:      false,
-		},
-		// Test case 15.
-		// NotIpAddress condition evaluates to true.
-		{
-			statementCondition: getStatementWithCondition("NotIpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "54.240.144.188"),
-			expectedMatch:      true,
-		},
-		// Test case 16.
-		// NotIpAddress condition evaluates to false.
-		{
-			statementCondition: getStatementWithCondition("NotIpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "54.240.143.243"),
-			expectedMatch:      false,
-		},
-	}
-
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("Case %d", i+1), func(t *testing.T) {
-			// call the function under test and assert the result with the expected result.
-			doesMatch := bucketPolicyConditionMatch(tc.condition, tc.statementCondition)
-			if tc.expectedMatch != doesMatch {
-				t.Errorf("Expected the match to be `%v`; got `%v` - %v %v.",
-					tc.expectedMatch, doesMatch, tc.condition, tc.statementCondition)
-			}
-		})
-	}
-}

cmd/bucket-policy-parser.go

@@ -1,295 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Package cmd This file implements AWS Access Policy Language parser in
-// accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
-package cmd
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"sort"
-	"strings"
-
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio-go/pkg/set"
-)
-
-var emptyBucketPolicy = policy.BucketAccessPolicy{}
-
-var conditionKeyActionMap = policy.ConditionKeyMap{
-	"s3:prefix": set.CreateStringSet("s3:ListBucket", "s3:ListBucketMultipartUploads"),
-	"s3:max-keys": set.CreateStringSet("s3:ListBucket", "s3:ListBucketMultipartUploads",
-		"s3:ListMultipartUploadParts"),
-}
-
-// supportedActionMap - lists all the actions supported by minio.
-var supportedActionMap = set.CreateStringSet("*", "s3:*", "s3:GetObject",
-	"s3:ListBucket", "s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject",
-	"s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts")
-
-// supported Conditions type.
-var supportedConditionsType = set.CreateStringSet("StringEquals", "StringNotEquals", "StringLike", "StringNotLike", "IpAddress", "NotIpAddress")
-
-// Validate s3:prefix, s3:max-keys are present if not
-// supported keys for the conditions.
-var supportedConditionsKey = set.CreateStringSet("s3:prefix", "s3:max-keys", "aws:Referer", "aws:SourceIp")
-
-// supportedEffectMap - supported effects.
-var supportedEffectMap = set.CreateStringSet("Allow", "Deny")
-
-// isValidActions - are actions valid.
-func isValidActions(actions set.StringSet) (err error) {
-	// Statement actions cannot be empty.
-	if actions.IsEmpty() {
-		err = errors.New("Action list cannot be empty")
-		return err
-	}
-	if unsupportedActions := actions.Difference(supportedActionMap); !unsupportedActions.IsEmpty() {
-		err = fmt.Errorf("Unsupported actions found: ‘%#v’, please validate your policy document",
-			unsupportedActions)
-		return err
-	}
-	return nil
-}
-
-// isValidEffect - is effect valid.
-func isValidEffect(effect string) (err error) {
-	// Statement effect cannot be empty.
-	if effect == "" {
-		err = errors.New("Policy effect cannot be empty")
-		return err
-	}
-	if !supportedEffectMap.Contains(effect) {
-		err = errors.New("Unsupported Effect found: ‘" + effect + "’, please validate your policy document")
-		return err
-	}
-	return nil
-}
-
-// isValidResources - are valid resources.
-func isValidResources(resources set.StringSet) (err error) {
-	// Statement resources cannot be empty.
-	if resources.IsEmpty() {
-		err = errors.New("Resource list cannot be empty")
-		return err
-	}
-	for resource := range resources {
-		if !hasPrefix(resource, bucketARNPrefix) {
-			err = errors.New("Unsupported resource style found: ‘" + resource + "’, please validate your policy document")
-			return err
-		}
-		resourceSuffix := strings.SplitAfter(resource, bucketARNPrefix)[1]
-		if len(resourceSuffix) == 0 || hasPrefix(resourceSuffix, "/") {
-			err = errors.New("Invalid resource style found: ‘" + resource + "’, please validate your policy document")
-			return err
-		}
-	}
-	return nil
-}
-
-// isValidPrincipals - are valid principals.
-func isValidPrincipals(principal policy.User) (err error) {
-	if principal.AWS.IsEmpty() {
-		return errors.New("Principal cannot be empty")
-	}
-	if diff := principal.AWS.Difference(set.CreateStringSet("*")); !diff.IsEmpty() {
-		// Minio does not support or implement IAM, "*" is the only valid value.
-		// Amazon s3 doc on principal:
-		// http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Principal
-		err = fmt.Errorf("Unsupported principals found: ‘%#v’, please validate your policy document",
-			diff)
-		return err
-	}
-	return nil
-}
-
-// isValidConditions - returns nil if the given conditions valid and
-// corresponding error otherwise.
-func isValidConditions(actions set.StringSet, conditions policy.ConditionMap) (err error) {
-	// Verify conditions should be valid. Validate if only
-	// supported condition keys are present and return error
-	// otherwise.
-	conditionKeyVal := make(map[string]set.StringSet)
-	for conditionType := range conditions {
-		if !supportedConditionsType.Contains(conditionType) {
-			err = fmt.Errorf("Unsupported condition type '%s', please validate your policy document", conditionType)
-			return err
-		}
-		for key, value := range conditions[conditionType] {
-			if !supportedConditionsKey.Contains(key) {
-				err = fmt.Errorf("Unsupported condition key '%s', please validate your policy document", conditionType)
-				return err
-			}
-
-			compatibleActions := conditionKeyActionMap[key]
-			if !compatibleActions.IsEmpty() &&
-				compatibleActions.Intersection(actions).IsEmpty() {
-				err = fmt.Errorf("Unsupported condition key %s for the given actions %s, "+
-					"please validate your policy document", key, actions)
-				return err
-			}
-
-			conditionVal, ok := conditionKeyVal[key]
-			if ok && !value.Intersection(conditionVal).IsEmpty() {
-				err = fmt.Errorf("Ambigious condition values for key '%s', please validate your policy document", key)
-				return err
-			}
-			conditionKeyVal[key] = value
-		}
-	}
-	return nil
-}
-
-// List of actions for which prefixes are not allowed.
-var invalidPrefixActions = set.StringSet{
-	"s3:GetBucketLocation":          {},
-	"s3:ListBucket":                 {},
-	"s3:ListBucketMultipartUploads": {},
-	// Add actions which do not honor prefixes.
-}
-
-// resourcePrefix - provides the prefix removing any wildcards.
-func resourcePrefix(resource string) string {
-	if strings.HasSuffix(resource, "*") {
-		resource = strings.TrimSuffix(resource, "*")
-	}
-	return resource
-}
-
-// checkBucketPolicyResources validates Resources in unmarshalled bucket policy structure.
-// - Resources are validated against the given set of Actions.
-// -
-func checkBucketPolicyResources(bucket string, bucketPolicy policy.BucketAccessPolicy) APIErrorCode {
-	// Validate statements for special actions and collect resources
-	// for others to validate nesting.
-	var resourceMap = set.NewStringSet()
-	for _, statement := range bucketPolicy.Statements {
-		for action := range statement.Actions {
-			for resource := range statement.Resources {
-				resourcePrefix := strings.SplitAfter(resource, bucketARNPrefix)[1]
-				if _, ok := invalidPrefixActions[action]; ok {
-					// Resource prefix is not equal to bucket for
-					// prefix invalid actions, reject them.
-					if resourcePrefix != bucket {
-						return ErrMalformedPolicy
-					}
-				} else {
-					// For all other actions validate if resourcePrefix begins
-					// with bucket name, if not reject them.
-					if strings.Split(resourcePrefix, "/")[0] != bucket {
-						return ErrMalformedPolicy
-					}
-					// All valid resources collect them separately to verify nesting.
-					resourceMap.Add(resourcePrefix)
-				}
-			}
-		}
-	}
-
-	var resources []string
-	for resource := range resourceMap {
-		resources = append(resources, resourcePrefix(resource))
-	}
-
-	// Sort strings as shorter first.
-	sort.Strings(resources)
-
-	for len(resources) > 1 {
-		var resource string
-		resource, resources = resources[0], resources[1:]
-		// Loop through all resources, if one of them matches with
-		// previous shorter one, it means we have detected
-		// nesting. Reject such rules.
-		for _, otherResource := range resources {
-			// Common prefix reject such rules.
-			if hasPrefix(otherResource, resource) {
-				return ErrPolicyNesting
-			}
-		}
-	}
-
-	// No errors found.
-	return ErrNone
-}
-
-// parseBucketPolicy - parses and validates if bucket policy is of
-// proper JSON and follows allowed restrictions with policy standards.
-func parseBucketPolicy(bucketPolicyReader io.Reader, bktPolicy *policy.BucketAccessPolicy) (err error) {
-	// Parse bucket policy reader.
-	decoder := json.NewDecoder(bucketPolicyReader)
-	if err = decoder.Decode(bktPolicy); err != nil {
-		return err
-	}
-
-	// Policy version cannot be empty.
-	if len(bktPolicy.Version) == 0 {
-		err = errors.New("Policy version cannot be empty")
-		return err
-	}
-
-	// Policy statements cannot be empty.
-	if len(bktPolicy.Statements) == 0 {
-		err = errors.New("Policy statement cannot be empty")
-		return err
-	}
-
-	// Loop through all policy statements and validate entries.
-	for _, statement := range bktPolicy.Statements {
-		// Statement effect should be valid.
-		if err := isValidEffect(statement.Effect); err != nil {
-			return err
-		}
-		// Statement principal should be supported format.
-		if err := isValidPrincipals(statement.Principal); err != nil {
-			return err
-		}
-		// Statement actions should be valid.
-		if err := isValidActions(statement.Actions); err != nil {
-			return err
-		}
-		// Statement resources should be valid.
-		if err := isValidResources(statement.Resources); err != nil {
-			return err
-		}
-		// Statement conditions should be valid.
-		if err := isValidConditions(statement.Actions, statement.Conditions); err != nil {
-			return err
-		}
-	}
-
-	// Separate deny and allow statements, so that we can apply deny
-	// statements in the beginning followed by Allow statements.
-	var denyStatements []policy.Statement
-	var allowStatements []policy.Statement
-	for _, statement := range bktPolicy.Statements {
-		if statement.Effect == "Deny" {
-			denyStatements = append(denyStatements, statement)
-			continue
-		}
-
-		// else if statement.Effect == "Allow"
-		allowStatements = append(allowStatements, statement)
-	}
-
-	// Deny statements are enforced first once matched.
-	bktPolicy.Statements = append(denyStatements, allowStatements...)
-
-	// Return successfully parsed policy structure.
-	return nil
-}

cmd/bucket-policy-parser_test.go

@@ -1,867 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package cmd
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"reflect"
-	"testing"
-
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio-go/pkg/set"
-)
-
-// Common bucket actions for both read and write policies.
-var (
-	readWriteBucketActions = []string{
-		"s3:GetBucketLocation",
-		"s3:ListBucket",
-		"s3:ListBucketMultipartUploads",
-		// Add more bucket level read-write actions here.
-	}
-	readWriteObjectActions = []string{
-		"s3:AbortMultipartUpload",
-		"s3:DeleteObject",
-		"s3:GetObject",
-		"s3:ListMultipartUploadParts",
-		"s3:PutObject",
-		// Add more object level read-write actions here.
-	}
-)
-
-// Write only actions.
-var (
-	writeOnlyBucketActions = []string{
-		"s3:GetBucketLocation",
-		"s3:ListBucketMultipartUploads",
-		// Add more bucket level write actions here.
-	}
-	writeOnlyObjectActions = []string{
-		"s3:AbortMultipartUpload",
-		"s3:DeleteObject",
-		"s3:ListMultipartUploadParts",
-		"s3:PutObject",
-		// Add more object level write actions here.
-	}
-)
-
-// Read only actions.
-var (
-	readOnlyBucketActions = []string{
-		"s3:GetBucketLocation",
-		"s3:ListBucket",
-		// Add more bucket level read actions here.
-	}
-	readOnlyObjectActions = []string{
-		"s3:GetObject",
-		// Add more object level read actions here.
-	}
-)
-
-// Obtain bucket statement for read-write bucketPolicy.
-func getReadWriteObjectStatement(bucketName, objectPrefix string) policy.Statement {
-	objectResourceStatement := policy.Statement{}
-	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
-	objectResourceStatement.Actions = set.CreateStringSet(readWriteObjectActions...)
-	return objectResourceStatement
-}
-
-// Obtain object statement for read-write bucketPolicy.
-func getReadWriteBucketStatement(bucketName, objectPrefix string) policy.Statement {
-	bucketResourceStatement := policy.Statement{}
-	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
-	bucketResourceStatement.Actions = set.CreateStringSet(readWriteBucketActions...)
-	return bucketResourceStatement
-}
-
-// Obtain statements for read-write bucketPolicy.
-func getReadWriteStatement(bucketName, objectPrefix string) []policy.Statement {
-	statements := []policy.Statement{}
-	// Save the read write policy.
-	statements = append(statements, getReadWriteBucketStatement(bucketName, objectPrefix), getReadWriteObjectStatement(bucketName, objectPrefix))
-	return statements
-}
-
-// Obtain bucket statement for read only bucketPolicy.
-func getReadOnlyBucketStatement(bucketName, objectPrefix string) policy.Statement {
-	bucketResourceStatement := policy.Statement{}
-	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
-	bucketResourceStatement.Actions = set.CreateStringSet(readOnlyBucketActions...)
-	return bucketResourceStatement
-}
-
-// Obtain object statement for read only bucketPolicy.
-func getReadOnlyObjectStatement(bucketName, objectPrefix string) policy.Statement {
-	objectResourceStatement := policy.Statement{}
-	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
-	objectResourceStatement.Actions = set.CreateStringSet(readOnlyObjectActions...)
-	return objectResourceStatement
-}
-
-// Obtain statements for read only bucketPolicy.
-func getReadOnlyStatement(bucketName, objectPrefix string) []policy.Statement {
-	statements := []policy.Statement{}
-	// Save the read only policy.
-	statements = append(statements, getReadOnlyBucketStatement(bucketName, objectPrefix), getReadOnlyObjectStatement(bucketName, objectPrefix))
-	return statements
-}
-
-// Obtain bucket statements for write only bucketPolicy.
-func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policy.Statement {
-
-	bucketResourceStatement := policy.Statement{}
-	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
-	bucketResourceStatement.Actions = set.CreateStringSet(writeOnlyBucketActions...)
-	return bucketResourceStatement
-}
-
-// Obtain object statements for write only bucketPolicy.
-func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policy.Statement {
-	objectResourceStatement := policy.Statement{}
-	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
-	}
-	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
-	objectResourceStatement.Actions = set.CreateStringSet(writeOnlyObjectActions...)
-	return objectResourceStatement
-}
-
-// Obtain statements for write only bucketPolicy.
-func getWriteOnlyStatement(bucketName, objectPrefix string) []policy.Statement {
-	statements := []policy.Statement{}
-	// Write only policy.
-	// Save the write only policy.
-	statements = append(statements, getWriteOnlyBucketStatement(bucketName, objectPrefix), getWriteOnlyBucketStatement(bucketName, objectPrefix))
-	return statements
-}
-
-// Tests validate Action validator.
-func TestIsValidActions(t *testing.T) {
-	testCases := []struct {
-		// input.
-		actions set.StringSet
-		// expected output.
-		err error
-		// flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Inputs with unsupported Action.
-		// Test case - 1.
-		// "s3:ListObject" is an invalid Action.
-		{set.CreateStringSet([]string{"s3:GetObject", "s3:ListObject", "s3:RemoveObject"}...),
-			errors.New("Unsupported actions found: ‘set.StringSet{\"s3:RemoveObject\":struct {}{}, \"s3:ListObject\":struct {}{}}’, please validate your policy document"), false},
-		// Test case - 2.
-		// Empty Actions.
-		{set.CreateStringSet([]string{}...), errors.New("Action list cannot be empty"), false},
-		// Test case - 3.
-		// "s3:DeleteEverything"" is an invalid Action.
-		{set.CreateStringSet([]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}...),
-			errors.New("Unsupported actions found: ‘set.StringSet{\"s3:DeleteEverything\":struct {}{}}’, please validate your policy document"), false},
-		// Inputs with valid Action.
-		// Test Case - 4.
-		{set.CreateStringSet([]string{
-			"s3:*", "*", "s3:GetObject", "s3:ListBucket",
-			"s3:PutObject", "s3:GetBucketLocation", "s3:DeleteObject",
-			"s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads",
-			"s3:ListMultipartUploadParts"}...), nil, true},
-	}
-	for i, testCase := range testCases {
-		err := isValidActions(testCase.actions)
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
-		}
-	}
-}
-
-// Tests validate Effect validator.
-func TestIsValidEffect(t *testing.T) {
-	testCases := []struct {
-		// input.
-		effect string
-		// expected output.
-		err error
-		// flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Inputs with unsupported Effect.
-		// Test case - 1.
-		{"", errors.New("Policy effect cannot be empty"), false},
-		// Test case - 2.
-		{"DontAllow", errors.New("Unsupported Effect found: ‘DontAllow’, please validate your policy document"), false},
-		// Test case - 3.
-		{"NeverAllow", errors.New("Unsupported Effect found: ‘NeverAllow’, please validate your policy document"), false},
-		// Test case - 4.
-		{"AllowAlways", errors.New("Unsupported Effect found: ‘AllowAlways’, please validate your policy document"), false},
-
-		// Inputs with valid Effect.
-		// Test Case - 5.
-		{"Allow", nil, true},
-		// Test Case - 6.
-		{"Deny", nil, true},
-	}
-	for i, testCase := range testCases {
-		err := isValidEffect(testCase.effect)
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if err != nil && !testCase.shouldPass {
-			if err.Error() != testCase.err.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
-			}
-		}
-	}
-}
-
-// Tests validate Resources validator.
-func TestIsValidResources(t *testing.T) {
-	testCases := []struct {
-		// input.
-		resources []string
-		// expected output.
-		err error
-		// flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Inputs with unsupported Action.
-		// Test case - 1.
-		// Empty Resources.
-		{[]string{}, errors.New("Resource list cannot be empty"), false},
-		// Test case - 2.
-		// A valid resource should have prefix bucketARNPrefix.
-		{[]string{"my-resource"}, errors.New("Unsupported resource style found: ‘my-resource’, please validate your policy document"), false},
-		// Test case - 3.
-		// A Valid resource should have bucket name followed by bucketARNPrefix.
-		{[]string{bucketARNPrefix}, errors.New("Invalid resource style found: ‘arn:aws:s3:::’, please validate your policy document"), false},
-		// Test Case - 4.
-		// Valid resource shouldn't have slash('/') followed by bucketARNPrefix.
-		{[]string{bucketARNPrefix + "/"}, errors.New("Invalid resource style found: ‘arn:aws:s3:::/’, please validate your policy document"), false},
-
-		// Test cases with valid Resources.
-		{[]string{bucketARNPrefix + "my-bucket"}, nil, true},
-		{[]string{bucketARNPrefix + "my-bucket/Asia/*"}, nil, true},
-		{[]string{bucketARNPrefix + "my-bucket/Asia/India/*"}, nil, true},
-	}
-	for i, testCase := range testCases {
-		err := isValidResources(set.CreateStringSet(testCase.resources...))
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if err != nil && !testCase.shouldPass {
-			if err.Error() != testCase.err.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
-			}
-		}
-	}
-}
-
-// Tests validate principals validator.
-func TestIsValidPrincipals(t *testing.T) {
-	testCases := []struct {
-		// input.
-		principals []string
-		// expected output.
-		err error
-		// flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Inputs with unsupported Principals.
-		// Test case - 1.
-		// Empty Principals list.
-		{[]string{}, errors.New("Principal cannot be empty"), false},
-		// Test case - 2.
-		// "*" is the only valid principal.
-		{[]string{"my-principal"}, errors.New("Unsupported principals found: ‘set.StringSet{\"my-principal\":struct {}{}}’, please validate your policy document"), false},
-		// Test case - 3.
-		{[]string{"*", "111122233"}, errors.New("Unsupported principals found: ‘set.StringSet{\"111122233\":struct {}{}}’, please validate your policy document"), false},
-		// Test case - 4.
-		// Test case with valid principal value.
-		{[]string{"*"}, nil, true},
-	}
-	for i, testCase := range testCases {
-		u := policy.User{
-			AWS: set.CreateStringSet(testCase.principals...),
-		}
-		err := isValidPrincipals(u)
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if err != nil && !testCase.shouldPass {
-			if err.Error() != testCase.err.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
-			}
-		}
-	}
-}
-
-// getEmptyConditionMap - returns a function that generates a
-// condition key map for a given key.
-func getEmptyConditionMap(conditionKey string) func() policy.ConditionMap {
-	emptyConditonGenerator := func() policy.ConditionMap {
-		emptyMap := make(policy.ConditionKeyMap)
-		conditions := make(policy.ConditionMap)
-		conditions[conditionKey] = emptyMap
-		return conditions
-	}
-	return emptyConditonGenerator
-}
-
-// Tests validate policy.Statement condition validator.
-func TestIsValidConditions(t *testing.T) {
-	// returns empty conditions map.
-	setEmptyConditions := func() policy.ConditionMap {
-		return make(policy.ConditionMap)
-	}
-
-	// returns map with the "StringEquals" set to empty map.
-	setEmptyStringEquals := getEmptyConditionMap("StringEquals")
-
-	// returns map with the "StringNotEquals" set to empty map.
-	setEmptyStringNotEquals := getEmptyConditionMap("StringNotEquals")
-
-	// returns map with the "StringLike" set to empty map.
-	setEmptyStringLike := getEmptyConditionMap("StringLike")
-
-	// returns map with the "StringNotLike" set to empty map.
-	setEmptyStringNotLike := getEmptyConditionMap("StringNotLike")
-
-	// returns map with the "IpAddress" set to empty map.
-	setEmptyIPAddress := getEmptyConditionMap("IpAddress")
-
-	// returns map with "NotIpAddress" set to empty map.
-	setEmptyNotIPAddress := getEmptyConditionMap("NotIpAddress")
-
-	// Generate conditions.
-	generateConditions := func(key1, key2, value string) policy.ConditionMap {
-		innerMap := make(policy.ConditionKeyMap)
-		innerMap[key2] = set.CreateStringSet(value)
-		conditions := make(policy.ConditionMap)
-		conditions[key1] = innerMap
-		return conditions
-	}
-
-	// generate ambigious conditions.
-	generateAmbigiousConditions := func() policy.ConditionMap {
-		prefixMap := make(policy.ConditionKeyMap)
-		prefixMap["s3:prefix"] = set.CreateStringSet("Asia/")
-		conditions := make(policy.ConditionMap)
-		conditions["StringEquals"] = prefixMap
-		conditions["StringNotEquals"] = prefixMap
-		return conditions
-	}
-
-	// generate valid and non valid type in the condition map.
-	generateValidInvalidConditions := func() policy.ConditionMap {
-		innerMap := make(policy.ConditionKeyMap)
-		innerMap["s3:prefix"] = set.CreateStringSet("Asia/")
-		conditions := make(policy.ConditionMap)
-		conditions["StringEquals"] = innerMap
-		conditions["InvalidType"] = innerMap
-		return conditions
-	}
-
-	// generate valid and invalid keys for valid types in the same condition map.
-	generateValidInvalidConditionKeys := func() policy.ConditionMap {
-		innerMapValid := make(policy.ConditionKeyMap)
-		innerMapValid["s3:prefix"] = set.CreateStringSet("Asia/")
-		innerMapInValid := make(map[string]set.StringSet)
-		innerMapInValid["s3:invalid"] = set.CreateStringSet("Asia/")
-		conditions := make(policy.ConditionMap)
-		conditions["StringEquals"] = innerMapValid
-		conditions["StringEquals"] = innerMapInValid
-		return conditions
-	}
-
-	// List of Conditions used for test cases.
-	testConditions := []policy.ConditionMap{
-		generateConditions("StringValues", "s3:max-keys", "100"),
-		generateConditions("StringEquals", "s3:Object", "100"),
-		generateAmbigiousConditions(),
-		generateValidInvalidConditions(),
-		generateValidInvalidConditionKeys(),
-		setEmptyConditions(),
-		setEmptyStringEquals(),
-		setEmptyStringNotEquals(),
-		setEmptyStringLike(),
-		setEmptyStringNotLike(),
-		setEmptyIPAddress(),
-		setEmptyNotIPAddress(),
-		generateConditions("StringEquals", "s3:prefix", "Asia/"),
-		generateConditions("StringEquals", "s3:max-keys", "100"),
-		generateConditions("StringNotEquals", "s3:prefix", "Asia/"),
-		generateConditions("StringNotEquals", "s3:max-keys", "100"),
-	}
-
-	getObjectActionSet := set.CreateStringSet("s3:GetObject")
-	roBucketActionSet := set.CreateStringSet(readOnlyBucketActions...)
-	maxKeysConditionErr := fmt.Errorf("Unsupported condition key %s for the given actions %s, "+
-		"please validate your policy document", "s3:max-keys", getObjectActionSet)
-	testCases := []struct {
-		inputActions   set.StringSet
-		inputCondition policy.ConditionMap
-		// expected result.
-		expectedErr error
-		// flag indicating whether test should pass.
-		shouldPass bool
-	}{
-		// Malformed conditions.
-		// Test case - 1.
-		// "StringValues" is an invalid type.
-		{roBucketActionSet, testConditions[0], fmt.Errorf("Unsupported condition type 'StringValues', " +
-			"please validate your policy document"), false},
-		// Test case - 2.
-		// "s3:Object" is an invalid key.
-		{roBucketActionSet, testConditions[1], fmt.Errorf("Unsupported condition key " +
-			"'StringEquals', please validate your policy document"), false},
-		// Test case - 3.
-		// Test case with Ambigious conditions set.
-		{roBucketActionSet, testConditions[2], fmt.Errorf("Ambigious condition values for key 's3:prefix', " +
-			"please validate your policy document"), false},
-		// Test case - 4.
-		// Test case with valid and invalid condition types.
-		{roBucketActionSet, testConditions[3], fmt.Errorf("Unsupported condition type 'InvalidType', " +
-			"please validate your policy document"), false},
-		// Test case - 5.
-		// Test case with valid and invalid condition keys.
-		{roBucketActionSet, testConditions[4], fmt.Errorf("Unsupported condition key 'StringEquals', " +
-			"please validate your policy document"), false},
-		// Test cases with valid conditions.
-		// Test case - 6.
-		{roBucketActionSet, testConditions[5], nil, true},
-		// Test case - 7.
-		{roBucketActionSet, testConditions[6], nil, true},
-		// Test case - 8.
-		{roBucketActionSet, testConditions[7], nil, true},
-		// Test case - 9.
-		{roBucketActionSet, testConditions[8], nil, true},
-		// Test case - 10.
-		{roBucketActionSet, testConditions[9], nil, true},
-		// Test case - 11.
-		{roBucketActionSet, testConditions[10], nil, true},
-		// Test case - 12.
-		{roBucketActionSet, testConditions[11], nil, true},
-		// Test case - 13.
-		{roBucketActionSet, testConditions[12], nil, true},
-		// Test case - 11.
-		{roBucketActionSet, testConditions[13], nil, true},
-		// Test case - 12.
-		{roBucketActionSet, testConditions[14], nil, true},
-		// Test case - 13.
-		{getObjectActionSet, testConditions[15], maxKeysConditionErr, false},
-	}
-	for i, testCase := range testCases {
-		actualErr := isValidConditions(testCase.inputActions, testCase.inputCondition)
-		if actualErr != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, actualErr.Error())
-		}
-		if actualErr == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.expectedErr.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if actualErr != nil && !testCase.shouldPass {
-			if actualErr.Error() != testCase.expectedErr.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.expectedErr.Error(), actualErr.Error())
-			}
-		}
-	}
-}
-
-// Tests validate Policy Action and Resource fields.
-func TestCheckbucketPolicyResources(t *testing.T) {
-	// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
-	setValidPrefixActions := func(statements []policy.Statement) []policy.Statement {
-		statements[0].Actions = set.CreateStringSet([]string{"s3:DeleteObject", "s3:PutObject"}...)
-		return statements
-	}
-	// contracting policy statement with recursive resources.
-	// should result in ErrMalformedPolicy
-	setRecurseResource := func(statements []policy.Statement) []policy.Statement {
-		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/Asia/*", "arn:aws:s3:::minio-bucket/Asia/India/*"}...)
-		return statements
-	}
-
-	// constructing policy statement with lexically close characters.
-	// should not result in ErrMalformedPolicy
-	setResourceLexical := func(statements []policy.Statement) []policy.Statement {
-		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/op*", "arn:aws:s3:::minio-bucket/oo*"}...)
-		return statements
-	}
-
-	// List of bucketPolicy used for tests.
-	bucketAccessPolicies := []policy.BucketAccessPolicy{
-		// bucketPolicy - 1.
-		// Contains valid read only policy statement.
-		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
-		// bucketPolicy - 2.
-		// Contains valid read-write only policy statement.
-		{Version: "1.0", Statements: getReadWriteStatement("minio-bucket", "Asia/")},
-		// bucketPolicy - 3.
-		// Contains valid write only policy statement.
-		{Version: "1.0", Statements: getWriteOnlyStatement("minio-bucket", "Asia/India/")},
-		// bucketPolicy - 4.
-		// Contains invalidPrefixActions.
-		// Since resourcePrefix is not to the bucket-name, it return ErrMalformedPolicy.
-		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket-fail", "Asia/India/")},
-		// bucketPolicy - 5.
-		// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
-		// but bucket part of the resource is not equal to the bucket name.
-		// this results in return of ErrMalformedPolicy.
-		{Version: "1.0", Statements: setValidPrefixActions(getWriteOnlyStatement("minio-bucket-fail", "Asia/India/"))},
-		// bucketPolicy - 6.
-		// contracting policy statement with recursive resources.
-		// should result in ErrMalformedPolicy
-		{Version: "1.0", Statements: setRecurseResource(setValidPrefixActions(getWriteOnlyStatement("minio-bucket", "")))},
-		// BucketPolciy - 7.
-		// constructing policy statement with non recursive but
-		// lexically close resources.
-		// should result in ErrNone.
-		{Version: "1.0", Statements: setResourceLexical(setValidPrefixActions(getWriteOnlyStatement("minio-bucket", "oo")))},
-	}
-
-	testCases := []struct {
-		inputPolicy policy.BucketAccessPolicy
-		// expected results.
-		apiErrCode APIErrorCode
-		// Flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Test case - 1.
-		{bucketAccessPolicies[0], ErrNone, true},
-		// Test case - 2.
-		{bucketAccessPolicies[1], ErrNone, true},
-		// Test case - 3.
-		{bucketAccessPolicies[2], ErrNone, true},
-		// Test case - 4.
-		// contains invalidPrefixActions (check bucket-policy-parser.go).
-		// Resource prefix will not be equal to the bucket name in this case.
-		{bucketAccessPolicies[3], ErrMalformedPolicy, false},
-		// Test case - 5.
-		// actions contain invalidPrefixActions (check bucket-policy-parser.go).
-		// Resource prefix bucket part is not equal to the bucket name in this case.
-		{bucketAccessPolicies[4], ErrMalformedPolicy, false},
-		// Test case - 6.
-		// contracting policy statement with recursive resources.
-		// should result in ErrPolicyNesting.
-		{bucketAccessPolicies[5], ErrPolicyNesting, false},
-		// Test case - 7.
-		// constructing policy statement with lexically close
-		// characters.
-		// should result in ErrNone.
-		{bucketAccessPolicies[6], ErrNone, true},
-	}
-	for i, testCase := range testCases {
-		apiErrCode := checkBucketPolicyResources("minio-bucket", testCase.inputPolicy)
-		if apiErrCode != ErrNone && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with Errocode %v", i+1, apiErrCode)
-		}
-		if apiErrCode == ErrNone && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with ErrCode %v, but passed instead", i+1, testCase.apiErrCode)
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if apiErrCode != ErrNone && !testCase.shouldPass {
-			if testCase.apiErrCode != apiErrCode {
-				t.Errorf("Test %d: Expected to fail with error code %v, but instead failed with error code %v", i+1, testCase.apiErrCode, apiErrCode)
-			}
-		}
-	}
-}
-
-// Tests validate parsing of BucketAccessPolicy.
-func TestParseBucketPolicy(t *testing.T) {
-	// set Unsupported Actions.
-	setUnsupportedActions := func(statements []policy.Statement) []policy.Statement {
-		// "s3:DeleteEverything"" is an Unsupported Action.
-		statements[0].Actions = set.CreateStringSet([]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}...)
-		return statements
-	}
-	// set unsupported Effect.
-	setUnsupportedEffect := func(statements []policy.Statement) []policy.Statement {
-		// Effect "Don't allow" is Unsupported.
-		statements[0].Effect = "DontAllow"
-		return statements
-	}
-	// set unsupported principals.
-	setUnsupportedPrincipals := func(statements []policy.Statement) []policy.Statement {
-		// "User1111"" is an Unsupported Principal.
-		statements[0].Principal = policy.User{
-			AWS: set.CreateStringSet([]string{"*", "User1111"}...),
-		}
-		return statements
-	}
-	// set unsupported Resources.
-	setUnsupportedResources := func(statements []policy.Statement) []policy.Statement {
-		// "s3:DeleteEverything"" is an Unsupported Action.
-		statements[0].Resources = set.CreateStringSet([]string{"my-resource"}...)
-		return statements
-	}
-	// List of bucketPolicy used for test cases.
-	bucketAccesPolicies := []policy.BucketAccessPolicy{
-		// bucketPolicy - 0.
-		// bucketPolicy statement empty.
-		{Version: "1.0"},
-		// bucketPolicy - 1.
-		// bucketPolicy version empty.
-		{Version: "", Statements: []policy.Statement{}},
-		// bucketPolicy - 2.
-		// Readonly bucketPolicy.
-		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
-		// bucketPolicy - 3.
-		// Read-Write bucket policy.
-		{Version: "1.0", Statements: getReadWriteStatement("minio-bucket", "Asia/")},
-		// bucketPolicy - 4.
-		// Write only bucket policy.
-		{Version: "1.0", Statements: getWriteOnlyStatement("minio-bucket", "Asia/India/")},
-		// bucketPolicy - 5.
-		// bucketPolicy statement contains unsupported action.
-		{Version: "1.0", Statements: setUnsupportedActions(getReadOnlyStatement("minio-bucket", ""))},
-		// bucketPolicy - 6.
-		// bucketPolicy statement contains unsupported Effect.
-		{Version: "1.0", Statements: setUnsupportedEffect(getReadWriteStatement("minio-bucket", "Asia/"))},
-		// bucketPolicy - 7.
-		// bucketPolicy statement contains unsupported Principal.
-		{Version: "1.0", Statements: setUnsupportedPrincipals(getWriteOnlyStatement("minio-bucket", "Asia/India/"))},
-		// bucketPolicy - 8.
-		// bucketPolicy statement contains unsupported Resource.
-		{Version: "1.0", Statements: setUnsupportedResources(getWriteOnlyStatement("minio-bucket", "Asia/India/"))},
-	}
-
-	testCases := []struct {
-		inputPolicy policy.BucketAccessPolicy
-		// expected results.
-		expectedPolicy policy.BucketAccessPolicy
-		err            error
-		// Flag indicating whether the test should pass.
-		shouldPass bool
-	}{
-		// Test case - 1.
-		// bucketPolicy statement empty.
-		{bucketAccesPolicies[0], policy.BucketAccessPolicy{}, errors.New("Policy statement cannot be empty"), false},
-		// Test case - 2.
-		// bucketPolicy version empty.
-		{bucketAccesPolicies[1], policy.BucketAccessPolicy{}, errors.New("Policy version cannot be empty"), false},
-		// Test case - 3.
-		// Readonly bucketPolicy.
-		{bucketAccesPolicies[2], bucketAccesPolicies[2], nil, true},
-		// Test case - 4.
-		// Read-Write bucket policy.
-		{bucketAccesPolicies[3], bucketAccesPolicies[3], nil, true},
-		// Test case - 5.
-		// Write only bucket policy.
-		{bucketAccesPolicies[4], bucketAccesPolicies[4], nil, true},
-		// Test case - 6.
-		// bucketPolicy statement contains unsupported action.
-		{bucketAccesPolicies[5], bucketAccesPolicies[5], fmt.Errorf("Unsupported actions found: ‘set.StringSet{\"s3:DeleteEverything\":struct {}{}}’, please validate your policy document"), false},
-		// Test case - 7.
-		// bucketPolicy statement contains unsupported Effect.
-		{bucketAccesPolicies[6], bucketAccesPolicies[6], fmt.Errorf("Unsupported Effect found: ‘DontAllow’, please validate your policy document"), false},
-		// Test case - 8.
-		// bucketPolicy statement contains unsupported Principal.
-		{bucketAccesPolicies[7], bucketAccesPolicies[7], fmt.Errorf("Unsupported principals found: ‘set.StringSet{\"User1111\":struct {}{}}’, please validate your policy document"), false},
-		// Test case - 9.
-		// bucketPolicy statement contains unsupported Resource.
-		{bucketAccesPolicies[8], bucketAccesPolicies[8], fmt.Errorf("Unsupported resource style found: ‘my-resource’, please validate your policy document"), false},
-	}
-	for i, testCase := range testCases {
-		var buffer bytes.Buffer
-		encoder := json.NewEncoder(&buffer)
-		err := encoder.Encode(testCase.inputPolicy)
-		if err != nil {
-			t.Fatalf("Test %d: Couldn't Marshal bucket policy %s", i+1, err)
-		}
-
-		var actualAccessPolicy = policy.BucketAccessPolicy{}
-		err = parseBucketPolicy(&buffer, &actualAccessPolicy)
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if err != nil && !testCase.shouldPass {
-			if err.Error() != testCase.err.Error() {
-				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\"", i+1, testCase.err.Error(), err.Error())
-			}
-		}
-		// Test passes as expected, but the output values are verified for correctness here.
-		if err == nil && testCase.shouldPass {
-			if !reflect.DeepEqual(testCase.expectedPolicy, actualAccessPolicy) {
-				t.Errorf("Test %d: The expected statements from resource statement generator doesn't match the actual statements", i+1)
-			}
-		}
-	}
-}
-
-func TestAWSRefererCondition(t *testing.T) {
-	resource := set.CreateStringSet([]string{
-		fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/"+"Asia"+"*"),
-	}...)
-
-	conditionsKeyMap := make(policy.ConditionKeyMap)
-	conditionsKeyMap.Add("aws:Referer",
-		set.CreateStringSet("www.example.com",
-			"http://www.example.com"))
-
-	requestConditionMap := make(policy.ConditionKeyMap)
-	requestConditionMap["referer"] = set.CreateStringSet("www.example.com")
-
-	testCases := []struct {
-		effect       string
-		conditionKey string
-		match        bool
-	}{
-		{
-			effect:       "Allow",
-			conditionKey: "StringLike",
-			match:        true,
-		},
-		{
-			effect:       "Allow",
-			conditionKey: "StringNotLike",
-			match:        false,
-		},
-		{
-			effect:       "Deny",
-			conditionKey: "StringLike",
-			match:        true,
-		},
-		{
-			effect:       "Deny",
-			conditionKey: "StringNotLike",
-			match:        false,
-		},
-	}
-
-	for i, test := range testCases {
-		conditions := make(policy.ConditionMap)
-		conditions[test.conditionKey] = conditionsKeyMap
-
-		allowStatement := policy.Statement{
-			Sid:    "Testing AWS referer condition",
-			Effect: test.effect,
-			Principal: policy.User{
-				AWS: set.CreateStringSet("*"),
-			},
-			Resources:  resource,
-			Conditions: conditions,
-		}
-
-		if result := bucketPolicyConditionMatch(requestConditionMap, allowStatement); result != test.match {
-			t.Errorf("Test %d -  Expected conditons to evaluate to %v but got %v",
-				i+1, test.match, result)
-		}
-	}
-}
-
-func TestAWSSourceIPCondition(t *testing.T) {
-	resource := set.CreateStringSet([]string{
-		fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/"+"Asia"+"*"),
-	}...)
-
-	conditionsKeyMap := make(policy.ConditionKeyMap)
-	// Test both IPv4 and IPv6 addresses.
-	conditionsKeyMap.Add("aws:SourceIp",
-		set.CreateStringSet("54.240.143.0/24",
-			"2001:DB8:1234:5678::/64"))
-
-	requestConditionMap := make(policy.ConditionKeyMap)
-	requestConditionMap["ip"] = set.CreateStringSet("54.240.143.2")
-
-	testCases := []struct {
-		effect       string
-		conditionKey string
-		match        bool
-	}{
-		{
-			effect:       "Allow",
-			conditionKey: "IpAddress",
-			match:        true,
-		},
-		{
-			effect:       "Allow",
-			conditionKey: "NotIpAddress",
-			match:        false,
-		},
-		{
-			effect:       "Deny",
-			conditionKey: "IpAddress",
-			match:        true,
-		},
-		{
-			effect:       "Deny",
-			conditionKey: "NotIpAddress",
-			match:        false,
-		},
-	}
-
-	for i, test := range testCases {
-		conditions := make(policy.ConditionMap)
-		conditions[test.conditionKey] = conditionsKeyMap
-
-		allowStatement := policy.Statement{
-			Sid:    "Testing AWS referer condition",
-			Effect: test.effect,
-			Principal: policy.User{
-				AWS: set.CreateStringSet("*"),
-			},
-			Resources:  resource,
-			Conditions: conditions,
-		}
-
-		if result := bucketPolicyConditionMatch(requestConditionMap, allowStatement); result != test.match {
-			t.Errorf("Test %d -  Expected conditons to evaluate to %v but got %v",
-				i+1, test.match, result)
-		}
-	}
-}

cmd/bucket-policy.go

@@ -1,206 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"io"
-	"reflect"
-	"sync"
-
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/hash"
-)
-
-const (
-	// Static prefix to be used while constructing bucket ARN.
-	// refer to S3 docs for more info.
-	bucketARNPrefix = "arn:aws:s3:::"
-
-	// Bucket policy config name.
-	bucketPolicyConfig = "policy.json"
-)
-
-// Global bucket policies list, policies are enforced on each bucket looking
-// through the policies here.
-type bucketPolicies struct {
-	rwMutex *sync.RWMutex
-
-	// Collection of 'bucket' policies.
-	bucketPolicyConfigs map[string]policy.BucketAccessPolicy
-}
-
-// Fetch bucket policy for a given bucket.
-func (bp bucketPolicies) GetBucketPolicy(bucket string) policy.BucketAccessPolicy {
-	bp.rwMutex.RLock()
-	defer bp.rwMutex.RUnlock()
-	return bp.bucketPolicyConfigs[bucket]
-}
-
-// Set a new bucket policy for a bucket, this operation will overwrite
-// any previous bucket policies for the bucket.
-func (bp *bucketPolicies) SetBucketPolicy(bucket string, newpolicy policy.BucketAccessPolicy) error {
-	bp.rwMutex.Lock()
-	defer bp.rwMutex.Unlock()
-
-	if reflect.DeepEqual(newpolicy, emptyBucketPolicy) {
-		return errInvalidArgument
-	}
-	bp.bucketPolicyConfigs[bucket] = newpolicy
-
-	return nil
-}
-
-// Delete bucket policy from struct for a given bucket.
-func (bp *bucketPolicies) DeleteBucketPolicy(bucket string) error {
-	bp.rwMutex.Lock()
-	defer bp.rwMutex.Unlock()
-	delete(bp.bucketPolicyConfigs, bucket)
-	return nil
-}
-
-// Intialize all bucket policies.
-func initBucketPolicies(objAPI ObjectLayer) (*bucketPolicies, error) {
-	if objAPI == nil {
-		return nil, errInvalidArgument
-	}
-
-	// List buckets to proceed loading all notification configuration.
-	buckets, err := objAPI.ListBuckets(context.Background())
-	if err != nil {
-		return nil, err
-	}
-
-	policies := make(map[string]policy.BucketAccessPolicy)
-	// Loads bucket policy.
-	for _, bucket := range buckets {
-		bp, pErr := ReadBucketPolicy(bucket.Name, objAPI)
-		if pErr != nil {
-			// net.Dial fails for rpc client or any
-			// other unexpected errors during net.Dial.
-			if !IsErrIgnored(pErr, errDiskNotFound) {
-				if !isErrBucketPolicyNotFound(pErr) {
-					return nil, pErr
-				}
-			}
-			// Continue to load other bucket policies if possible.
-			continue
-		}
-		policies[bucket.Name] = bp
-	}
-
-	// Return all bucket policies.
-	return &bucketPolicies{
-		rwMutex:             &sync.RWMutex{},
-		bucketPolicyConfigs: policies,
-	}, nil
-}
-
-// readBucketPolicyJSON - reads bucket policy for an input bucket, returns BucketPolicyNotFound
-// if bucket policy is not found.
-func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader io.Reader, err error) {
-	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
-
-	var buffer bytes.Buffer
-	ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{BucketName: bucket})
-	err = objAPI.GetObject(ctx, minioMetaBucket, policyPath, 0, -1, &buffer, "")
-	if err != nil {
-		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
-			return nil, PolicyNotFound{Bucket: bucket}
-		}
-		return nil, err
-	}
-
-	return &buffer, nil
-}
-
-// ReadBucketPolicy - reads bucket policy for an input bucket, returns BucketPolicyNotFound
-// if bucket policy is not found. This function also parses the bucket policy into an object.
-func ReadBucketPolicy(bucket string, objAPI ObjectLayer) (policy.BucketAccessPolicy, error) {
-	// Read bucket policy JSON.
-	bucketPolicyReader, err := readBucketPolicyJSON(bucket, objAPI)
-	if err != nil {
-		return emptyBucketPolicy, err
-	}
-
-	// Parse the saved policy.
-	var bp policy.BucketAccessPolicy
-	if err = parseBucketPolicy(bucketPolicyReader, &bp); err != nil {
-		return emptyBucketPolicy, err
-
-	}
-	return bp, nil
-}
-
-// removeBucketPolicy - removes any previously written bucket policy. Returns BucketPolicyNotFound
-// if no policies are found.
-func removeBucketPolicy(ctx context.Context, bucket string, objAPI ObjectLayer) error {
-	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
-	err := objAPI.DeleteObject(ctx, minioMetaBucket, policyPath)
-	if err != nil {
-		if _, ok := err.(ObjectNotFound); ok {
-			return BucketPolicyNotFound{Bucket: bucket}
-		}
-		return err
-	}
-	return nil
-}
-
-// writeBucketPolicy - save a bucket policy that is assumed to be validated.
-func writeBucketPolicy(ctx context.Context, bucket string, objAPI ObjectLayer, bpy policy.BucketAccessPolicy) error {
-	buf, err := json.Marshal(bpy)
-
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
-	hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-
-	if _, err = objAPI.PutObject(ctx, minioMetaBucket, policyPath, hashReader, nil); err != nil {
-		return err
-	}
-	return nil
-}
-
-// persistAndNotifyBucketPolicyChange - takes a policyChange argument,
-// persists it to storage, and notify nodes in the cluster about the
-// change. In-memory state is updated in response to the notification.
-func persistAndNotifyBucketPolicyChange(ctx context.Context, bucket string, isRemove bool, bktPolicy policy.BucketAccessPolicy, objAPI ObjectLayer) error {
-	if isRemove {
-		err := removeBucketPolicy(ctx, bucket, objAPI)
-		if err != nil {
-			return err
-		}
-	} else {
-		if reflect.DeepEqual(bktPolicy, emptyBucketPolicy) {
-			return errInvalidArgument
-		}
-		if err := writeBucketPolicy(ctx, bucket, objAPI, bktPolicy); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

cmd/certs.go

@@ -17,10 +17,12 @@
 package cmd
 
 import (
+	"bytes"
+	"crypto"
+	"crypto/ecdsa"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
-	"fmt"
 	"io/ioutil"
 	"os"
 )
@@ -37,24 +39,27 @@ func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err er
 		return nil, err
 	}
 
+	// Trimming leading and tailing white spaces.
+	data = bytes.TrimSpace(data)
+
 	// Parse all certs in the chain.
 	current := data
 	for len(current) > 0 {
 		var pemBlock *pem.Block
 		if pemBlock, current = pem.Decode(current); pemBlock == nil {
-			return nil, fmt.Errorf("Could not read PEM block from file %s", certFile)
+			return nil, uiErrSSLUnexpectedData(nil).Msg("Could not read PEM block from file %s", certFile)
 		}
 
 		var x509Cert *x509.Certificate
 		if x509Cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil {
-			return nil, err
+			return nil, uiErrSSLUnexpectedData(err)
 		}
 
 		x509Certs = append(x509Certs, x509Cert)
 	}
 
 	if len(x509Certs) == 0 {
-		return nil, fmt.Errorf("Empty public certificate file %s", certFile)
+		return nil, uiErrSSLUnexpectedData(nil).Msg("Empty public certificate file %s", certFile)
 	}
 
 	return x509Certs, nil
@@ -105,28 +110,32 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
 	certPEMBlock, err := ioutil.ReadFile(certFile)
 	if err != nil {
-		return tls.Certificate{}, fmt.Errorf("TLS: failed to read cert file: %v", err)
+		return tls.Certificate{}, uiErrSSLUnexpectedError(err)
 	}
 	keyPEMBlock, err := ioutil.ReadFile(keyFile)
 	if err != nil {
-		return tls.Certificate{}, fmt.Errorf("TLS: failed to read private key: %v", err)
+		return tls.Certificate{}, uiErrSSLUnexpectedError(err)
 	}
 	key, rest := pem.Decode(keyPEMBlock)
 	if len(rest) > 0 {
-		return tls.Certificate{}, fmt.Errorf("TLS: private key contains additional data")
+		return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg("The private key contains additional data")
 	}
 	if x509.IsEncryptedPEMBlock(key) {
 		password, ok := os.LookupEnv(TLSPrivateKeyPassword)
 		if !ok {
-			return tls.Certificate{}, fmt.Errorf("TLS: private key is encrypted but no password is present - set env var: %s", TLSPrivateKeyPassword)
+			return tls.Certificate{}, uiErrSSLNoPassword(nil)
 		}
 		decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password))
 		if decErr != nil {
-			return tls.Certificate{}, fmt.Errorf("TLS: failed to decrypt private key: %v", decErr)
+			return tls.Certificate{}, uiErrSSLWrongPassword(decErr)
 		}
 		keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: key.Type, Bytes: decryptedKey})
 	}
-	return tls.X509KeyPair(certPEMBlock, keyPEMBlock)
+	cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
+	if err != nil {
+		return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg(err.Error())
+	}
+	return cert, nil
 }
 
 func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsCert *tls.Certificate, secureConn bool, err error) {
@@ -142,6 +151,16 @@ func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsC
 	if cert, err = loadX509KeyPair(getPublicCertFile(), getPrivateKeyFile()); err != nil {
 		return nil, nil, nil, false, err
 	}
+	// Ensure that the private key is not a P-384 or P-521 EC key.
+	// The Go TLS stack does not provide constant-time implementations of P-384 and P-521.
+	if priv, ok := cert.PrivateKey.(crypto.Signer); ok {
+		if pub, ok := priv.Public().(*ecdsa.PublicKey); ok {
+			if name := pub.Params().Name; name == "P-384" || name == "P-521" { // unfortunately there is no cleaner way to check
+				return nil, nil, nil, false, uiErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", name)
+			}
+		}
+
+	}
 
 	tlsCert = &cert
 

cmd/common-main.go

@@ -44,10 +44,10 @@ func checkUpdate(mode string) {
 func initConfig() {
 	// Config file does not exist, we create it fresh and return upon success.
 	if isFile(getConfigFile()) {
-		logger.FatalIf(migrateConfig(), "Config migration failed.")
-		logger.FatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion)
+		logger.FatalIf(migrateConfig(), "Config migration failed")
+		logger.FatalIf(loadConfig(), "Unable to load the configuration file")
 	} else {
-		logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time.")
+		logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time")
 		logger.Info("Created minio configuration file successfully at " + getConfigDir())
 	}
 }
@@ -71,12 +71,12 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 		// default config directory.
 		configDir = getConfigDir()
 		if configDir == "" {
-			logger.FatalIf(errors.New("missing option"), "config-dir option must be provided.")
+			logger.FatalIf(errors.New("missing option"), "config-dir option must be provided")
 		}
 	}
 
 	if configDir == "" {
-		logger.FatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.")
+		logger.FatalIf(errors.New("empty directory"), "Configuration directory cannot be empty")
 	}
 
 	// Disallow relative paths, figure out absolute paths.
@@ -95,7 +95,9 @@ func handleCommonEnvVars() {
 	secretKey := os.Getenv("MINIO_SECRET_KEY")
 	if accessKey != "" && secretKey != "" {
 		cred, err := auth.CreateCredentials(accessKey, secretKey)
-		logger.FatalIf(err, "Invalid access/secret Key set in environment.")
+		if err != nil {
+			logger.Fatal(uiErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment")
+		}
 
 		// credential Envs are set globally.
 		globalIsEnvCreds = true
@@ -105,7 +107,7 @@ func handleCommonEnvVars() {
 	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
 		browserFlag, err := ParseBrowserFlag(browser)
 		if err != nil {
-			logger.FatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser)
+			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Unable to validate MINIO_BROWSER environment variable")
 		}
 
 		// browser Envs are set globally, this does not represent
@@ -128,18 +130,26 @@ func handleCommonEnvVars() {
 
 	if drives := os.Getenv("MINIO_CACHE_DRIVES"); drives != "" {
 		driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter))
-		logger.FatalIf(err, "Invalid value set in environment variable MINIO_CACHE_DRIVES %s.", drives)
+		if err != nil {
+			logger.Fatal(err, "Unable to parse MINIO_CACHE_DRIVES value (%s)", drives)
+		}
 		globalCacheDrives = driveList
 		globalIsDiskCacheEnabled = true
 	}
+
 	if excludes := os.Getenv("MINIO_CACHE_EXCLUDE"); excludes != "" {
 		excludeList, err := parseCacheExcludes(strings.Split(excludes, cacheEnvDelimiter))
-		logger.FatalIf(err, "Invalid value set in environment variable MINIO_CACHE_EXCLUDE %s.", excludes)
+		if err != nil {
+			logger.Fatal(err, "Unable to parse MINIO_CACHE_EXCLUDE value (`%s`)", excludes)
+		}
 		globalCacheExcludes = excludeList
 	}
+
 	if expiryStr := os.Getenv("MINIO_CACHE_EXPIRY"); expiryStr != "" {
 		expiry, err := strconv.Atoi(expiryStr)
-		logger.FatalIf(err, "Invalid value set in environment variable MINIO_CACHE_EXPIRY %s.", expiryStr)
+		if err != nil {
+			logger.Fatal(uiErrInvalidCacheExpiryValue(err), "Unable to parse MINIO_CACHE_EXPIRY value (`%s`)", expiryStr)
+		}
 		globalCacheExpiry = expiry
 	}
 
@@ -155,25 +165,25 @@ func handleCommonEnvVars() {
 		// Check for environment variables and parse into storageClass struct
 		if ssc := os.Getenv(standardStorageClassEnv); ssc != "" {
 			globalStandardStorageClass, err = parseStorageClass(ssc)
-			logger.FatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv)
+			logger.FatalIf(err, "Invalid value set in environment variable %s", standardStorageClassEnv)
 		}
 
 		if rrsc := os.Getenv(reducedRedundancyStorageClassEnv); rrsc != "" {
 			globalRRStorageClass, err = parseStorageClass(rrsc)
-			logger.FatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv)
+			logger.FatalIf(err, "Invalid value set in environment variable %s", reducedRedundancyStorageClassEnv)
 		}
 
 		// Validation is done after parsing both the storage classes. This is needed because we need one
 		// storage class value to deduce the correct value of the other storage class.
 		if globalRRStorageClass.Scheme != "" {
 			err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
-			logger.FatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv)
+			logger.FatalIf(err, "Invalid value set in environment variable %s", reducedRedundancyStorageClassEnv)
 			globalIsStorageClass = true
 		}
 
 		if globalStandardStorageClass.Scheme != "" {
 			err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
-			logger.FatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv)
+			logger.FatalIf(err, "Invalid value set in environment variable %s", standardStorageClassEnv)
 			globalIsStorageClass = true
 		}
 	}

cmd/config-current.go

@@ -22,6 +22,8 @@ import (
 	"reflect"
 	"sync"
 
+	"github.com/minio/minio/cmd/logger"
+
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
@@ -167,9 +169,12 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 }
 
 func newServerConfig() *serverConfig {
+	cred, err := auth.GetNewCredentials()
+	logger.FatalIf(err, "")
+
 	srvCfg := &serverConfig{
 		Version:    serverConfigVersion,
-		Credential: auth.MustGetNewCredentials(),
+		Credential: cred,
 		Region:     globalMinioDefaultRegion,
 		Browser:    true,
 		StorageClass: storageClassConfig{
@@ -282,7 +287,7 @@ func getValidConfig() (*serverConfig, error) {
 func loadConfig() error {
 	srvCfg, err := getValidConfig()
 	if err != nil {
-		return err
+		return uiErrInvalidConfig(nil).Msg(err.Error())
 	}
 
 	// If env is set override the credentials from config file.

cmd/crossdomain-xml-handler_test.go

@@ -21,14 +21,14 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 )
 
 // Test cross domain xml handler.
 func TestCrossXMLHandler(t *testing.T) {
 	// Server initialization.
-	mux := router.NewRouter().SkipClean(true)
-	handler := setCrossDomainPolicy(mux)
+	router := mux.NewRouter().SkipClean(true)
+	handler := setCrossDomainPolicy(router)
 	srv := httptest.NewServer(handler)
 
 	resp, err := http.Get(srv.URL + crossDomainXMLEntity)

cmd/disk-cache-config.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"encoding/json"
-	"fmt"
 	"path/filepath"
 )
 
@@ -54,7 +53,7 @@ func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {
 func parseCacheDrives(drives []string) ([]string, error) {
 	for _, d := range drives {
 		if !filepath.IsAbs(d) {
-			return nil, fmt.Errorf("cache dir should be absolute path: %s", d)
+			return nil, uiErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
 		}
 	}
 	return drives, nil
@@ -64,10 +63,10 @@ func parseCacheDrives(drives []string) ([]string, error) {
 func parseCacheExcludes(excludes []string) ([]string, error) {
 	for _, e := range excludes {
 		if len(e) == 0 {
-			return nil, fmt.Errorf("cache exclude path (%s) cannot be empty", e)
+			return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude path (%s) cannot be empty", e)
 		}
 		if hasPrefix(e, slashSeparator) {
-			return nil, fmt.Errorf("cache exclude pattern (%s) cannot start with / as prefix", e)
+			return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude pattern (%s) cannot start with / as prefix", e)
 		}
 	}
 	return excludes, nil

cmd/disk-cache-fs.go

@@ -155,11 +155,11 @@ func (cfs *cacheFSObjects) diskAvailable(size int64) bool {
 // purges all content marked trash from the cache.
 func (cfs *cacheFSObjects) purgeTrash() {
 	ticker := time.NewTicker(time.Minute * cacheCleanupInterval)
+	defer ticker.Stop()
+
 	for {
 		select {
 		case <-globalServiceDoneCh:
-			// Stop the timer.
-			ticker.Stop()
 			return
 		case <-ticker.C:
 			trashPath := path.Join(cfs.fsPath, minioMetaBucket, cacheTrashDir)

cmd/disk-cache.go

@@ -289,12 +289,20 @@ func listDirCacheFactory(isLeaf isLeafFunc, treeWalkIgnoredErrs []error, disks [
 	listCacheDirs := func(bucket, prefixDir, prefixEntry string) (dirs []string, err error) {
 		var entries []string
 		for _, disk := range disks {
+			// ignore disk-caches that might be missing/offline
+			if disk == nil {
+				continue
+			}
 			fs := disk.FSObjects
 			entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
+
+			// For any reason disk was deleted or goes offline, continue
+			// and list from other disks if possible.
 			if err != nil {
-				// For any reason disk was deleted or goes offline, continue
-				// and list from other disks if possible.
-				continue
+				if IsErrIgnored(err, treeWalkIgnoredErrs...) {
+					continue
+				}
+				return nil, err
 			}
 
 			// Filter entries that have the prefix prefixEntry.
@@ -349,8 +357,16 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 			return err == nil
 		}
 
+		isLeafDir := func(bucket, object string) bool {
+			fs, err := c.cache.getCacheFS(ctx, bucket, object)
+			if err != nil {
+				return false
+			}
+			return fs.isObjectDir(bucket, object)
+		}
+
 		listDir := listDirCacheFactory(isLeaf, cacheTreeWalkIgnoredErrs, c.cache.cfs)
-		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, endWalkCh)
+		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
 	}
 
 	for i := 0; i < maxKeys; {
@@ -377,16 +393,16 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 			var err error
 			fs, err := c.cache.getCacheFS(ctx, bucket, entry)
 			if err != nil {
-				// Ignore errFileNotFound
-				if err == errFileNotFound {
+				// Ignore errDiskNotFound
+				if err == errDiskNotFound {
 					continue
 				}
 				return result, toObjectErr(err, bucket, prefix)
 			}
 			objInfo, err = fs.getObjectInfo(ctx, bucket, entry)
 			if err != nil {
-				// Ignore errFileNotFound
-				if err == errFileNotFound {
+				// Ignore ObjectNotFound error
+				if _, ok := err.(ObjectNotFound); ok {
 					continue
 				}
 				return result, toObjectErr(err, bucket, prefix)
@@ -409,7 +425,7 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 	result = ListObjectsInfo{IsTruncated: !eof}
 	for _, objInfo := range objInfos {
 		result.NextMarker = objInfo.Name
-		if objInfo.IsDir {
+		if objInfo.IsDir && delimiter == slashSeparator {
 			result.Prefixes = append(result.Prefixes, objInfo.Name)
 			continue
 		}
@@ -469,6 +485,10 @@ func (c cacheObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continu
 func (c cacheObjects) listBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
 	m := make(map[string]string)
 	for _, cache := range c.cache.cfs {
+		// ignore disk-caches that might be missing/offline
+		if cache == nil {
+			continue
+		}
 		entries, err := cache.ListBuckets(ctx)
 
 		if err != nil {
@@ -507,6 +527,10 @@ func (c cacheObjects) GetBucketInfo(ctx context.Context, bucket string) (bucketI
 	bucketInfo, err = getBucketInfoFn(ctx, bucket)
 	if backendDownError(err) {
 		for _, cache := range c.cache.cfs {
+			// ignore disk-caches that might be missing/offline
+			if cache == nil {
+				continue
+			}
 			if bucketInfo, err = cache.GetBucketInfo(ctx, bucket); err == nil {
 				return
 			}
@@ -772,6 +796,10 @@ func (c cacheObjects) DeleteBucket(ctx context.Context, bucket string) (err erro
 	deleteBucketFn := c.DeleteBucketFn
 	var toDel []*cacheFSObjects
 	for _, cfs := range c.cache.cfs {
+		// ignore disk-caches that might be missing/offline
+		if cfs == nil {
+			continue
+		}
 		if _, cerr := cfs.GetBucketInfo(ctx, bucket); cerr == nil {
 			toDel = append(toDel, cfs)
 		}

cmd/encryption-v1.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"crypto/hmac"
 	"crypto/md5"
 	"crypto/rand"
@@ -27,7 +28,9 @@ import (
 	"errors"
 	"io"
 	"net/http"
+	"strconv"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/ioutil"
 	sha256 "github.com/minio/sha256-simd"
 	"github.com/minio/sio"
@@ -739,10 +742,9 @@ func (li *ListPartsInfo) IsEncrypted() bool {
 // DecryptedSize returns the size of the object after decryption in bytes.
 // It returns an error if the object is not encrypted or marked as encrypted
 // but has an invalid size.
-// DecryptedSize panics if the referred object is not encrypted.
 func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	if !o.IsEncrypted() {
-		panic("cannot compute decrypted size of an object which is not encrypted")
+		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}
 	size, err := sio.DecryptedSize(uint64(o.Size))
 	if err != nil {
@@ -757,7 +759,11 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 func (o *ObjectInfo) EncryptedSize() int64 {
 	size, err := sio.EncryptedSize(uint64(o.Size))
 	if err != nil {
-		panic(err) // Since AWS S3 allows parts to be 5GB at most this cannot happen - sio max. size is 256 TB
+		// This cannot happen since AWS S3 allows parts to be 5GB at most
+		// sio max. size is 256 TB
+		reqInfo := (&logger.ReqInfo{}).AppendTags("size", strconv.FormatUint(size, 10))
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.CriticalIf(ctx, err)
 	}
 	return int64(size)
 }

cmd/endpoint-ellipses.go

@@ -70,11 +70,10 @@ func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, e
 	}
 
 	setIndexes = make([][]uint64, len(totalSizes))
-	for i, totalSize := range totalSizes {
+	for _, totalSize := range totalSizes {
 		// Check if totalSize has minimum range upto setSize
 		if totalSize < setSizes[0] {
-			return nil, fmt.Errorf("Invalid inputs (%s). Ellipses range or number of args %d should be atleast divisible by least possible set size %d",
-				args[i], totalSize, setSizes[0])
+			return nil, uiErrInvalidNumberOfErasureEndpoints(nil)
 		}
 	}
 
@@ -103,8 +102,7 @@ func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, e
 
 	// Check whether setSize is with the supported range.
 	if !isValidSetSize(setSize) {
-		return nil, fmt.Errorf("Invalid inputs (%s). Ellipses range or number of args %d should be atleast divisible by least possible set size %d",
-			args, setSize, setSizes[0])
+		return nil, uiErrInvalidNumberOfErasureEndpoints(nil)
 	}
 
 	for i := range totalSizes {
@@ -167,14 +165,14 @@ func parseEndpointSet(args ...string) (ep endpointSet, err error) {
 	for i, arg := range args {
 		patterns, perr := ellipses.FindEllipsesPatterns(arg)
 		if perr != nil {
-			return endpointSet{}, perr
+			return endpointSet{}, uiErrInvalidErasureEndpoints(nil).Msg(perr.Error())
 		}
 		argPatterns[i] = patterns
 	}
 
 	ep.setIndexes, err = getSetIndexes(args, getTotalSizes(argPatterns))
 	if err != nil {
-		return endpointSet{}, err
+		return endpointSet{}, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
 	}
 
 	ep.argPatterns = argPatterns
@@ -223,7 +221,7 @@ func getAllSets(args ...string) ([][]string, error) {
 	for _, sargs := range setArgs {
 		for _, arg := range sargs {
 			if uniqueArgs.Contains(arg) {
-				return nil, fmt.Errorf("Input args (%s) has duplicate ellipses", args)
+				return nil, uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Input args (%s) has duplicate ellipses", args))
 			}
 			uniqueArgs.Add(arg)
 		}

cmd/endpoint.go

@@ -252,14 +252,14 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			return serverAddr, endpoints, setupType, err
 		}
 		if endpoint.Type() != PathEndpointType {
-			return serverAddr, endpoints, setupType, fmt.Errorf("use path style endpoint for FS setup")
+			return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
 		}
 		endpoints = append(endpoints, endpoint)
 		setupType = FSSetupType
 
 		// Check for cross device mounts if any.
 		if err = checkCrossDeviceMounts(endpoints); err != nil {
-			return serverAddr, endpoints, setupType, err
+			return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg(err.Error())
 		}
 		return serverAddr, endpoints, setupType, nil
 	}
@@ -270,12 +270,12 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		var eps EndpointList
 		eps, err = NewEndpointList(iargs...)
 		if err != nil {
-			return serverAddr, endpoints, setupType, err
+			return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
 		}
 
 		// Check for cross device mounts if any.
 		if err = checkCrossDeviceMounts(eps); err != nil {
-			return serverAddr, endpoints, setupType, err
+			return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
 		}
 
 		for _, ep := range eps {
@@ -316,7 +316,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 
 	// No local endpoint found.
 	if localEndpointCount == 0 {
-		return serverAddr, endpoints, setupType, fmt.Errorf("no endpoint found for this host")
+		return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg("no endpoint pointing to the local machine is found")
 	}
 
 	// Check whether same path is not used in endpoints of a host on different port.
@@ -331,8 +331,8 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			hostIPSet, _ := getHostIP4(host)
 			if IPSet, ok := pathIPMap[endpoint.Path]; ok {
 				if !IPSet.Intersection(hostIPSet).IsEmpty() {
-					err = fmt.Errorf("path '%s' can not be served by different port on same address", endpoint.Path)
-					return serverAddr, endpoints, setupType, err
+					return serverAddr, endpoints, setupType,
+						uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path))
 				}
 				pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
 			} else {
@@ -349,8 +349,8 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 				continue
 			}
 			if localPathSet.Contains(endpoint.Path) {
-				err = fmt.Errorf("path '%s' cannot be served by different address on same server", endpoint.Path)
-				return serverAddr, endpoints, setupType, err
+				return serverAddr, endpoints, setupType,
+					uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' cannot be served by different address on same server", endpoint.Path))
 			}
 			localPathSet.Add(endpoint.Path)
 		}
@@ -360,12 +360,11 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 	{
 		if !localPortSet.Contains(serverAddrPort) {
 			if len(localPortSet) > 1 {
-				err = fmt.Errorf("port number in server address must match with one of the port in local endpoints")
-			} else {
-				err = fmt.Errorf("server address and local endpoint have different ports")
+				return serverAddr, endpoints, setupType,
+					uiErrInvalidErasureEndpoints(nil).Msg("port number in server address must match with one of the port in local endpoints")
 			}
-
-			return serverAddr, endpoints, setupType, err
+			return serverAddr, endpoints, setupType,
+				uiErrInvalidErasureEndpoints(nil).Msg("server address and local endpoint have different ports")
 		}
 	}
 
@@ -374,7 +373,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		// If all endpoints have same port number, then this is XL setup using URL style endpoints.
 		if len(localPortSet) == 1 {
 			if len(localServerAddrSet) > 1 {
-				// TODO: Eventhough all endpoints are local, the local host is referred by different IP/name.
+				// TODO: Even though all endpoints are local, the local host is referred by different IP/name.
 				// eg '172.0.0.1', 'localhost' and 'mylocalhostname' point to same local host.
 				//
 				// In this case, we bind to 0.0.0.0 ie to all interfaces.
@@ -388,7 +387,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			return serverAddr, endpoints, setupType, nil
 		}
 
-		// Eventhough all endpoints are local, but those endpoints use different ports.
+		// Even though all endpoints are local, but those endpoints use different ports.
 		// This means it is DistXL setup.
 	} else {
 		// This is DistXL setup.

cmd/endpoint_test.go

@@ -225,8 +225,8 @@ func TestCreateEndpoints(t *testing.T) {
 		{"localhost:10000", [][]string{{"./d1"}}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
 		{"localhost:10000", [][]string{{`\d1`}}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `\d1`}, IsLocal: true}}, FSSetupType, nil},
 		{"localhost:10000", [][]string{{`.\d1`}}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `.\d1`}, IsLocal: true}}, FSSetupType, nil},
-		{":8080", [][]string{{"https://example.org/d1", "https://example.org/d2", "https://example.org/d3", "https://example.org/d4"}}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
-		{":8080", [][]string{{"https://example.org/d1", "https://example.com/d2", "https://example.net:8000/d3", "https://example.edu/d1"}}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
+		{":8080", [][]string{{"https://example.org/d1", "https://example.org/d2", "https://example.org/d3", "https://example.org/d4"}}, "", EndpointList{}, -1, fmt.Errorf("no endpoint pointing to the local machine is found")},
+		{":8080", [][]string{{"https://example.org/d1", "https://example.com/d2", "https://example.net:8000/d3", "https://example.edu/d1"}}, "", EndpointList{}, -1, fmt.Errorf("no endpoint pointing to the local machine is found")},
 		{"localhost:9000", [][]string{{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}}, "", EndpointList{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
 		{"localhost:9000", [][]string{{"https://127.0.0.1:8000/d1", "https://localhost:9001/d2", "https://example.com/d1", "https://example.com/d2"}}, "", EndpointList{}, -1, fmt.Errorf("port number in server address must match with one of the port in local endpoints")},
 		{"localhost:10000", [][]string{{"https://127.0.0.1:8000/d1", "https://localhost:8000/d2", "https://example.com/d1", "https://example.com/d2"}}, "", EndpointList{}, -1, fmt.Errorf("server address and local endpoint have different ports")},

cmd/format-fs.go

@@ -24,7 +24,6 @@ import (
 	"path"
 	"time"
 
-	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
 )
 
@@ -113,7 +112,6 @@ func formatFSMigrateV1ToV2(ctx context.Context, wlk *lock.LockedFile, fsPath str
 	}
 
 	if err = os.MkdirAll(path.Join(fsPath, minioMetaMultipartBucket), 0755); err != nil {
-		logger.LogIf(ctx, err)
 		return err
 	}
 
@@ -147,7 +145,7 @@ func formatFSMigrate(ctx context.Context, wlk *lock.LockedFile, fsPath string) e
 		return err
 	}
 	if version != formatFSVersionV2 {
-		return fmt.Errorf(`%s file: expected FS version: %s, found FS version: %s`, formatConfigFile, formatFSVersionV2, version)
+		return uiErrUnexpectedBackendVersion(fmt.Errorf(`%s file: expected FS version: %s, found FS version: %s`, formatConfigFile, formatFSVersionV2, version))
 	}
 	return nil
 }
@@ -158,7 +156,6 @@ func createFormatFS(ctx context.Context, fsFormatPath string) error {
 	// file stored in minioMetaBucket(.minio.sys) directory.
 	lk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0600)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		return err
 	}
 	// Close the locked file upon return.
@@ -166,7 +163,6 @@ func createFormatFS(ctx context.Context, fsFormatPath string) error {
 
 	fi, err := lk.Stat()
 	if err != nil {
-		logger.LogIf(ctx, err)
 		return err
 	}
 	if fi.Size() != 0 {
@@ -195,7 +191,6 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
 			var fi os.FileInfo
 			fi, err = rlk.Stat()
 			if err != nil {
-				logger.LogIf(ctx, err)
 				return nil, err
 			}
 			isEmpty = fi.Size() == 0
@@ -214,7 +209,6 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
 				continue
 			}
 			if err != nil {
-				logger.LogIf(ctx, err)
 				return nil, err
 			}
 			// After successfully creating format.json try to hold a read-lock on
@@ -222,13 +216,11 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
 			continue
 		}
 		if err != nil {
-			logger.LogIf(ctx, err)
 			return nil, err
 		}
 
 		formatBackend, err := formatMetaGetFormatBackendFS(rlk)
 		if err != nil {
-			logger.LogIf(ctx, err)
 			return nil, err
 		}
 		if formatBackend != formatBackendFS {

cmd/format-xl.go

@@ -288,10 +288,10 @@ func formatXLMigrateV2ToV3(export string) error {
 	return ioutil.WriteFile(formatPath, b, 0644)
 }
 
-// Returns true, if one of the errors is non-nil.
-func hasAnyErrors(errs []error) bool {
+// Returns true, if one of the errors is non-nil and is Unformatted disk.
+func hasAnyErrorsUnformatted(errs []error) bool {
 	for _, err := range errs {
-		if err != nil {
+		if err != nil && err == errUnformattedDisk {
 			return true
 		}
 	}

cmd/fs-v1-helpers.go

@@ -42,14 +42,13 @@ func fsRemoveFile(ctx context.Context, filePath string) (err error) {
 	}
 
 	if err = os.Remove((filePath)); err != nil {
-		fsErr := osErrToFSFileErr(err)
-		if fsErr != errFileNotFound {
+		err = osErrToFSFileErr(err)
+		if err != errFileNotFound {
 			logger.LogIf(ctx, err)
 		}
-		return fsErr
 	}
 
-	return nil
+	return err
 }
 
 // Removes all files and folders at a given path, handles
@@ -95,10 +94,8 @@ func fsRemoveDir(ctx context.Context, dirPath string) (err error) {
 
 	if err = os.Remove((dirPath)); err != nil {
 		if os.IsNotExist(err) {
-			logger.LogIf(ctx, errVolumeNotFound)
 			return errVolumeNotFound
 		} else if isSysErrNotEmpty(err) {
-			logger.LogIf(ctx, errVolumeNotEmpty)
 			return errVolumeNotEmpty
 		}
 		logger.LogIf(ctx, err)
@@ -125,7 +122,6 @@ func fsMkdir(ctx context.Context, dirPath string) (err error) {
 
 	if err = os.Mkdir((dirPath), 0777); err != nil {
 		if os.IsExist(err) {
-			logger.LogIf(ctx, errVolumeExists)
 			return errVolumeExists
 		} else if os.IsPermission(err) {
 			logger.LogIf(ctx, errDiskAccessDenied)
@@ -161,9 +157,8 @@ func fsStat(ctx context.Context, statLoc string) (os.FileInfo, error) {
 		logger.LogIf(ctx, err)
 		return nil, err
 	}
-	fi, err := os.Stat((statLoc))
+	fi, err := os.Stat(statLoc)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		return nil, err
 	}
 
@@ -217,19 +212,37 @@ func osErrToFSFileErr(err error) error {
 func fsStatDir(ctx context.Context, statDir string) (os.FileInfo, error) {
 	fi, err := fsStat(ctx, statDir)
 	if err != nil {
-		return nil, osErrToFSFileErr(err)
+		err = osErrToFSFileErr(err)
+		if err != errFileNotFound {
+			logger.LogIf(ctx, err)
+		}
+		return nil, err
 	}
 	if !fi.IsDir() {
+		logger.LogIf(ctx, errFileAccessDenied)
 		return nil, errFileAccessDenied
 	}
 	return fi, nil
 }
 
+// Returns if the dirPath is a directory.
+func fsIsDir(ctx context.Context, dirPath string) bool {
+	fi, err := fsStat(ctx, dirPath)
+	if err != nil {
+		return false
+	}
+	return fi.IsDir()
+}
+
 // Lookup if file exists, returns file attributes upon success.
 func fsStatFile(ctx context.Context, statFile string) (os.FileInfo, error) {
 	fi, err := fsStat(ctx, statFile)
 	if err != nil {
-		return nil, osErrToFSFileErr(err)
+		err = osErrToFSFileErr(err)
+		if err != errFileNotFound {
+			logger.LogIf(ctx, err)
+		}
+		return nil, err
 	}
 	if fi.IsDir() {
 		logger.LogIf(ctx, errFileAccessDenied)
@@ -238,6 +251,15 @@ func fsStatFile(ctx context.Context, statFile string) (os.FileInfo, error) {
 	return fi, nil
 }
 
+// Returns if the filePath is a regular file.
+func fsIsFile(ctx context.Context, filePath string) bool {
+	fi, err := fsStat(ctx, filePath)
+	if err != nil {
+		return false
+	}
+	return fi.Mode().IsRegular()
+}
+
 // Opens the file at given path, optionally from an offset. Upon success returns
 // a readable stream and the size of the readable stream.
 func fsOpenFile(ctx context.Context, readPath string, offset int64) (io.ReadCloser, int64, error) {
@@ -250,15 +272,18 @@ func fsOpenFile(ctx context.Context, readPath string, offset int64) (io.ReadClos
 		return nil, 0, err
 	}
 
-	fr, err := os.Open((readPath))
+	fr, err := os.Open(readPath)
 	if err != nil {
 		return nil, 0, osErrToFSFileErr(err)
 	}
 
 	// Stat to get the size of the file at path.
-	st, err := os.Stat((readPath))
+	st, err := os.Stat(readPath)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		err = osErrToFSFileErr(err)
+		if err != errFileNotFound {
+			logger.LogIf(ctx, err)
+		}
 		return nil, 0, err
 	}
 
@@ -395,7 +420,9 @@ func fsDeleteFile(ctx context.Context, basePath, deletePath string) error {
 	}
 
 	if err := deleteFile(basePath, deletePath); err != nil {
-		logger.LogIf(ctx, err)
+		if err != errFileNotFound {
+			logger.LogIf(ctx, err)
+		}
 		return err
 	}
 	return nil

cmd/fs-v1-helpers_test.go

@@ -547,3 +547,33 @@ func TestFSRemoveMeta(t *testing.T) {
 		t.Fatalf("`%s` parent directory found though it should have been deleted.", filePath)
 	}
 }
+
+func TestFSIsDir(t *testing.T) {
+	dirPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
+	if err != nil {
+		t.Fatalf("Unable to create tmp directory %s", err)
+	}
+	defer os.RemoveAll(dirPath)
+
+	if !fsIsDir(context.Background(), dirPath) {
+		t.Fatalf("Expected %s to be a directory", dirPath)
+	}
+}
+
+func TestFSIsFile(t *testing.T) {
+	dirPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
+	if err != nil {
+		t.Fatalf("Unable to create tmp directory %s", err)
+	}
+	defer os.RemoveAll(dirPath)
+
+	filePath := pathJoin(dirPath, "tmpfile")
+
+	if err = ioutil.WriteFile(filePath, nil, 0777); err != nil {
+		t.Fatalf("Unable to create file %s", filePath)
+	}
+
+	if !fsIsFile(context.Background(), filePath) {
+		t.Fatalf("Expected %s to be a file", filePath)
+	}
+}

cmd/fs-v1-multipart.go

@@ -714,11 +714,11 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 // blocking and should be run in a go-routine.
 func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh chan struct{}) {
 	ticker := time.NewTicker(cleanupInterval)
+	defer ticker.Stop()
+
 	for {
 		select {
 		case <-doneCh:
-			// Stop the timer.
-			ticker.Stop()
 			return
 		case <-ticker.C:
 			now := time.Now()

cmd/fs-v1-rwpool.go

@@ -168,12 +168,7 @@ func (fsi *fsIOPool) Create(path string) (wlk *lock.LockedFile, err error) {
 	}
 
 	// Creates parent if missing.
-	if err = os.MkdirAll(pathutil.Dir(path), 0777); err != nil {
-		if os.IsPermission(err) {
-			return nil, errFileAccessDenied
-		} else if isSysErrNotDir(err) {
-			return nil, errFileAccessDenied
-		}
+	if err = mkdirAll(pathutil.Dir(path), 0777); err != nil {
 		return nil, err
 	}
 

cmd/fs-v1.go

@@ -19,23 +19,26 @@ package cmd
 import (
 	"context"
 	"encoding/hex"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"path"
-	"reflect"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
-	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/mimedb"
+	"github.com/minio/minio/pkg/policy"
 )
 
+// Default etag is used for pre-existing objects.
+var defaultEtag = "00000000000000000000000000000000-1"
+
 // FSObjects - Implements fs object layer.
 type FSObjects struct {
 	// Path to be exported over S3 API.
@@ -60,9 +63,6 @@ type FSObjects struct {
 
 	// To manage the appendRoutine go-routines
 	nsMutex *nsLockMap
-
-	// Variable represents bucket policies in memory.
-	bucketPolicies *bucketPolicies
 }
 
 // Represents the background append file.
@@ -79,6 +79,7 @@ func initMetaVolumeFS(fsPath, fsUUID string) error {
 	// optimizing all other calls. Create minio meta volume,
 	// if it doesn't exist yet.
 	metaBucketPath := pathJoin(fsPath, minioMetaBucket)
+
 	if err := os.MkdirAll(metaBucketPath, 0777); err != nil {
 		return err
 	}
@@ -102,7 +103,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 
 	var err error
 	if fsPath, err = checkPathValid(fsPath); err != nil {
-		return nil, err
+		return nil, uiErrUnableToWriteInBackend(err)
 	}
 
 	// Assign a new UUID for FS minio mode. Each server instance
@@ -111,7 +112,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 
 	// Initialize meta volume, if volume already exists ignores it.
 	if err = initMetaVolumeFS(fsPath, fsUUID); err != nil {
-		return nil, fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
+		return nil, err
 	}
 
 	// Initialize `format.json`, this function also returns.
@@ -139,15 +140,14 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 	// or cause changes on backend format.
 	fs.fsFormatRlk = rlk
 
-	// Initialize and load bucket policies.
-	fs.bucketPolicies, err = initBucketPolicies(fs)
-	if err != nil {
-		return nil, fmt.Errorf("Unable to load all bucket policies. %s", err)
-	}
-
 	// Initialize notification system.
 	if err = globalNotificationSys.Init(fs); err != nil {
-		return nil, fmt.Errorf("Unable to initialize event notification. %s", err)
+		return nil, uiErrUnableToReadFromBackend(err).Msg("Unable to initialize notification system")
+	}
+
+	// Initialize policy system.
+	if err = globalPolicySys.Init(fs); err != nil {
+		return nil, uiErrUnableToReadFromBackend(err).Msg("Unable to initialize policy system")
 	}
 
 	go fs.cleanupStaleMultipartUploads(ctx, globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh)
@@ -195,13 +195,9 @@ func (fs *FSObjects) ClearLocks(ctx context.Context, info []VolumeLockInfo) erro
 // corresponding valid bucket names on the backend in a platform
 // compatible way for all operating systems.
 func (fs *FSObjects) getBucketDir(ctx context.Context, bucket string) (string, error) {
-	// Verify if bucket is valid.
-	if !IsValidBucketName(bucket) {
-		err := BucketNameInvalid{Bucket: bucket}
-		logger.LogIf(ctx, err)
-		return "", err
+	if bucket == "" || bucket == "." || bucket == ".." {
+		return "", errVolumeNotFound
 	}
-
 	bucketDir := pathJoin(fs.fsPath, bucket)
 	return bucketDir, nil
 }
@@ -226,6 +222,12 @@ func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket, locatio
 		return err
 	}
 	defer bucketLock.Unlock()
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		err := BucketNameInvalid{Bucket: bucket}
+		logger.LogIf(ctx, err)
+		return err
+	}
 	bucketDir, err := fs.getBucketDir(ctx, bucket)
 	if err != nil {
 		return toObjectErr(err, bucket)
@@ -478,7 +480,7 @@ func (fs *FSObjects) getObject(ctx context.Context, bucket, object string, offse
 		}
 	}
 
-	if etag != "" {
+	if etag != "" && etag != defaultEtag {
 		objEtag, perr := fs.getObjectETag(ctx, bucket, object, lock)
 		if perr != nil {
 			return toObjectErr(perr, bucket, object)
@@ -522,21 +524,55 @@ func (fs *FSObjects) getObject(ctx context.Context, bucket, object string, offse
 	return toObjectErr(err, bucket, object)
 }
 
+// Create a new fs.json file, if the existing one is corrupt. Should happen very rarely.
+func (fs *FSObjects) createFsJSON(object, fsMetaPath string) error {
+	fsMeta := newFSMetaV1()
+	fsMeta.Meta = make(map[string]string)
+	fsMeta.Meta["etag"] = GenETag()
+	if objectExt := path.Ext(object); objectExt != "" {
+		if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok {
+			fsMeta.Meta["content-type"] = content.ContentType
+		} else {
+			fsMeta.Meta["content-type"] = "application/octet-stream"
+		}
+	}
+	wlk, werr := fs.rwPool.Create(fsMetaPath)
+	if werr == nil {
+		_, err := fsMeta.WriteTo(wlk)
+		wlk.Close()
+		return err
+	}
+	return werr
+}
+
+// Used to return default etag values when a pre-existing object's meta data is queried.
+func (fs *FSObjects) defaultFsJSON(object string) fsMetaV1 {
+	fsMeta := newFSMetaV1()
+	fsMeta.Meta = make(map[string]string)
+	fsMeta.Meta["etag"] = defaultEtag
+	if objectExt := path.Ext(object); objectExt != "" {
+		if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok {
+			fsMeta.Meta["content-type"] = content.ContentType
+		} else {
+			fsMeta.Meta["content-type"] = "application/octet-stream"
+		}
+	}
+	return fsMeta
+}
+
 // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
 func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
 	fsMeta := fsMetaV1{}
-	fi, err := fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object))
-	if err != nil && err != errFileAccessDenied {
-		return oi, toObjectErr(err, bucket, object)
-	}
-	if fi != nil {
-		// If file found and request was with object ending with "/", consider it
-		// as directory and return object info
-		if hasSuffix(object, slashSeparator) {
-			return fsMeta.ToObjectInfo(bucket, object, fi), nil
+	if hasSuffix(object, slashSeparator) {
+		// Since we support PUT of a "directory" object, we allow HEAD.
+		if !fsIsDir(ctx, pathJoin(fs.fsPath, bucket, object)) {
+			return oi, errFileNotFound
 		}
-		logger.LogIf(ctx, errFileNotFound)
-		return oi, toObjectErr(errFileNotFound, bucket, object)
+		fi, err := fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object))
+		if err != nil {
+			return oi, err
+		}
+		return fsMeta.ToObjectInfo(bucket, object, fi), nil
 	}
 
 	fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)
@@ -546,34 +582,35 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
 	rlk, err := fs.rwPool.Open(fsMetaPath)
 	if err == nil {
 		// Read from fs metadata only if it exists.
-		defer fs.rwPool.Close(fsMetaPath)
-		if _, rerr := fsMeta.ReadFrom(ctx, rlk.LockedFile); rerr != nil {
-			// `fs.json` can be empty due to previously failed
-			// PutObject() transaction, if we arrive at such
-			// a situation we just ignore and continue.
-			if rerr != io.EOF {
-				return oi, toObjectErr(rerr, bucket, object)
-			}
+		_, rerr := fsMeta.ReadFrom(ctx, rlk.LockedFile)
+		fs.rwPool.Close(fsMetaPath)
+		if rerr != nil {
+			return oi, rerr
 		}
 	}
 
+	// Return a default etag and content-type based on the object's extension.
+	if err == errFileNotFound {
+		fsMeta = fs.defaultFsJSON(object)
+	}
+
 	// Ignore if `fs.json` is not available, this is true for pre-existing data.
 	if err != nil && err != errFileNotFound {
 		logger.LogIf(ctx, err)
-		return oi, toObjectErr(err, bucket, object)
+		return oi, err
 	}
 
 	// Stat the file to get file size.
-	fi, err = fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object))
+	fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object))
 	if err != nil {
-		return oi, toObjectErr(err, bucket, object)
+		return oi, err
 	}
 
 	return fsMeta.ToObjectInfo(bucket, object, fi), nil
 }
 
-// GetObjectInfo - reads object metadata and replies back ObjectInfo.
-func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
+// getObjectInfoWithLock - reads object metadata and replies back ObjectInfo.
+func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
 	// Lock the object before reading.
 	objectLock := fs.nsMutex.NewNSLock(bucket, object)
 	if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
@@ -589,9 +626,34 @@ func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string) (
 		return oi, toObjectErr(err, bucket)
 	}
 
+	if strings.HasSuffix(object, slashSeparator) && !fs.isObjectDir(bucket, object) {
+		return oi, errFileNotFound
+	}
+
 	return fs.getObjectInfo(ctx, bucket, object)
 }
 
+// GetObjectInfo - reads object metadata and replies back ObjectInfo.
+func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
+	oi, err := fs.getObjectInfoWithLock(ctx, bucket, object)
+	if err == errCorruptedFormat || err == io.EOF {
+		objectLock := fs.nsMutex.NewNSLock(bucket, object)
+		if err = objectLock.GetLock(globalObjectTimeout); err != nil {
+			return oi, toObjectErr(err, bucket, object)
+		}
+
+		fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)
+		err = fs.createFsJSON(object, fsMetaPath)
+		objectLock.Unlock()
+		if err != nil {
+			return oi, toObjectErr(err, bucket, object)
+		}
+
+		oi, err = fs.getObjectInfoWithLock(ctx, bucket, object)
+	}
+	return oi, toObjectErr(err, bucket, object)
+}
+
 // This function does the following check, suppose
 // object is "a/b/c/d", stat makes sure that objects ""a/b/c""
 // "a/b" and "a" do not exist.
@@ -601,7 +663,7 @@ func (fs *FSObjects) parentDirIsObject(ctx context.Context, bucket, parent strin
 		if p == "." || p == "/" {
 			return false
 		}
-		if _, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, p)); err == nil {
+		if fsIsFile(ctx, pathJoin(fs.fsPath, bucket, p)) {
 			// If there is already a file at prefix "p", return true.
 			return true
 		}
@@ -831,6 +893,17 @@ func (fs *FSObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc {
 	return listDir
 }
 
+// isObjectDir returns true if the specified bucket & prefix exists
+// and the prefix represents an empty directory. An S3 empty directory
+// is also an empty directory in the FS backend.
+func (fs *FSObjects) isObjectDir(bucket, prefix string) bool {
+	entries, err := readDirN(pathJoin(fs.fsPath, bucket, prefix), 1)
+	if err != nil {
+		return false
+	}
+	return len(entries) == 0
+}
+
 // getObjectETag is a helper function, which returns only the md5sum
 // of the file on the disk.
 func (fs *FSObjects) getObjectETag(ctx context.Context, bucket, entry string, lock bool) (string, error) {
@@ -960,8 +1033,15 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
 			// object string does not end with "/".
 			return !hasSuffix(object, slashSeparator)
 		}
+		// Return true if the specified object is an empty directory
+		isLeafDir := func(bucket, object string) bool {
+			if !hasSuffix(object, slashSeparator) {
+				return false
+			}
+			return fs.isObjectDir(bucket, object)
+		}
 		listDir := fs.listDirFactory(isLeaf)
-		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, endWalkCh)
+		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
 	}
 
 	var objInfos []ObjectInfo
@@ -1006,7 +1086,7 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
 	result := ListObjectsInfo{IsTruncated: !eof}
 	for _, objInfo := range objInfos {
 		result.NextMarker = objInfo.Name
-		if objInfo.IsDir {
+		if objInfo.IsDir && delimiter == slashSeparator {
 			result.Prefixes = append(result.Prefixes, objInfo.Name)
 			continue
 		}
@@ -1056,22 +1136,18 @@ func (fs *FSObjects) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error)
 }
 
 // SetBucketPolicy sets policy on bucket
-func (fs *FSObjects) SetBucketPolicy(ctx context.Context, bucket string, policy policy.BucketAccessPolicy) error {
-	return persistAndNotifyBucketPolicyChange(ctx, bucket, false, policy, fs)
+func (fs *FSObjects) SetBucketPolicy(ctx context.Context, bucket string, policy *policy.Policy) error {
+	return savePolicyConfig(fs, bucket, policy)
 }
 
 // GetBucketPolicy will get policy on bucket
-func (fs *FSObjects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
-	policy := fs.bucketPolicies.GetBucketPolicy(bucket)
-	if reflect.DeepEqual(policy, emptyBucketPolicy) {
-		return ReadBucketPolicy(bucket, fs)
-	}
-	return policy, nil
+func (fs *FSObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
+	return GetPolicyConfig(fs, bucket)
 }
 
 // DeleteBucketPolicy deletes all policies on bucket
 func (fs *FSObjects) DeleteBucketPolicy(ctx context.Context, bucket string) error {
-	return persistAndNotifyBucketPolicyChange(ctx, bucket, true, emptyBucketPolicy, fs)
+	return removePolicyConfig(ctx, fs, bucket)
 }
 
 // ListObjectsV2 lists all blobs in bucket filtered by prefix
@@ -1091,19 +1167,6 @@ func (fs *FSObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continua
 	return listObjectsV2Info, err
 }
 
-// RefreshBucketPolicy refreshes cache policy with what's on disk.
-func (fs *FSObjects) RefreshBucketPolicy(ctx context.Context, bucket string) error {
-	policy, err := ReadBucketPolicy(bucket, fs)
-
-	if err != nil {
-		if reflect.DeepEqual(policy, emptyBucketPolicy) {
-			return fs.bucketPolicies.DeleteBucketPolicy(bucket)
-		}
-		return err
-	}
-	return fs.bucketPolicies.SetBucketPolicy(bucket, policy)
-}
-
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
 func (fs *FSObjects) IsNotificationSupported() bool {
 	return true

cmd/fs-v1_test.go

@@ -166,7 +166,15 @@ func TestFSGetBucketInfo(t *testing.T) {
 	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 
-	obj.MakeBucketWithLocation(context.Background(), bucketName, "")
+	err := obj.MakeBucketWithLocation(context.Background(), "a", "")
+	if !isSameType(err, BucketNameInvalid{}) {
+		t.Fatal("BucketNameInvalid error not returned")
+	}
+
+	err = obj.MakeBucketWithLocation(context.Background(), bucketName, "")
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	// Test with valid parameters
 	info, err := fs.GetBucketInfo(context.Background(), bucketName)
@@ -177,10 +185,10 @@ func TestFSGetBucketInfo(t *testing.T) {
 		t.Fatalf("wrong bucket name, expected: %s, found: %s", bucketName, info.Name)
 	}
 
-	// Test with inexistant bucket
+	// Test with non-existent bucket
 	_, err = fs.GetBucketInfo(context.Background(), "a")
-	if !isSameType(err, BucketNameInvalid{}) {
-		t.Fatal("BucketNameInvalid error not returned")
+	if !isSameType(err, BucketNotFound{}) {
+		t.Fatal("BucketNotFound error not returned")
 	}
 
 	// Check for buckets and should get disk not found.
@@ -319,9 +327,10 @@ func TestFSDeleteBucket(t *testing.T) {
 	}
 
 	// Test with an invalid bucket name
-	if err = fs.DeleteBucket(context.Background(), "fo"); !isSameType(err, BucketNameInvalid{}) {
+	if err = fs.DeleteBucket(context.Background(), "fo"); !isSameType(err, BucketNotFound{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
+
 	// Test with an inexistant bucket
 	if err = fs.DeleteBucket(context.Background(), "foobucket"); !isSameType(err, BucketNotFound{}) {
 		t.Fatal("Unexpected error: ", err)

cmd/gateway-common.go

@@ -281,7 +281,7 @@ func ErrorRespToObjectError(err error, params ...string) error {
 	case "BucketNotEmpty":
 		err = BucketNotEmpty{}
 	case "NoSuchBucketPolicy":
-		err = PolicyNotFound{}
+		err = BucketPolicyNotFound{}
 	case "InvalidBucketName":
 		err = BucketNameInvalid{Bucket: bucket}
 	case "NoSuchBucket":

cmd/gateway-main.go

@@ -29,7 +29,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/cli"
-	miniohttp "github.com/minio/minio/cmd/http"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 )
 
@@ -109,9 +109,13 @@ func init() {
 // StartGateway - handler for 'minio gateway <name>'.
 func StartGateway(ctx *cli.Context, gw Gateway) {
 	if gw == nil {
-		logger.FatalIf(errUnexpected, "Gateway implementation not initialized, exiting.")
+		logger.FatalIf(errUnexpected, "Gateway implementation not initialized")
 	}
 
+	// Disable logging until gateway initialization is complete, any
+	// error during initialization will be shown as a fatal message
+	logger.Disable = true
+
 	// Validate if we have access, secret set through environment.
 	gatewayName := gw.Name()
 	if ctx.Args().First() == "help" {
@@ -152,7 +156,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	}
 
 	// Create certs path.
-	logger.FatalIf(createConfigDir(), "Unable to create configuration directories.")
+	logger.FatalIf(createConfigDir(), "Unable to create configuration directories")
 
 	// Initialize gateway config.
 	initConfig()
@@ -167,9 +171,12 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	initNSLock(false) // Enable local namespace lock.
 
-	// Initialize notification system.
+	// Create new notification system.
 	globalNotificationSys, err = NewNotificationSys(globalServerConfig, EndpointList{})
-	logger.FatalIf(err, "Unable to initialize notification system.")
+	logger.FatalIf(err, "Unable to create new notification system")
+
+	// Create new policy system.
+	globalPolicySys = NewPolicySys()
 
 	newObject, err := gw.NewGatewayLayer(globalServerConfig.GetCredential())
 	logger.FatalIf(err, "Unable to initialize gateway layer")
@@ -187,7 +194,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Add API router.
 	registerAPIRouter(router)
 
-	globalHTTPServer = miniohttp.NewServer([]string{gatewayAddr}, registerHandlers(router, globalHandlers...), globalTLSCertificate)
+	globalHTTPServer = xhttp.NewServer([]string{gatewayAddr}, registerHandlers(router, globalHandlers...), globalTLSCertificate)
 
 	// Start server, automatically configures TLS if certs are available.
 	go func() {
@@ -216,5 +223,8 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), gatewayName)
 	}
 
+	// Reenable logging
+	logger.Disable = false
+
 	handleSignals()
 }

cmd/gateway-unsupported.go

@@ -20,10 +20,10 @@ import (
 	"context"
 	"time"
 
-	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/policy"
 )
 
 // GatewayUnsupported list of unsupported call stubs for gateway.
@@ -72,15 +72,15 @@ func (a GatewayUnsupported) CompleteMultipartUpload(ctx context.Context, bucket
 }
 
 // SetBucketPolicy sets policy on bucket
-func (a GatewayUnsupported) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
+func (a GatewayUnsupported) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
 	logger.LogIf(ctx, NotImplemented{})
 	return NotImplemented{}
 }
 
 // GetBucketPolicy will get policy on bucket
-func (a GatewayUnsupported) GetBucketPolicy(ctx context.Context, bucket string) (bal policy.BucketAccessPolicy, err error) {
+func (a GatewayUnsupported) GetBucketPolicy(ctx context.Context, bucket string) (bucketPolicy *policy.Policy, err error) {
 	logger.LogIf(ctx, NotImplemented{})
-	return bal, NotImplemented{}
+	return nil, NotImplemented{}
 }
 
 // DeleteBucketPolicy deletes all policies on bucket

cmd/gateway/azure/gateway-azure.go

@@ -34,10 +34,12 @@ import (
 	"github.com/Azure/azure-sdk-for-go/storage"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/cli"
-	"github.com/minio/minio-go/pkg/policy"
+	miniogopolicy "github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/policy/condition"
 	sha256 "github.com/minio/sha256-simd"
 
 	minio "github.com/minio/minio/cmd"
@@ -49,6 +51,7 @@ const (
 	azureS3MinPartSize         = 5 * humanize.MiByte
 	metadataObjectNameTemplate = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x/azure.json"
 	azureBackend               = "azure"
+	azureMarkerPrefix          = "{minio}"
 )
 
 func init() {
@@ -113,6 +116,12 @@ EXAMPLES:
 	})
 }
 
+// Returns true if marker was returned by Azure, i.e prefixed with
+// {minio}
+func isAzureMarker(marker string) bool {
+	return strings.HasPrefix(marker, azureMarkerPrefix)
+}
+
 // Handler for 'minio gateway azure' command line.
 func azureGatewayMain(ctx *cli.Context) {
 	// Validate gateway arguments.
@@ -344,21 +353,21 @@ func azureToObjectError(err error, params ...string) error {
 	return err
 }
 
-// mustGetAzureUploadID - returns new upload ID which is hex encoded 8 bytes random value.
+// getAzureUploadID - returns new upload ID which is hex encoded 8 bytes random value.
 // this 8 byte restriction is needed because Azure block id has a restriction of length
 // upto 8 bytes.
-func mustGetAzureUploadID() string {
+func getAzureUploadID() (string, error) {
 	var id [8]byte
 
 	n, err := io.ReadFull(rand.Reader, id[:])
 	if err != nil {
-		panic(fmt.Errorf("unable to generate upload ID for azure. %s", err))
+		return "", err
 	}
 	if n != len(id) {
-		panic(fmt.Errorf("insufficient random data (expected: %d, read: %d)", len(id), n))
+		return "", fmt.Errorf("Unexpected random data size. Expected: %d, read: %d)", len(id), n)
 	}
 
-	return hex.EncodeToString(id[:])
+	return hex.EncodeToString(id[:]), nil
 }
 
 // checkAzureUploadID - returns error in case of given string is upload ID.
@@ -431,6 +440,15 @@ func (a *azureObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
 
 // MakeBucketWithLocation - Create a new container on azure backend.
 func (a *azureObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
+	// Verify if bucket (container-name) is valid.
+	// IsValidBucketName has same restrictions as container names mentioned
+	// in azure documentation, so we will simply use the same function here.
+	// Ref - https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata
+	if !minio.IsValidBucketName(bucket) {
+		logger.LogIf(ctx, minio.BucketNameInvalid{Bucket: bucket})
+		return minio.BucketNameInvalid{Bucket: bucket}
+	}
+
 	container := a.client.GetContainerReference(bucket)
 	err := container.Create(&storage.CreateContainerOptions{
 		Access: storage.ContainerAccessTypePrivate,
@@ -441,15 +459,6 @@ func (a *azureObjects) MakeBucketWithLocation(ctx context.Context, bucket, locat
 
 // GetBucketInfo - Get bucket metadata..
 func (a *azureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, e error) {
-	// Verify if bucket (container-name) is valid.
-	// IsValidBucketName has same restrictions as container names mentioned
-	// in azure documentation, so we will simply use the same function here.
-	// Ref - https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata
-	if !minio.IsValidBucketName(bucket) {
-		logger.LogIf(ctx, minio.BucketNameInvalid{Bucket: bucket})
-		return bi, minio.BucketNameInvalid{Bucket: bucket}
-	}
-
 	// Azure does not have an equivalent call, hence use
 	// ListContainers with prefix
 	resp, err := a.client.ListContainers(storage.ListContainersParameters{
@@ -505,14 +514,34 @@ func (a *azureObjects) DeleteBucket(ctx context.Context, bucket string) error {
 
 // ListObjects - lists all blobs on azure with in a container filtered by prefix
 // and marker, uses Azure equivalent ListBlobs.
+// To accommodate S3-compatible applications using
+// ListObjectsV1 to use object keys as markers to control the
+// listing of objects, we use the following encoding scheme to
+// distinguish between Azure continuation tokens and application
+// supplied markers.
+//
+// - NextMarker in ListObjectsV1 response is constructed by
+//   prefixing "{minio}" to the Azure continuation token,
+//   e.g, "{minio}CgRvYmoz"
+//
+// - Application supplied markers are used as-is to list
+//   object keys that appear after it in the lexicographical order.
 func (a *azureObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result minio.ListObjectsInfo, err error) {
 	var objects []minio.ObjectInfo
 	var prefixes []string
+
+	azureListMarker := ""
+	if isAzureMarker(marker) {
+		// If application is using Azure continuation token we should
+		// strip the azureTokenPrefix we added in the previous list response.
+		azureListMarker = strings.TrimPrefix(marker, azureMarkerPrefix)
+	}
+
 	container := a.client.GetContainerReference(bucket)
 	for len(objects) == 0 && len(prefixes) == 0 {
 		resp, err := container.ListBlobs(storage.ListBlobsParameters{
 			Prefix:     prefix,
-			Marker:     marker,
+			Marker:     azureListMarker,
 			Delimiter:  delimiter,
 			MaxResults: uint(maxKeys),
 		})
@@ -521,38 +550,57 @@ func (a *azureObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
 			return result, azureToObjectError(err, bucket, prefix)
 		}
 
-		for _, object := range resp.Blobs {
-			if strings.HasPrefix(object.Name, minio.GatewayMinioSysTmp) {
+		for _, blob := range resp.Blobs {
+			if delimiter == "" && strings.HasPrefix(blob.Name, minio.GatewayMinioSysTmp) {
+				// We filter out minio.GatewayMinioSysTmp entries in the recursive listing.
+				continue
+			}
+			if !isAzureMarker(marker) && blob.Name <= marker {
+				// If the application used ListObjectsV1 style marker then we
+				// skip all the entries till we reach the marker.
 				continue
 			}
 			objects = append(objects, minio.ObjectInfo{
 				Bucket:          bucket,
-				Name:            object.Name,
-				ModTime:         time.Time(object.Properties.LastModified),
-				Size:            object.Properties.ContentLength,
-				ETag:            minio.ToS3ETag(object.Properties.Etag),
-				ContentType:     object.Properties.ContentType,
-				ContentEncoding: object.Properties.ContentEncoding,
+				Name:            blob.Name,
+				ModTime:         time.Time(blob.Properties.LastModified),
+				Size:            blob.Properties.ContentLength,
+				ETag:            minio.ToS3ETag(blob.Properties.Etag),
+				ContentType:     blob.Properties.ContentType,
+				ContentEncoding: blob.Properties.ContentEncoding,
 			})
 		}
 
-		// Remove minio.sys.tmp prefix.
-		for _, prefix := range resp.BlobPrefixes {
-			if prefix != minio.GatewayMinioSysTmp {
-				prefixes = append(prefixes, prefix)
+		for _, blobPrefix := range resp.BlobPrefixes {
+			if blobPrefix == minio.GatewayMinioSysTmp {
+				// We don't do strings.HasPrefix(blob.Name, minio.GatewayMinioSysTmp) here so that
+				// we can use tools like mc to inspect the contents of minio.sys.tmp/
+				// It is OK to allow listing of minio.sys.tmp/ in non-recursive mode as it aids in debugging.
+				continue
 			}
+			if !isAzureMarker(marker) && blobPrefix <= marker {
+				// If the application used ListObjectsV1 style marker then we
+				// skip all the entries till we reach the marker.
+				continue
+			}
+			prefixes = append(prefixes, blobPrefix)
 		}
 
-		marker = resp.NextMarker
-		if resp.NextMarker == "" {
+		azureListMarker = resp.NextMarker
+		if azureListMarker == "" {
+			// Reached end of listing.
 			break
 		}
 	}
 
 	result.Objects = objects
 	result.Prefixes = prefixes
-	result.NextMarker = marker
-	result.IsTruncated = (marker != "")
+	if azureListMarker != "" {
+		// We add the {minio} prefix so that we know in the subsequent request that this
+		// marker is a azure continuation token and not ListObjectV1 marker.
+		result.NextMarker = azureMarkerPrefix + azureListMarker
+		result.IsTruncated = true
+	}
 	return result, nil
 }
 
@@ -668,6 +716,17 @@ func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, des
 		logger.LogIf(ctx, err)
 		return objInfo, azureToObjectError(err, srcBucket, srcObject)
 	}
+	// Azure will copy metadata from the source object when an empty metadata map is provided.
+	// To handle the case where the source object should be copied without its metadata,
+	// the metadata must be removed from the dest. object after the copy completes
+	if len(azureMeta) == 0 && len(destBlob.Metadata) != 0 {
+		destBlob.Metadata = azureMeta
+		err = destBlob.SetMetadata(nil)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return objInfo, azureToObjectError(err, srcBucket, srcObject)
+		}
+	}
 	destBlob.Properties = props
 	err = destBlob.SetProperties(nil)
 	if err != nil {
@@ -724,7 +783,11 @@ func (a *azureObjects) checkUploadIDExists(ctx context.Context, bucketName, obje
 
 // NewMultipartUpload - Use Azure equivalent CreateBlockBlob.
 func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
-	uploadID = mustGetAzureUploadID()
+	uploadID, err = getAzureUploadID()
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return "", err
+	}
 	metadataObject := getAzureMetadataObjectName(object, uploadID)
 
 	var jsonData []byte
@@ -1016,10 +1079,16 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 // storage.ContainerAccessTypePrivate - none in minio terminology
 // As the common denominator for minio and azure is readonly and none, we support
 // these two policies at the bucket level.
-func (a *azureObjects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
-	var policies []minio.BucketAccessPolicy
+func (a *azureObjects) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
+	policyInfo, err := minio.PolicyToBucketAccessPolicy(bucketPolicy)
+	if err != nil {
+		// This should not happen.
+		logger.LogIf(ctx, err)
+		return azureToObjectError(err, bucket)
+	}
 
-	for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket, "") {
+	var policies []minio.BucketAccessPolicy
+	for prefix, policy := range miniogopolicy.GetPolicies(policyInfo.Statements, bucket, "") {
 		policies = append(policies, minio.BucketAccessPolicy{
 			Prefix: prefix,
 			Policy: policy,
@@ -1034,7 +1103,7 @@ func (a *azureObjects) SetBucketPolicy(ctx context.Context, bucket string, polic
 		logger.LogIf(ctx, minio.NotImplemented{})
 		return minio.NotImplemented{}
 	}
-	if policies[0].Policy != policy.BucketPolicyReadOnly {
+	if policies[0].Policy != miniogopolicy.BucketPolicyReadOnly {
 		logger.LogIf(ctx, minio.NotImplemented{})
 		return minio.NotImplemented{}
 	}
@@ -1043,31 +1112,47 @@ func (a *azureObjects) SetBucketPolicy(ctx context.Context, bucket string, polic
 		AccessPolicies: nil,
 	}
 	container := a.client.GetContainerReference(bucket)
-	err := container.SetPermissions(perm, nil)
+	err = container.SetPermissions(perm, nil)
 	logger.LogIf(ctx, err)
 	return azureToObjectError(err, bucket)
 }
 
 // GetBucketPolicy - Get the container ACL and convert it to canonical []bucketAccessPolicy
-func (a *azureObjects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
-	policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
+func (a *azureObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
 	container := a.client.GetContainerReference(bucket)
 	perm, err := container.GetPermissions(nil)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		return policy.BucketAccessPolicy{}, azureToObjectError(err, bucket)
+		return nil, azureToObjectError(err, bucket)
 	}
-	switch perm.AccessType {
-	case storage.ContainerAccessTypePrivate:
-		logger.LogIf(ctx, minio.PolicyNotFound{Bucket: bucket})
-		return policy.BucketAccessPolicy{}, minio.PolicyNotFound{Bucket: bucket}
-	case storage.ContainerAccessTypeContainer:
-		policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "")
-	default:
+
+	if perm.AccessType == storage.ContainerAccessTypePrivate {
+		logger.LogIf(ctx, minio.BucketPolicyNotFound{Bucket: bucket})
+		return nil, minio.BucketPolicyNotFound{Bucket: bucket}
+	} else if perm.AccessType != storage.ContainerAccessTypeContainer {
 		logger.LogIf(ctx, minio.NotImplemented{})
-		return policy.BucketAccessPolicy{}, azureToObjectError(minio.NotImplemented{})
+		return nil, azureToObjectError(minio.NotImplemented{})
 	}
-	return policyInfo, nil
+
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(
+					policy.GetBucketLocationAction,
+					policy.ListBucketAction,
+					policy.GetObjectAction,
+				),
+				policy.NewResourceSet(
+					policy.NewResource(bucket, ""),
+					policy.NewResource(bucket, "*"),
+				),
+				condition.NewFunctions(),
+			),
+		},
+	}, nil
 }
 
 // DeleteBucketPolicy - Set the container ACL to "private"

cmd/gateway/b2/gateway-b2.go

@@ -30,10 +30,12 @@ import (
 
 	b2 "github.com/minio/blazer/base"
 	"github.com/minio/cli"
-	"github.com/minio/minio-go/pkg/policy"
+	miniogopolicy "github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	h2 "github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/policy/condition"
 
 	minio "github.com/minio/minio/cmd"
 )
@@ -722,10 +724,15 @@ func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string,
 // bucketType.AllPublic - bucketTypeReadOnly means that anybody can download the files is the bucket;
 // bucketType.AllPrivate - bucketTypePrivate means that you need an authorization token to download them.
 // Default is AllPrivate for all buckets.
-func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
-	var policies []minio.BucketAccessPolicy
+func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
+	policyInfo, err := minio.PolicyToBucketAccessPolicy(bucketPolicy)
+	if err != nil {
+		// This should not happen.
+		return b2ToObjectError(err, bucket)
+	}
 
-	for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket, "") {
+	var policies []minio.BucketAccessPolicy
+	for prefix, policy := range miniogopolicy.GetPolicies(policyInfo.Statements, bucket, "") {
 		policies = append(policies, minio.BucketAccessPolicy{
 			Prefix: prefix,
 			Policy: policy,
@@ -740,7 +747,7 @@ func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, policyIn
 		logger.LogIf(ctx, minio.NotImplemented{})
 		return minio.NotImplemented{}
 	}
-	if policies[0].Policy != policy.BucketPolicyReadOnly {
+	if policies[0].Policy != miniogopolicy.BucketPolicyReadOnly {
 		logger.LogIf(ctx, minio.NotImplemented{})
 		return minio.NotImplemented{}
 	}
@@ -756,21 +763,39 @@ func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, policyIn
 
 // GetBucketPolicy, returns the current bucketType from B2 backend and convert
 // it into S3 compatible bucket policy info.
-func (l *b2Objects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
-	policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
+func (l *b2Objects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
-		return policyInfo, err
-	}
-	if bkt.Type == bucketTypeReadOnly {
-		policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "")
-		return policyInfo, nil
+		return nil, err
 	}
+
 	// bkt.Type can also be snapshot, but it is only allowed through B2 browser console,
 	// just return back as policy not found for all cases.
 	// CreateBucket always sets the value to allPrivate by default.
-	logger.LogIf(ctx, minio.PolicyNotFound{Bucket: bucket})
-	return policy.BucketAccessPolicy{}, minio.PolicyNotFound{Bucket: bucket}
+	if bkt.Type != bucketTypeReadOnly {
+		logger.LogIf(ctx, minio.BucketPolicyNotFound{Bucket: bucket})
+		return nil, minio.BucketPolicyNotFound{Bucket: bucket}
+	}
+
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(
+					policy.GetBucketLocationAction,
+					policy.ListBucketAction,
+					policy.GetObjectAction,
+				),
+				policy.NewResourceSet(
+					policy.NewResource(bucket, ""),
+					policy.NewResource(bucket, "*"),
+				),
+				condition.NewFunctions(),
+			),
+		},
+	}, nil
 }
 
 // DeleteBucketPolicy - resets the bucketType of bucket on B2 to 'allPrivate'.

cmd/gateway/gcs/gateway-gcs.go

@@ -33,10 +33,12 @@ import (
 	"cloud.google.com/go/storage"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/cli"
-	"github.com/minio/minio-go/pkg/policy"
+	miniogopolicy "github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/policy/condition"
 
 	"google.golang.org/api/googleapi"
 	"google.golang.org/api/iterator"
@@ -570,8 +572,8 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri
 	// supplied markers.
 	//
 	// - NextMarker in ListObjectsV1 response is constructed by
-	//   prefixing "##minio" to the GCS continuation token,
-	//   e.g, "##minioCgRvYmoz"
+	//   prefixing "{minio}" to the GCS continuation token,
+	//   e.g, "{minio}CgRvYmoz"
 	//
 	// - Application supplied markers are used as-is to list
 	//   object keys that appear after it in the lexicographical order.
@@ -839,6 +841,11 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, d
 	object := l.client.Bucket(bucket).Object(key)
 
 	w := object.NewWriter(l.ctx)
+	// Disable "chunked" uploading in GCS client if the size of the data to be uploaded is below
+	// the current chunk-size of the writer. This avoids an unnecessary memory allocation.
+	if data.Size() < int64(w.ChunkSize) {
+		w.ChunkSize = 0
+	}
 	applyMetadataToGCSAttrs(metadata, &w.ObjectAttrs)
 
 	if _, err := io.Copy(w, data); err != nil {
@@ -1127,10 +1134,16 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
 }
 
 // SetBucketPolicy - Set policy on bucket
-func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
-	var policies []minio.BucketAccessPolicy
+func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
+	policyInfo, err := minio.PolicyToBucketAccessPolicy(bucketPolicy)
+	if err != nil {
+		// This should not happen.
+		logger.LogIf(ctx, err)
+		return gcsToObjectError(err, bucket)
+	}
 
-	for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket, "") {
+	var policies []minio.BucketAccessPolicy
+	for prefix, policy := range miniogopolicy.GetPolicies(policyInfo.Statements, bucket, "") {
 		policies = append(policies, minio.BucketAccessPolicy{
 			Prefix: prefix,
 			Policy: policy,
@@ -1149,7 +1162,7 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyI
 	}
 
 	acl := l.client.Bucket(bucket).ACL()
-	if policies[0].Policy == policy.BucketPolicyNone {
+	if policies[0].Policy == miniogopolicy.BucketPolicyNone {
 		if err := acl.Delete(l.ctx, storage.AllUsers); err != nil {
 			logger.LogIf(ctx, err)
 			return gcsToObjectError(err, bucket)
@@ -1159,9 +1172,9 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyI
 
 	var role storage.ACLRole
 	switch policies[0].Policy {
-	case policy.BucketPolicyReadOnly:
+	case miniogopolicy.BucketPolicyReadOnly:
 		role = storage.RoleReader
-	case policy.BucketPolicyWriteOnly:
+	case miniogopolicy.BucketPolicyWriteOnly:
 		role = storage.RoleWriter
 	default:
 		logger.LogIf(ctx, minio.NotImplemented{})
@@ -1177,30 +1190,63 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, policyI
 }
 
 // GetBucketPolicy - Get policy on bucket
-func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
+func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
 	rules, err := l.client.Bucket(bucket).ACL().List(l.ctx)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		return policy.BucketAccessPolicy{}, gcsToObjectError(err, bucket)
+		return nil, gcsToObjectError(err, bucket)
 	}
-	policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
+
+	var readOnly, writeOnly bool
 	for _, r := range rules {
 		if r.Entity != storage.AllUsers || r.Role == storage.RoleOwner {
 			continue
 		}
+
 		switch r.Role {
 		case storage.RoleReader:
-			policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "")
+			readOnly = true
 		case storage.RoleWriter:
-			policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyWriteOnly, bucket, "")
+			writeOnly = true
 		}
 	}
-	// Return NoSuchBucketPolicy error, when policy is not set
-	if len(policyInfo.Statements) == 0 {
-		logger.LogIf(ctx, minio.PolicyNotFound{})
-		return policy.BucketAccessPolicy{}, gcsToObjectError(minio.PolicyNotFound{}, bucket)
+
+	actionSet := policy.NewActionSet()
+	if readOnly {
+		actionSet.Add(policy.GetBucketLocationAction)
+		actionSet.Add(policy.ListBucketAction)
+		actionSet.Add(policy.GetObjectAction)
 	}
-	return policyInfo, nil
+	if writeOnly {
+		actionSet.Add(policy.GetBucketLocationAction)
+		actionSet.Add(policy.ListBucketMultipartUploadsAction)
+		actionSet.Add(policy.AbortMultipartUploadAction)
+		actionSet.Add(policy.DeleteObjectAction)
+		actionSet.Add(policy.ListMultipartUploadPartsAction)
+		actionSet.Add(policy.PutObjectAction)
+	}
+
+	// Return NoSuchBucketPolicy error, when policy is not set
+	if len(actionSet) == 0 {
+		logger.LogIf(ctx, minio.BucketPolicyNotFound{})
+		return nil, gcsToObjectError(minio.BucketPolicyNotFound{}, bucket)
+	}
+
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				actionSet,
+				policy.NewResourceSet(
+					policy.NewResource(bucket, ""),
+					policy.NewResource(bucket, "*"),
+				),
+				condition.NewFunctions(),
+			),
+		},
+	}, nil
 }
 
 // DeleteBucketPolicy - Delete all policies on bucket

cmd/gateway/nas/gateway-nas.go

@@ -20,9 +20,9 @@ import (
 	"context"
 
 	"github.com/minio/cli"
-	"github.com/minio/minio-go/pkg/policy"
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/policy"
 )
 
 const (
@@ -132,6 +132,6 @@ func (l *nasObjects) IsNotificationSupported() bool {
 }
 
 // GetBucketPolicy will get policy on bucket
-func (l *nasObjects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
-	return minio.ReadBucketPolicy(bucket, l)
+func (l *nasObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
+	return minio.GetPolicyConfig(l, bucket)
 }

cmd/gateway/oss/gateway-oss.go

@@ -30,11 +30,13 @@ import (
 	"github.com/dustin/go-humanize"
 
 	"github.com/minio/cli"
-	"github.com/minio/minio-go/pkg/policy"
+	miniogopolicy "github.com/minio/minio-go/pkg/policy"
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/policy/condition"
 )
 
 const (
@@ -368,11 +370,6 @@ func (l *ossObjects) MakeBucketWithLocation(ctx context.Context, bucket, locatio
 
 // ossGeBucketInfo gets bucket metadata.
 func ossGeBucketInfo(ctx context.Context, client *oss.Client, bucket string) (bi minio.BucketInfo, err error) {
-	if !ossIsValidBucketName(bucket) {
-		logger.LogIf(ctx, minio.BucketNameInvalid{Bucket: bucket})
-		return bi, minio.BucketNameInvalid{Bucket: bucket}
-	}
-
 	bgir, err := client.GetBucketInfo(bucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -967,8 +964,14 @@ func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
 // oss.ACLPublicReadWrite: readwrite in minio terminology
 // oss.ACLPublicRead: readonly in minio terminology
 // oss.ACLPrivate: none in minio terminology
-func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
-	bucketPolicies := policy.GetPolicies(policyInfo.Statements, bucket, "")
+func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
+	policyInfo, err := minio.PolicyToBucketAccessPolicy(bucketPolicy)
+	if err != nil {
+		// This should not happen.
+		return ossToObjectError(err, bucket)
+	}
+
+	bucketPolicies := miniogopolicy.GetPolicies(policyInfo.Statements, bucket, "")
 	if len(bucketPolicies) != 1 {
 		logger.LogIf(ctx, minio.NotImplemented{})
 		return minio.NotImplemented{}
@@ -983,11 +986,11 @@ func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, policyI
 
 		var acl oss.ACLType
 		switch bucketPolicy {
-		case policy.BucketPolicyNone:
+		case miniogopolicy.BucketPolicyNone:
 			acl = oss.ACLPrivate
-		case policy.BucketPolicyReadOnly:
+		case miniogopolicy.BucketPolicyReadOnly:
 			acl = oss.ACLPublicRead
-		case policy.BucketPolicyReadWrite:
+		case miniogopolicy.BucketPolicyReadWrite:
 			acl = oss.ACLPublicReadWrite
 		default:
 			logger.LogIf(ctx, minio.NotImplemented{})
@@ -1005,29 +1008,60 @@ func (l *ossObjects) SetBucketPolicy(ctx context.Context, bucket string, policyI
 }
 
 // GetBucketPolicy will get policy on bucket.
-func (l *ossObjects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
+func (l *ossObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
 	result, err := l.Client.GetBucketACL(bucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		return policy.BucketAccessPolicy{}, ossToObjectError(err)
+		return nil, ossToObjectError(err)
 	}
 
-	policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
+	var readOnly, readWrite bool
 	switch result.ACL {
 	case string(oss.ACLPrivate):
 		// By default, all buckets starts with a "private" policy.
-		logger.LogIf(ctx, minio.PolicyNotFound{})
-		return policy.BucketAccessPolicy{}, ossToObjectError(minio.PolicyNotFound{}, bucket)
+		logger.LogIf(ctx, minio.BucketPolicyNotFound{})
+		return nil, ossToObjectError(minio.BucketPolicyNotFound{}, bucket)
 	case string(oss.ACLPublicRead):
-		policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "")
+		readOnly = true
 	case string(oss.ACLPublicReadWrite):
-		policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadWrite, bucket, "")
+		readWrite = true
 	default:
 		logger.LogIf(ctx, minio.NotImplemented{})
-		return policy.BucketAccessPolicy{}, minio.NotImplemented{}
+		return nil, minio.NotImplemented{}
 	}
 
-	return policyInfo, nil
+	actionSet := policy.NewActionSet()
+	if readOnly {
+		actionSet.Add(policy.GetBucketLocationAction)
+		actionSet.Add(policy.ListBucketAction)
+		actionSet.Add(policy.GetObjectAction)
+	}
+	if readWrite {
+		actionSet.Add(policy.GetBucketLocationAction)
+		actionSet.Add(policy.ListBucketAction)
+		actionSet.Add(policy.GetObjectAction)
+		actionSet.Add(policy.ListBucketMultipartUploadsAction)
+		actionSet.Add(policy.AbortMultipartUploadAction)
+		actionSet.Add(policy.DeleteObjectAction)
+		actionSet.Add(policy.ListMultipartUploadPartsAction)
+		actionSet.Add(policy.PutObjectAction)
+	}
+
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				actionSet,
+				policy.NewResourceSet(
+					policy.NewResource(bucket, ""),
+					policy.NewResource(bucket, "*"),
+				),
+				condition.NewFunctions(),
+			),
+		},
+	}, nil
 }
 
 // DeleteBucketPolicy deletes all policies on bucket.

cmd/gateway/s3/gateway-s3.go

@@ -20,14 +20,15 @@ import (
 	"context"
 	"encoding/json"
 	"io"
+	"strings"
 
 	"github.com/minio/cli"
 	miniogo "github.com/minio/minio-go"
-	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio-go/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/policy"
 
 	minio "github.com/minio/minio/cmd"
 )
@@ -172,6 +173,18 @@ func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
 
 // MakeBucket creates a new container on S3 backend.
 func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
+	// Verify if bucket name is valid.
+	// We are using a separate helper function here to validate bucket
+	// names instead of IsValidBucketName() because there is a possibility
+	// that certains users might have buckets which are non-DNS compliant
+	// in us-east-1 and we might severely restrict them by not allowing
+	// access to these buckets.
+	// Ref - http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
+	if s3utils.CheckValidBucketName(bucket) != nil {
+		logger.LogIf(ctx, minio.BucketNameInvalid{Bucket: bucket})
+		return minio.BucketNameInvalid{Bucket: bucket}
+	}
+
 	err := l.Client.MakeBucket(bucket, location)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -182,18 +195,6 @@ func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket, location
 
 // GetBucketInfo gets bucket metadata..
 func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, e error) {
-	// Verify if bucket name is valid.
-	// We are using a separate helper function here to validate bucket
-	// names instead of IsValidBucketName() because there is a possibility
-	// that certains users might have buckets which are non-DNS compliant
-	// in us-east-1 and we might severely restrict them by not allowing
-	// access to these buckets.
-	// Ref - http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
-	if s3utils.CheckValidBucketName(bucket) != nil {
-		logger.LogIf(ctx, minio.BucketNameInvalid{Bucket: bucket})
-		return bi, minio.BucketNameInvalid{Bucket: bucket}
-	}
-
 	buckets, err := l.Client.ListBuckets()
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -428,31 +429,32 @@ func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string,
 }
 
 // SetBucketPolicy sets policy on bucket
-func (l *s3Objects) SetBucketPolicy(ctx context.Context, bucket string, policyInfo policy.BucketAccessPolicy) error {
-	data, err := json.Marshal(&policyInfo)
+func (l *s3Objects) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
+	data, err := json.Marshal(bucketPolicy)
 	if err != nil {
-		return err
+		// This should not happen.
+		logger.LogIf(ctx, err)
+		return minio.ErrorRespToObjectError(err, bucket)
 	}
+
 	if err := l.Client.SetBucketPolicy(bucket, string(data)); err != nil {
 		logger.LogIf(ctx, err)
-		return minio.ErrorRespToObjectError(err, bucket, "")
+		return minio.ErrorRespToObjectError(err, bucket)
 	}
 
 	return nil
 }
 
 // GetBucketPolicy will get policy on bucket
-func (l *s3Objects) GetBucketPolicy(ctx context.Context, bucket string) (policy.BucketAccessPolicy, error) {
+func (l *s3Objects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
 	data, err := l.Client.GetBucketPolicy(bucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		return policy.BucketAccessPolicy{}, minio.ErrorRespToObjectError(err, bucket, "")
+		return nil, minio.ErrorRespToObjectError(err, bucket)
 	}
-	var policyInfo policy.BucketAccessPolicy
-	if err = json.Unmarshal([]byte(data), &policyInfo); err != nil {
-		return policyInfo, err
-	}
-	return policyInfo, nil
+
+	bucketPolicy, err := policy.ParseConfig(strings.NewReader(data), bucket)
+	return bucketPolicy, minio.ErrorRespToObjectError(err, bucket)
 }
 
 // DeleteBucketPolicy deletes all policies on bucket

cmd/gateway/s3/gateway-s3_test.go

@@ -52,7 +52,7 @@ func TestS3ToObjectError(t *testing.T) {
 		},
 		{
 			inputErr:    errResponse("NoSuchBucketPolicy"),
-			expectedErr: minio.PolicyNotFound{},
+			expectedErr: minio.BucketPolicyNotFound{},
 		},
 		{
 			inputErr:    errResponse("NoSuchBucket"),

cmd/generic-handlers.go

@@ -25,6 +25,7 @@ import (
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/sys"
 	"github.com/rs/cors"
 	"golang.org/x/time/rate"
@@ -448,6 +449,9 @@ var notimplementedBucketResourceNames = map[string]bool{
 	"requestPayment": true,
 	"versioning":     true,
 	"website":        true,
+	"inventory":      true,
+	"metrics":        true,
+	"accelerate":     true,
 }
 
 // List of not implemented object queries
@@ -455,6 +459,9 @@ var notimplementedObjectResourceNames = map[string]bool{
 	"torrent": true,
 	"acl":     true,
 	"policy":  true,
+	"tagging": true,
+	"restore": true,
+	"select":  true,
 }
 
 // Resource handler ServeHTTP() wrapper
@@ -602,9 +609,7 @@ func (h pathValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 // cancelled immediately.
 func setRateLimitHandler(h http.Handler) http.Handler {
 	_, maxLimit, err := sys.GetMaxOpenFileLimit()
-	if err != nil {
-		panic(err)
-	}
+	logger.CriticalIf(context.Background(), err)
 	// Burst value is set to 1 to allow only maxOpenFileLimit
 	// requests to happen at once.
 	l := rate.NewLimiter(rate.Limit(maxLimit), 1)

cmd/globals.go

@@ -25,7 +25,7 @@ import (
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/fatih/color"
-	miniohttp "github.com/minio/minio/cmd/http"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/pkg/auth"
 )
 
@@ -76,6 +76,9 @@ const (
 	globalMultipartExpiry = time.Hour * 24 * 14 // 2 weeks.
 	// Cleanup interval when the stale multipart cleanup is initiated.
 	globalMultipartCleanupInterval = time.Hour * 24 // 24 hrs.
+
+	// Limit of location constraint XML for unauthenticted PUT bucket operations.
+	maxLocationConstraintSize = 3 * humanize.MiByte
 )
 
 var (
@@ -120,6 +123,7 @@ var (
 	globalMinioHost = ""
 
 	globalNotificationSys *NotificationSys
+	globalPolicySys       *PolicySys
 
 	// CA root certificates, a nil value means system certs pool will be used
 	globalRootCAs *x509.CertPool
@@ -129,7 +133,7 @@ var (
 
 	globalTLSCertificate *tls.Certificate
 
-	globalHTTPServer        *miniohttp.Server
+	globalHTTPServer        *xhttp.Server
 	globalHTTPServerErrorCh = make(chan error)
 	globalOSSignalCh        = make(chan os.Signal, 1)
 
@@ -188,9 +192,12 @@ var (
 
 // global colors.
 var (
-	colorBold   = color.New(color.Bold).SprintFunc()
-	colorBlue   = color.New(color.FgBlue).SprintfFunc()
-	colorYellow = color.New(color.FgYellow).SprintfFunc()
+	colorBold     = color.New(color.Bold).SprintFunc()
+	colorRed      = color.New(color.FgRed).SprintfFunc()
+	colorBlue     = color.New(color.FgBlue).SprintfFunc()
+	colorYellow   = color.New(color.FgYellow).SprintfFunc()
+	colorBgYellow = color.New(color.BgYellow).SprintfFunc()
+	colorBlack    = color.New(color.FgBlack).SprintfFunc()
 )
 
 // Returns minio global information, as a key value map.

cmd/healthcheck-router.go

@@ -19,7 +19,7 @@ package cmd
 import (
 	"net/http"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 )
 
 const (
@@ -30,10 +30,10 @@ const (
 )
 
 // registerHealthCheckRouter - add handler functions for liveness and readiness routes.
-func registerHealthCheckRouter(mux *router.Router) {
+func registerHealthCheckRouter(router *mux.Router) {
 
 	// Healthcheck router
-	healthRouter := mux.NewRoute().PathPrefix(healthCheckPathPrefix).Subrouter()
+	healthRouter := router.PathPrefix(healthCheckPathPrefix).Subrouter()
 
 	// Liveness handler
 	healthRouter.Methods(http.MethodGet).Path(healthCheckLivenessPath).HandlerFunc(LivenessCheckHandler)

cmd/http/server.go

@@ -38,10 +38,10 @@ const (
 	DefaultTCPKeepAliveTimeout = 10 * time.Second
 
 	// DefaultReadTimeout - default timout to read data from accepted connection.
-	DefaultReadTimeout = 30 * time.Second
+	DefaultReadTimeout = 5 * time.Minute
 
 	// DefaultWriteTimeout - default timout to write data to accepted connection.
-	DefaultWriteTimeout = 30 * time.Second
+	DefaultWriteTimeout = 5 * time.Minute
 
 	// DefaultMaxHeaderBytes - default maximum HTTP header size in bytes.
 	DefaultMaxHeaderBytes = 1 * humanize.MiByte
@@ -172,6 +172,9 @@ var defaultCipherSuites = []uint16{
 	tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
 }
 
+// Go only provides constant-time implementations of Curve25519 and NIST P-256 curve.
+var secureCurves = []tls.CurveID{tls.X25519, tls.CurveP256}
+
 // NewServer - creates new HTTP server using given arguments.
 func NewServer(addrs []string, handler http.Handler, certificate *tls.Certificate) *Server {
 	var tlsConfig *tls.Config
@@ -179,6 +182,7 @@ func NewServer(addrs []string, handler http.Handler, certificate *tls.Certificat
 		tlsConfig = &tls.Config{
 			PreferServerCipherSuites: true,
 			CipherSuites:             defaultCipherSuites,
+			CurvePreferences:         secureCurves,
 			MinVersion:               tls.VersionTLS12,
 			NextProtos:               []string{"http/1.1", "h2"},
 		}

cmd/jwt_test.go

@@ -30,7 +30,10 @@ func testAuthenticate(authType string, t *testing.T) {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
 	defer os.RemoveAll(testPath)
-	cred := auth.MustGetNewCredentials()
+	cred, err := auth.GetNewCredentials()
+	if err != nil {
+		t.Fatalf("Error getting new credentials: %s", err)
+	}
 	globalServerConfig.SetCredential(cred)
 
 	// Define test cases.

cmd/lock-rpc-server.go

@@ -23,7 +23,7 @@ import (
 	"sync"
 	"time"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 	"github.com/minio/dsync"
 	"github.com/minio/minio/cmd/logger"
 )
@@ -69,6 +69,8 @@ func startLockMaintenance(lkSrv *lockServer) {
 	go func(lk *lockServer) {
 		// Initialize a new ticker with a minute between each ticks.
 		ticker := time.NewTicker(lockMaintenanceInterval)
+		// Stop the timer upon service closure and cleanup the go-routine.
+		defer ticker.Stop()
 
 		// Start with random sleep time, so as to avoid "synchronous checks" between servers
 		time.Sleep(time.Duration(rand.Float64() * float64(lockMaintenanceInterval)))
@@ -76,8 +78,6 @@ func startLockMaintenance(lkSrv *lockServer) {
 			// Verifies every minute for locks held more than 2minutes.
 			select {
 			case <-globalServiceDoneCh:
-				// Stop the timer upon service closure and cleanup the go-routine.
-				ticker.Stop()
 				return
 			case <-ticker.C:
 				lk.lockMaintenance(lockValidityCheckInterval)
@@ -87,22 +87,22 @@ func startLockMaintenance(lkSrv *lockServer) {
 }
 
 // Register distributed NS lock handlers.
-func registerDistNSLockRouter(mux *router.Router, endpoints EndpointList) error {
+func registerDistNSLockRouter(router *mux.Router, endpoints EndpointList) error {
 	// Start lock maintenance from all lock servers.
 	startLockMaintenance(globalLockServer)
 
 	// Register initialized lock servers to their respective rpc endpoints.
-	return registerStorageLockers(mux, globalLockServer)
+	return registerStorageLockers(router, globalLockServer)
 }
 
 // registerStorageLockers - register locker rpc handlers for net/rpc library clients
-func registerStorageLockers(mux *router.Router, lkSrv *lockServer) error {
+func registerStorageLockers(router *mux.Router, lkSrv *lockServer) error {
 	lockRPCServer := newRPCServer()
 	if err := lockRPCServer.RegisterName(lockServiceName, lkSrv); err != nil {
 		logger.LogIf(context.Background(), err)
 		return err
 	}
-	lockRouter := mux.PathPrefix(minioReservedBucketPath).Subrouter()
+	lockRouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
 	lockRouter.Path(lockServicePath).Handler(lockRPCServer)
 	return nil
 }

cmd/lockinfo-handlers.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"time"
 )
 
@@ -66,9 +65,3 @@ type OpsLockState struct {
 	Status      statusType `json:"status"` // Status can be Running/Ready/Blocked.
 	Since       time.Time  `json:"since"`  // Time when the lock was initially held.
 }
-
-// listLocksInfo - Fetches locks held on bucket, matching prefix held for longer than duration.
-func listLocksInfo(bucket, prefix string, duration time.Duration) []VolumeLockInfo {
-	locksInfo, _ := newObjectLayerFn().ListLocks(context.Background(), bucket, prefix, duration)
-	return locksInfo
-}

cmd/lockinfo-handlers_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"testing"
@@ -99,7 +100,10 @@ func TestListLocksInfo(t *testing.T) {
 	}
 
 	for i, test := range testCases {
-		actual := listLocksInfo(test.bucket, test.prefix, test.duration)
+		actual, err := objAPI.ListLocks(context.Background(), test.bucket, test.prefix, test.duration)
+		if err != nil {
+			t.Errorf("Test %d - Expected success, got %s", i+1, err)
+		}
 		if len(actual) != test.numLocks {
 			t.Errorf("Test %d - Expected %d locks but observed %d locks",
 				i+1, test.numLocks, len(actual))

cmd/logger/logger.go

@@ -27,15 +27,12 @@ import (
 	"strings"
 	"time"
 
-	"github.com/fatih/color"
 	c "github.com/minio/mc/pkg/console"
+	"golang.org/x/crypto/ssh/terminal"
 )
 
-// global colors.
-var (
-	colorBold = color.New(color.Bold).SprintFunc()
-	colorRed  = color.New(color.FgRed).SprintfFunc()
-)
+// Disable disables all logging, false by default. (used for "go test")
+var Disable = false
 
 var trimStrings []string
 
@@ -44,9 +41,9 @@ type Level int8
 
 // Enumerated level types
 const (
-	Information Level = iota + 1
-	Error
-	Fatal
+	InformationLvl Level = iota + 1
+	ErrorLvl
+	FatalLvl
 )
 
 const loggerTimeFormat string = "15:04:05 MST 01/02/2006"
@@ -61,11 +58,11 @@ var matchingFuncNames = [...]string{
 func (level Level) String() string {
 	var lvlStr string
 	switch level {
-	case Information:
+	case InformationLvl:
 		lvlStr = "INFO"
-	case Error:
+	case ErrorLvl:
 		lvlStr = "ERROR"
-	case Fatal:
+	case FatalLvl:
 		lvlStr = "FATAL"
 	}
 	return lvlStr
@@ -80,6 +77,8 @@ type Console interface {
 
 func consoleLog(console Console, msg string, args ...interface{}) {
 	if jsonFlag {
+		// Strip escape control characters from json message
+		msg = ansiRE.ReplaceAllLiteralString(msg, "")
 		console.json(msg, args...)
 	} else if quiet {
 		console.quiet(msg, args...)
@@ -118,6 +117,8 @@ type logEntry struct {
 // jsonFlag: Display in JSON format, if enabled
 var (
 	quiet, jsonFlag bool
+	// Custom function to format error
+	errorFmtFunc func(string, error, bool) string
 )
 
 // EnableQuiet - turns quiet option on.
@@ -131,11 +132,18 @@ func EnableJSON() {
 	quiet = true
 }
 
+// RegisterUIError registers the specified rendering function. This latter
+// will be called for a pretty rendering of fatal errors.
+func RegisterUIError(f func(string, error, bool) string) {
+	errorFmtFunc = f
+}
+
 // Init sets the trimStrings to possible GOPATHs
 // and GOROOT directories. Also append github.com/minio/minio
 // This is done to clean up the filename, when stack trace is
 // displayed when an error happens.
 func Init(goPath string) {
+
 	var goPathList []string
 	var defaultgoPathList []string
 	// Add all possible GOPATH paths into trimStrings
@@ -176,8 +184,8 @@ func trimTrace(f string) string {
 	return filepath.FromSlash(f)
 }
 
-func getSource() string {
-	pc, file, lineNumber, ok := runtime.Caller(5)
+func getSource(level int) string {
+	pc, file, lineNumber, ok := runtime.Caller(level)
 	if ok {
 		// Clean up the common prefixes
 		file = trimTrace(file)
@@ -219,8 +227,13 @@ func getTrace(traceLevel int) []string {
 	return trace
 }
 
-// LogIf :
+// LogIf prints a detailed error message during
+// the execution of the server.
 func LogIf(ctx context.Context, err error) {
+	if Disable {
+		return
+	}
+
 	if err == nil {
 		return
 	}
@@ -241,15 +254,17 @@ func LogIf(ctx context.Context, err error) {
 		tags[entry.Key] = entry.Val
 	}
 
-	// Get the cause for the Error
-	message := err.Error()
 	// Get full stack trace
 	trace := getTrace(2)
+
+	// Get the cause for the Error
+	message := err.Error()
+
 	// Output the formatted log message at console
 	var output string
 	if jsonFlag {
 		logJSON, err := json.Marshal(&logEntry{
-			Level:      Error.String(),
+			Level:      ErrorLvl.String(),
 			RemoteHost: req.RemoteHost,
 			RequestID:  req.RequestID,
 			UserAgent:  req.UserAgent,
@@ -307,24 +322,48 @@ func LogIf(ctx context.Context, err error) {
 			tagString = "\n       " + tagString
 		}
 
+		var msg = colorFgRed(colorBold(message))
 		output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
 			apiString, timeString, requestID, remoteHost, userAgent,
-			colorRed(colorBold(message)), tagString, strings.Join(trace, "\n"))
+			msg, tagString, strings.Join(trace, "\n"))
 	}
 	fmt.Println(output)
 }
 
-// FatalIf :
-func FatalIf(err error, msg string, data ...interface{}) {
+// CriticalIf :
+// Like LogIf with exit
+// It'll be called for fatal error conditions during run-time
+func CriticalIf(ctx context.Context, err error) {
 	if err != nil {
-		if msg != "" {
-			consoleLog(fatalMessage, msg, data...)
-		} else {
-			consoleLog(fatalMessage, err.Error())
-		}
+		LogIf(ctx, err)
+		os.Exit(1)
 	}
 }
 
+// FatalIf is similar to Fatal() but it ignores passed nil error
+func FatalIf(err error, msg string, data ...interface{}) {
+	if err == nil {
+		return
+	}
+	fatal(err, msg, data...)
+}
+
+// Fatal prints only fatal error message without no stack trace
+// it will be called for input validation failures
+func Fatal(err error, msg string, data ...interface{}) {
+	fatal(err, msg, data...)
+}
+
+func fatal(err error, msg string, data ...interface{}) {
+	var errMsg string
+	if msg != "" {
+		errMsg = errorFmtFunc(fmt.Sprintf(msg, data...), err, jsonFlag)
+	} else {
+		errMsg = err.Error()
+	}
+	consoleLog(fatalMessage, errMsg)
+}
+
 var fatalMessage fatalMsg
 
 type fatalMsg struct {
@@ -332,14 +371,15 @@ type fatalMsg struct {
 
 func (f fatalMsg) json(msg string, args ...interface{}) {
 	logJSON, err := json.Marshal(&logEntry{
-		Level: Fatal.String(),
+		Level: FatalLvl.String(),
 		Time:  time.Now().UTC().Format(time.RFC3339Nano),
-		Trace: &traceEntry{Message: fmt.Sprintf(msg, args...), Source: []string{getSource()}},
+		Trace: &traceEntry{Message: fmt.Sprintf(msg, args...), Source: []string{getSource(6)}},
 	})
 	if err != nil {
 		panic(err)
 	}
 	fmt.Println(string(logJSON))
+
 	os.Exit(1)
 
 }
@@ -348,9 +388,65 @@ func (f fatalMsg) quiet(msg string, args ...interface{}) {
 	f.pretty(msg, args...)
 }
 
+var (
+	logTag       = "ERROR"
+	logBanner    = colorBgRed(colorFgWhite(colorBold(logTag))) + " "
+	emptyBanner  = colorBgRed(strings.Repeat(" ", len(logTag))) + " "
+	minimumWidth = 80
+	bannerWidth  = len(logTag) + 1
+)
+
 func (f fatalMsg) pretty(msg string, args ...interface{}) {
+	// Build the passed error message
 	errMsg := fmt.Sprintf(msg, args...)
-	fmt.Println(colorRed(colorBold("Error: " + errMsg)))
+	// Check terminal width
+	termWidth, _, err := terminal.GetSize(0)
+	if err != nil || termWidth < minimumWidth {
+		termWidth = minimumWidth
+	}
+	// Calculate available widht without the banner
+	width := termWidth - bannerWidth
+
+	tagPrinted := false
+
+	// Print the error message: the following code takes care
+	// of splitting error text and always pretty printing the
+	// red banner along with the error message. Since the error
+	// message itself contains some colored text, we needed
+	// to use some ANSI control escapes to cursor color state
+	// and freely move in the screen.
+	for _, line := range strings.Split(errMsg, "\n") {
+		if len(line) == 0 {
+			// No more text to print, just quit.
+			break
+		}
+
+		for {
+			// Save the attributes of the current cursor helps
+			// us save the text color of the passed error message
+			ansiSaveAttributes()
+			// Print banner with or without the log tag
+			if !tagPrinted {
+				fmt.Print(logBanner)
+				tagPrinted = true
+			} else {
+				fmt.Print(emptyBanner)
+			}
+			// Restore the text color of the error message
+			ansiRestoreAttributes()
+			ansiMoveRight(bannerWidth)
+			// Continue  error message printing
+			if len(line) > width {
+				fmt.Println(line[:width])
+				line = line[width:]
+			} else {
+				fmt.Println(line)
+				break
+			}
+		}
+	}
+
+	// Exit because this is a fatal error message
 	os.Exit(1)
 }
 
@@ -361,7 +457,7 @@ type infoMsg struct {
 
 func (i infoMsg) json(msg string, args ...interface{}) {
 	logJSON, err := json.Marshal(&logEntry{
-		Level:   Information.String(),
+		Level:   InformationLvl.String(),
 		Message: fmt.Sprintf(msg, args...),
 		Time:    time.Now().UTC().Format(time.RFC3339Nano),
 	})

cmd/logger/logonce.go

@@ -0,0 +1,82 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+import (
+	"context"
+	"sync"
+
+	"time"
+)
+
+// Holds a map of recently logged errors.
+type logOnceType struct {
+	IDMap map[interface{}]error
+	sync.Mutex
+}
+
+// One log message per error.
+func (l *logOnceType) logOnceIf(ctx context.Context, err error, id interface{}) {
+	if err == nil {
+		return
+	}
+	l.Lock()
+	shouldLog := false
+	prevErr := l.IDMap[id]
+	if prevErr == nil {
+		l.IDMap[id] = err
+		shouldLog = true
+	} else {
+		if prevErr.Error() != err.Error() {
+			l.IDMap[id] = err
+			shouldLog = true
+		}
+	}
+	l.Unlock()
+
+	if shouldLog {
+		LogIf(ctx, err)
+	}
+}
+
+// Cleanup the map every 30 minutes so that the log message is printed again for the user to notice.
+func (l *logOnceType) cleanupRoutine() {
+	for {
+		select {
+		case <-time.After(time.Minute * 30):
+			l.Lock()
+			l.IDMap = make(map[interface{}]error)
+			l.Unlock()
+		}
+	}
+}
+
+// Returns logOnceType
+func newLogOnceType() *logOnceType {
+	l := &logOnceType{IDMap: make(map[interface{}]error)}
+	go l.cleanupRoutine()
+	return l
+}
+
+var logOnce = newLogOnceType()
+
+// LogOnceIf - Logs notification errors - once per error.
+// id is a unique identifier for related log messages, refer to cmd/notification.go
+// on how it is used.
+func LogOnceIf(ctx context.Context, err error, id interface{}) {
+	logOnce.logOnceIf(ctx, err, id)
+}

cmd/logger/utils.go

@@ -0,0 +1,52 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+import (
+	"fmt"
+	"regexp"
+
+	"github.com/fatih/color"
+)
+
+// Global colors.
+var (
+	colorBold    = color.New(color.Bold).SprintFunc()
+	colorFgRed   = color.New(color.FgRed).SprintfFunc()
+	colorBgRed   = color.New(color.BgRed).SprintfFunc()
+	colorFgWhite = color.New(color.FgWhite).SprintfFunc()
+)
+
+var ansiRE = regexp.MustCompile("(\x1b[^m]*m)")
+
+// Print ANSI Control escape
+func ansiEscape(format string, args ...interface{}) {
+	var Esc = "\x1b"
+	fmt.Printf("%s%s", Esc, fmt.Sprintf(format, args...))
+}
+
+func ansiMoveRight(n int) {
+	ansiEscape("[%dC", n)
+}
+
+func ansiSaveAttributes() {
+	ansiEscape("7")
+}
+
+func ansiRestoreAttributes() {
+	ansiEscape("8")
+}

cmd/metrics-router.go

@@ -17,8 +17,7 @@
 package cmd
 
 import (
-	router "github.com/gorilla/mux"
-	"github.com/prometheus/client_golang/prometheus"
+	"github.com/gorilla/mux"
 )
 
 const (
@@ -26,8 +25,8 @@ const (
 )
 
 // registerMetricsRouter - add handler functions for metrics.
-func registerMetricsRouter(mux *router.Router) {
+func registerMetricsRouter(router *mux.Router) {
 	// metrics router
-	metricsRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
-	metricsRouter.Handle(prometheusMetricsPath, metricsHandler(prometheus.Handler()))
+	metricsRouter := router.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
+	metricsRouter.Handle(prometheusMetricsPath, metricsHandler())
 }

cmd/metrics.go

@@ -20,7 +20,9 @@ import (
 	"context"
 	"net/http"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
 var (
@@ -39,65 +41,36 @@ var (
 		},
 		[]string{"request_type"},
 	)
-	networkSentBytes = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Name: "minio_network_sent_bytes",
-			Help: "Total number of bytes sent by current Minio server instance",
-		},
-	)
-	networkReceivedBytes = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Name: "minio_network_received_bytes",
-			Help: "Total number of bytes received by current Minio server instance",
-		},
-	)
-	onlineMinioDisks = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Name: "minio_online_disks_total",
-			Help: "Total number of online disks for current Minio server instance",
-		},
-	)
-	offlineMinioDisks = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Name: "minio_offline_disks_total",
-			Help: "Total number of offline disks for current Minio server instance",
-		},
-	)
-	serverUptime = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Name: "minio_server_uptime_seconds",
-			Help: "Time elapsed since current Minio server instance started",
-		},
-	)
-	minioStorageTotal = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Name: "minio_disk_storage_bytes",
-			Help: "Total disk storage available to current Minio server instance",
-		},
-	)
-	minioStorageFree = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Name: "minio_disk_storage_free_bytes",
-			Help: "Total free disk storage available to current Minio server instance",
-		},
-	)
 )
 
 func init() {
 	prometheus.MustRegister(httpRequests)
 	prometheus.MustRegister(httpRequestsDuration)
-	prometheus.MustRegister(networkSentBytes)
-	prometheus.MustRegister(networkReceivedBytes)
-	prometheus.MustRegister(onlineMinioDisks)
-	prometheus.MustRegister(offlineMinioDisks)
-	prometheus.MustRegister(serverUptime)
-	prometheus.MustRegister(minioStorageTotal)
-	prometheus.MustRegister(minioStorageFree)
+	prometheus.MustRegister(newMinioCollector())
 }
 
-func updateGeneralMetrics() {
-	// Increment server uptime
-	serverUptime.Set(UTCNow().Sub(globalBootTime).Seconds())
+// newMinioCollector describes the collector
+// and returns reference of minioCollector
+// It creates the Prometheus Description which is used
+// to define metric and  help string
+func newMinioCollector() *minioCollector {
+	return &minioCollector{
+		desc: prometheus.NewDesc("minio_stats", "Statistics exposed by Minio server", nil, nil),
+	}
+}
+
+// minioCollector is the Custom Collector
+type minioCollector struct {
+	desc *prometheus.Desc
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+func (c *minioCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.desc
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 
 	// Fetch disk space info
 	objLayer := newObjectLayerFn()
@@ -105,25 +78,103 @@ func updateGeneralMetrics() {
 	if objLayer == nil {
 		return
 	}
-
-	// Update total/free disk space
 	s := objLayer.StorageInfo(context.Background())
-	minioStorageTotal.Set(float64(s.Total))
-	minioStorageFree.Set(float64(s.Free))
 
-	// Update online/offline disks
-	onlineMinioDisks.Set(float64(s.Backend.OnlineDisks))
-	offlineMinioDisks.Set(float64(s.Backend.OfflineDisks))
+	var totalDisks, offlineDisks int
+	// Setting totalDisks to 1 and offlineDisks to 0 in FS mode
+	if s.Backend.Type == FS {
+		totalDisks = 1
+		offlineDisks = 0
+	} else {
+		offlineDisks = s.Backend.OfflineDisks
+		totalDisks = s.Backend.OfflineDisks + s.Backend.OnlineDisks
+	}
 
-	// Update prometheus metric
-	networkSentBytes.Set(float64(globalConnStats.getTotalOutputBytes()))
-	networkReceivedBytes.Set(float64(globalConnStats.getTotalInputBytes()))
+	// Network Sent/Received Bytes
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName("minio", "network", "sent_bytes_total"),
+			"Total number of bytes sent by current Minio server instance",
+			nil, nil),
+		prometheus.CounterValue,
+		float64(globalConnStats.getTotalOutputBytes()),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName("minio", "network", "received_bytes_total"),
+			"Total number of bytes received by current Minio server instance",
+			nil, nil),
+		prometheus.CounterValue,
+		float64(globalConnStats.getTotalInputBytes()),
+	)
+	// Total/Free Storage Bytes
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName("minio", "disk", "storage_bytes"),
+			"Total disk storage available to current Minio server instance",
+			nil, nil),
+		prometheus.GaugeValue,
+		float64(s.Total),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName("minio", "disk", "storage_free_bytes"),
+			"Total free disk storage available to current Minio server instance",
+			nil, nil),
+		prometheus.GaugeValue,
+		float64(s.Free),
+	)
+	// Set Server Start Time
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName("minio", "server", "start_time_seconds"),
+			"Time when current Minio server instance started",
+			nil, nil),
+		prometheus.GaugeValue,
+		float64(globalBootTime.Unix()),
+	)
+
+	// Minio Total Disk/Offline Disk
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName("minio", "total", "disks"),
+			"Total number of disks for current Minio server instance",
+			nil, nil),
+		prometheus.GaugeValue,
+		float64(totalDisks),
+	)
+	ch <- prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName("minio", "offline", "disks"),
+			"Total number of offline disks for current Minio server instance",
+			nil, nil),
+		prometheus.GaugeValue,
+		float64(offlineDisks),
+	)
 }
 
-func metricsHandler(h http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// Update generic metrics before handling the prometheus scrape request
-		updateGeneralMetrics()
-		prometheus.Handler().ServeHTTP(w, r)
-	})
+func metricsHandler() http.Handler {
+	registry := prometheus.NewRegistry()
+
+	err := registry.Register(httpRequests)
+	logger.LogIf(context.Background(), err)
+
+	err = registry.Register(httpRequestsDuration)
+	logger.LogIf(context.Background(), err)
+
+	err = registry.Register(newMinioCollector())
+	logger.LogIf(context.Background(), err)
+
+	gatherers := prometheus.Gatherers{
+		prometheus.DefaultGatherer,
+		registry,
+	}
+	// Delegate http serving to Prometheus client library, which will call collector.Collect.
+	return promhttp.InstrumentMetricHandler(
+		registry,
+		promhttp.HandlerFor(gatherers,
+			promhttp.HandlerOpts{
+				ErrorHandling: promhttp.ContinueOnError,
+			}),
+	)
 }

cmd/naughty-disk_test.go

@@ -108,11 +108,11 @@ func (d *naughtyDisk) DeleteVol(volume string) (err error) {
 	return d.disk.DeleteVol(volume)
 }
 
-func (d *naughtyDisk) ListDir(volume, path string) (entries []string, err error) {
+func (d *naughtyDisk) ListDir(volume, path string, count int) (entries []string, err error) {
 	if err := d.calcError(); err != nil {
 		return []string{}, err
 	}
-	return d.disk.ListDir(volume, path)
+	return d.disk.ListDir(volume, path, count)
 }
 
 func (d *naughtyDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {

cmd/net.go

@@ -48,7 +48,7 @@ func mustSplitHostPort(hostPort string) (host, port string) {
 func mustGetLocalIP4() (ipList set.StringSet) {
 	ipList = set.NewStringSet()
 	addrs, err := net.InterfaceAddrs()
-	logger.FatalIf(err, "Unable to get IP addresses of this host.")
+	logger.FatalIf(err, "Unable to get IP addresses of this host")
 
 	for _, addr := range addrs {
 		var ip net.IP
@@ -352,15 +352,15 @@ func sameLocalAddrs(addr1, addr2 string) (bool, error) {
 func CheckLocalServerAddr(serverAddr string) error {
 	host, port, err := net.SplitHostPort(serverAddr)
 	if err != nil {
-		return err
+		return uiErrInvalidAddressFlag(err)
 	}
 
 	// Check whether port is a valid port number.
 	p, err := strconv.Atoi(port)
 	if err != nil {
-		return fmt.Errorf("invalid port number")
+		return uiErrInvalidAddressFlag(err).Msg("invalid port number")
 	} else if p < 1 || p > 65535 {
-		return fmt.Errorf("port number must be between 1 to 65535")
+		return uiErrInvalidAddressFlag(nil).Msg("port number must be between 1 to 65535")
 	}
 
 	// 0.0.0.0 is a wildcard address and refers to local network
@@ -372,7 +372,7 @@ func CheckLocalServerAddr(serverAddr string) error {
 			return err
 		}
 		if !isLocalHost {
-			return fmt.Errorf("host in server address should be this server")
+			return uiErrInvalidAddressFlag(nil).Msg("host in server address should be this server")
 		}
 	}
 

cmd/notification.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/json"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"net/url"
 	"path"
@@ -30,6 +31,7 @@ import (
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/policy"
 )
 
 // NotificationSys - notification system.
@@ -57,76 +59,136 @@ func (sys *NotificationSys) GetPeerRPCClient(addr xnet.Host) *PeerRPCClient {
 	return sys.peerRPCClientMap[addr]
 }
 
-// DeleteBucket - calls DeleteBucket RPC call on all peers.
-func (sys *NotificationSys) DeleteBucket(bucketName string) map[xnet.Host]error {
-	errors := make(map[xnet.Host]error)
-	var wg sync.WaitGroup
-	for addr, client := range sys.peerRPCClientMap {
-		wg.Add(1)
-		go func(addr xnet.Host, client *PeerRPCClient) {
-			defer wg.Done()
-			if err := client.DeleteBucket(bucketName); err != nil {
-				errors[addr] = err
-			}
-		}(addr, client)
-	}
-	wg.Wait()
-
-	return errors
+// NotificationPeerErr returns error associated for a remote peer.
+type NotificationPeerErr struct {
+	Host xnet.Host // Remote host on which the rpc call was initiated
+	Err  error     // Error returned by the remote peer for an rpc call
 }
 
-// UpdateBucketPolicy - calls UpdateBucketPolicy RPC call on all peers.
-func (sys *NotificationSys) UpdateBucketPolicy(bucketName string) map[xnet.Host]error {
-	errors := make(map[xnet.Host]error)
-	var wg sync.WaitGroup
-	for addr, client := range sys.peerRPCClientMap {
-		wg.Add(1)
-		go func(addr xnet.Host, client *PeerRPCClient) {
-			defer wg.Done()
-			if err := client.UpdateBucketPolicy(bucketName); err != nil {
-				errors[addr] = err
-			}
-		}(addr, client)
-	}
-	wg.Wait()
+// DeleteBucket - calls DeleteBucket RPC call on all peers.
+func (sys *NotificationSys) DeleteBucket(bucketName string) <-chan NotificationPeerErr {
+	errCh := make(chan NotificationPeerErr)
+	go func() {
+		defer close(errCh)
 
-	return errors
+		var wg sync.WaitGroup
+		for addr, client := range sys.peerRPCClientMap {
+			wg.Add(1)
+			go func(addr xnet.Host, client *PeerRPCClient) {
+				defer wg.Done()
+				if err := client.DeleteBucket(bucketName); err != nil {
+					errCh <- NotificationPeerErr{
+						Host: addr,
+						Err:  err,
+					}
+				}
+			}(addr, client)
+		}
+		wg.Wait()
+	}()
+
+	return errCh
+}
+
+// SetBucketPolicy - calls SetBucketPolicy RPC call on all peers.
+func (sys *NotificationSys) SetBucketPolicy(bucketName string, bucketPolicy *policy.Policy) <-chan NotificationPeerErr {
+	errCh := make(chan NotificationPeerErr)
+	go func() {
+		defer close(errCh)
+
+		var wg sync.WaitGroup
+		for addr, client := range sys.peerRPCClientMap {
+			wg.Add(1)
+			go func(addr xnet.Host, client *PeerRPCClient) {
+				defer wg.Done()
+				if err := client.SetBucketPolicy(bucketName, bucketPolicy); err != nil {
+					errCh <- NotificationPeerErr{
+						Host: addr,
+						Err:  err,
+					}
+				}
+			}(addr, client)
+		}
+		wg.Wait()
+	}()
+
+	return errCh
+}
+
+// RemoveBucketPolicy - calls RemoveBucketPolicy RPC call on all peers.
+func (sys *NotificationSys) RemoveBucketPolicy(bucketName string) <-chan NotificationPeerErr {
+	errCh := make(chan NotificationPeerErr)
+	go func() {
+		defer close(errCh)
+
+		var wg sync.WaitGroup
+		for addr, client := range sys.peerRPCClientMap {
+			wg.Add(1)
+			go func(addr xnet.Host, client *PeerRPCClient) {
+				defer wg.Done()
+				if err := client.RemoveBucketPolicy(bucketName); err != nil {
+					errCh <- NotificationPeerErr{
+						Host: addr,
+						Err:  err,
+					}
+				}
+			}(addr, client)
+		}
+		wg.Wait()
+	}()
+
+	return errCh
 }
 
 // PutBucketNotification - calls PutBucketNotification RPC call on all peers.
-func (sys *NotificationSys) PutBucketNotification(bucketName string, rulesMap event.RulesMap) map[xnet.Host]error {
-	errors := make(map[xnet.Host]error)
-	var wg sync.WaitGroup
-	for addr, client := range sys.peerRPCClientMap {
-		wg.Add(1)
-		go func(addr xnet.Host, client *PeerRPCClient, rulesMap event.RulesMap) {
-			defer wg.Done()
-			if err := client.PutBucketNotification(bucketName, rulesMap); err != nil {
-				errors[addr] = err
-			}
-		}(addr, client, rulesMap.Clone())
-	}
-	wg.Wait()
+func (sys *NotificationSys) PutBucketNotification(bucketName string, rulesMap event.RulesMap) <-chan NotificationPeerErr {
+	errCh := make(chan NotificationPeerErr)
+	go func() {
+		defer close(errCh)
 
-	return errors
+		var wg sync.WaitGroup
+		for addr, client := range sys.peerRPCClientMap {
+			wg.Add(1)
+			go func(addr xnet.Host, client *PeerRPCClient, rulesMap event.RulesMap) {
+				defer wg.Done()
+				if err := client.PutBucketNotification(bucketName, rulesMap); err != nil {
+					errCh <- NotificationPeerErr{
+						Host: addr,
+						Err:  err,
+					}
+				}
+			}(addr, client, rulesMap.Clone())
+		}
+		wg.Wait()
+	}()
+
+	return errCh
 }
 
 // ListenBucketNotification - calls ListenBucketNotification RPC call on all peers.
-func (sys *NotificationSys) ListenBucketNotification(bucketName string, eventNames []event.Name, pattern string, targetID event.TargetID, localPeer xnet.Host) map[xnet.Host]error {
-	errors := make(map[xnet.Host]error)
-	var wg sync.WaitGroup
-	for addr, client := range sys.peerRPCClientMap {
-		wg.Add(1)
-		go func(addr xnet.Host, client *PeerRPCClient) {
-			defer wg.Done()
-			if err := client.ListenBucketNotification(bucketName, eventNames, pattern, targetID, localPeer); err != nil {
-				errors[addr] = err
-			}
-		}(addr, client)
-	}
-	wg.Wait()
+func (sys *NotificationSys) ListenBucketNotification(bucketName string, eventNames []event.Name, pattern string,
+	targetID event.TargetID, localPeer xnet.Host) <-chan NotificationPeerErr {
+	errCh := make(chan NotificationPeerErr)
+	go func() {
+		defer close(errCh)
 
-	return errors
+		var wg sync.WaitGroup
+		for addr, client := range sys.peerRPCClientMap {
+			wg.Add(1)
+			go func(addr xnet.Host, client *PeerRPCClient) {
+				defer wg.Done()
+				if err := client.ListenBucketNotification(bucketName, eventNames, pattern, targetID, localPeer); err != nil {
+					errCh <- NotificationPeerErr{
+						Host: addr,
+						Err:  err,
+					}
+				}
+			}(addr, client)
+		}
+		wg.Wait()
+	}()
+
+	return errCh
 }
 
 // AddRemoteTarget - adds event rules map, HTTP/PeerRPC client target to bucket name.
@@ -181,14 +243,15 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 	}
 	defer objLock.Unlock()
 
-	reader, err := readConfig(ctx, objAPI, configFile)
-	if err != nil && !IsErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
-		return err
+	reader, e := readConfig(ctx, objAPI, configFile)
+	if e != nil && !IsErrIgnored(e, errDiskNotFound, errConfigNotFound) {
+		return e
 	}
 
 	listenerList := []ListenBucketNotificationArgs{}
 	if reader != nil {
-		if err = json.NewDecoder(reader).Decode(&listenerList); err != nil {
+		err := json.NewDecoder(reader).Decode(&listenerList)
+		if err != nil {
 			logger.LogIf(ctx, err)
 			return err
 		}
@@ -201,8 +264,8 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 
 	activeListenerList := []ListenBucketNotificationArgs{}
 	for _, args := range listenerList {
-		var found bool
-		if found, err = isLocalHost(args.Addr.Name); err != nil {
+		found, err := isLocalHost(args.Addr.Name)
+		if err != nil {
 			logger.GetReqInfo(ctx).AppendTags("host", args.Addr.Name)
 			logger.LogIf(ctx, err)
 			return err
@@ -217,8 +280,8 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 			return fmt.Errorf("unable to find PeerRPCClient by address %v in listener.json for bucket %v", args.Addr, bucketName)
 		}
 
-		var exist bool
-		if exist, err = rpcClient.RemoteTargetExist(bucketName, args.TargetID); err != nil {
+		exist, err := rpcClient.RemoteTargetExist(bucketName, args.TargetID)
+		if err != nil {
 			logger.GetReqInfo(ctx).AppendTags("targetID", args.TargetID.Name)
 			logger.LogIf(ctx, err)
 			return err
@@ -231,7 +294,7 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 		target := NewPeerRPCClientTarget(bucketName, args.TargetID, rpcClient)
 		rulesMap := event.NewRulesMap(args.EventNames, args.Pattern, target.ID())
 		if err = sys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
-			logger.GetReqInfo(ctx).AppendTags("targetID", target.id.Name)
+			logger.GetReqInfo(ctx).AppendTags("targetName", target.id.Name)
 			logger.LogIf(ctx, err)
 			return err
 		}
@@ -329,10 +392,10 @@ func (sys *NotificationSys) RemoveAllRemoteTargets() {
 
 // RemoveRemoteTarget - closes and removes target by target ID.
 func (sys *NotificationSys) RemoveRemoteTarget(bucketName string, targetID event.TargetID) {
-	for id, err := range sys.targetList.Remove(targetID) {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("targetID", id.Name)
+	for terr := range sys.targetList.Remove(targetID) {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("targetID", terr.ID.Name)
 		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogIf(ctx, err)
+		logger.LogIf(ctx, terr.Err)
 	}
 
 	sys.Lock()
@@ -346,19 +409,20 @@ func (sys *NotificationSys) RemoveRemoteTarget(bucketName string, targetID event
 	}
 }
 
-func (sys *NotificationSys) send(bucketName string, eventData event.Event, targetIDs ...event.TargetID) map[event.TargetID]error {
-	errMap := sys.targetList.Send(eventData, targetIDs...)
-	for targetID := range errMap {
-		if sys.RemoteTargetExist(bucketName, targetID) {
-			sys.RemoveRemoteTarget(bucketName, targetID)
+func (sys *NotificationSys) send(bucketName string, eventData event.Event, targetIDs ...event.TargetID) (errs []event.TargetIDErr) {
+	errCh := sys.targetList.Send(eventData, targetIDs...)
+	for terr := range errCh {
+		errs = append(errs, terr)
+		if sys.RemoteTargetExist(bucketName, terr.ID) {
+			sys.RemoveRemoteTarget(bucketName, terr.ID)
 		}
 	}
 
-	return errMap
+	return errs
 }
 
 // Send - sends event data to all matching targets.
-func (sys *NotificationSys) Send(args eventArgs) map[event.TargetID]error {
+func (sys *NotificationSys) Send(args eventArgs) []event.TargetIDErr {
 	sys.RLock()
 	targetIDSet := sys.bucketRulesMap[args.BucketName].Match(args.EventName, args.Object.Name)
 	sys.RUnlock()
@@ -432,7 +496,7 @@ func (args eventArgs) ToEvent() event.Event {
 			Bucket: event.Bucket{
 				Name:          args.BucketName,
 				OwnerIdentity: event.Identity{creds.AccessKey},
-				ARN:           bucketARNPrefix + args.BucketName,
+				ARN:           policy.ResourceARNPrefix + args.BucketName,
 			},
 			Object: event.Object{
 				Key:       url.QueryEscape(args.Object.Name),
@@ -463,12 +527,12 @@ func sendEvent(args eventArgs) {
 		return
 	}
 
-	for targetID, err := range globalNotificationSys.Send(args) {
+	for _, err := range globalNotificationSys.Send(args) {
 		reqInfo := &logger.ReqInfo{BucketName: args.BucketName, ObjectName: args.Object.Name}
 		reqInfo.AppendTags("EventName", args.EventName.String())
-		reqInfo.AppendTags("targetID", targetID.Name)
+		reqInfo.AppendTags("targetID", err.ID.Name)
 		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err.Err, err.ID)
 	}
 }
 
@@ -482,6 +546,8 @@ func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error {
 	return err
 }
 
+var errConfigNotFound = errors.New("config file not found")
+
 func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*bytes.Buffer, error) {
 	var buffer bytes.Buffer
 	// Read entire content by setting size to -1
@@ -489,8 +555,9 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*by
 	if err != nil {
 		// Ignore if err is ObjectNotFound or IncompleteBody when bucket is not configured with notification
 		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
-			return nil, errNoSuchNotifications
+			return nil, errConfigNotFound
 		}
+
 		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
 		logger.LogIf(ctx, err)
 		return nil, err
@@ -509,6 +576,10 @@ func readNotificationConfig(ctx context.Context, objAPI ObjectLayer, bucketName
 	configFile := path.Join(bucketConfigPrefix, bucketName, bucketNotificationConfig)
 	reader, err := readConfig(ctx, objAPI, configFile)
 	if err != nil {
+		if err == errConfigNotFound {
+			err = errNoSuchNotifications
+		}
+
 		return nil, err
 	}
 
@@ -550,7 +621,7 @@ func SaveListener(objAPI ObjectLayer, bucketName string, eventNames []event.Name
 	defer objLock.Unlock()
 
 	reader, err := readConfig(ctx, objAPI, configFile)
-	if err != nil && !IsErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
+	if err != nil && !IsErrIgnored(err, errDiskNotFound, errConfigNotFound) {
 		return err
 	}
 
@@ -601,7 +672,7 @@ func RemoveListener(objAPI ObjectLayer, bucketName string, targetID event.Target
 	defer objLock.Unlock()
 
 	reader, err := readConfig(ctx, objAPI, configFile)
-	if err != nil && !IsErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
+	if err != nil && !IsErrIgnored(err, errDiskNotFound, errConfigNotFound) {
 		return err
 	}
 

cmd/object-api-common.go

@@ -85,7 +85,7 @@ func dirObjectInfo(bucket, object string, size int64, metadata map[string]string
 
 func deleteBucketMetadata(ctx context.Context, bucket string, objAPI ObjectLayer) {
 	// Delete bucket access policy, if present - ignore any errors.
-	removeBucketPolicy(ctx, bucket, objAPI)
+	removePolicyConfig(ctx, objAPI, bucket)
 
 	// Delete notification config, if present - ignore any errors.
 	removeNotificationConfig(ctx, objAPI, bucket)
@@ -116,7 +116,7 @@ func cleanupDir(ctx context.Context, storage StorageAPI, volume, dirPath string)
 		}
 
 		// If it's a directory, list and call delFunc() for each entry.
-		entries, err := storage.ListDir(volume, entryPath)
+		entries, err := storage.ListDir(volume, entryPath, -1)
 		// If entryPath prefix never existed, safe to ignore.
 		if err == errFileNotFound {
 			return nil

cmd/object-api-errors.go

@@ -366,13 +366,6 @@ func (e PolicyNesting) Error() string {
 	return "New bucket policy conflicts with an existing policy. Please try again with new prefix."
 }
 
-// PolicyNotFound - policy not found
-type PolicyNotFound GenericError
-
-func (e PolicyNotFound) Error() string {
-	return "Policy not found"
-}
-
 // UnsupportedMetadata - unsupported metadata
 type UnsupportedMetadata struct{}
 
@@ -396,15 +389,6 @@ func isErrIncompleteBody(err error) bool {
 	return false
 }
 
-// isErrBucketPolicyNotFound - Check if error type is BucketPolicyNotFound.
-func isErrBucketPolicyNotFound(err error) bool {
-	switch err.(type) {
-	case PolicyNotFound:
-		return true
-	}
-	return false
-}
-
 // isErrObjectNotFound - Check if error type is ObjectNotFound.
 func isErrObjectNotFound(err error) bool {
 	switch err.(type) {

cmd/object-api-interface.go

@@ -21,9 +21,9 @@ import (
 	"io"
 	"time"
 
-	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/policy"
 )
 
 // ObjectLayer implements primitives for object API layer.
@@ -70,9 +70,8 @@ type ObjectLayer interface {
 	ClearLocks(context.Context, []VolumeLockInfo) error
 
 	// Policy operations
-	SetBucketPolicy(context.Context, string, policy.BucketAccessPolicy) error
-	GetBucketPolicy(context.Context, string) (policy.BucketAccessPolicy, error)
-	RefreshBucketPolicy(context.Context, string) error
+	SetBucketPolicy(context.Context, string, *policy.Policy) error
+	GetBucketPolicy(context.Context, string) (*policy.Policy, error)
 	DeleteBucketPolicy(context.Context, string) error
 
 	// Supported operations check

cmd/object-api-listobjects_test.go

@@ -425,10 +425,10 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 		shouldPass bool
 	}{
 		// Test cases with invalid bucket names ( Test number 1-4 ).
-		{".test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
-		{"Test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
-		{"---", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
-		{"ad", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		{".test", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: ".test"}, false},
+		{"Test", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "Test"}, false},
+		{"---", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "---"}, false},
+		{"ad", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "ad"}, false},
 		// Using an existing file for bucket name, but its not a directory (5).
 		{"simple-file.txt", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "simple-file.txt"}, false},
 		// Valid bucket names, but they donot exist (6-8).

cmd/object-api-multipart_test.go

@@ -111,7 +111,7 @@ func testObjectAbortMultipartUpload(obj ObjectLayer, instanceType string, t Test
 		uploadID        string
 		expectedErrType error
 	}{
-		{"--", object, uploadID, BucketNameInvalid{}},
+		{"--", object, uploadID, BucketNotFound{}},
 		{bucket, "\\", uploadID, ObjectNameInvalid{}},
 		{"foo", object, uploadID, BucketNotFound{}},
 		{bucket, object, "foo-foo", InvalidUploadID{}},
@@ -293,11 +293,11 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
 	}{
 		// Test case  1-4.
 		// Cases with invalid bucket name.
-		{".test", "obj", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: .test")},
-		{"------", "obj", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: ------")},
+		{".test", "obj", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket not found: .test")},
+		{"------", "obj", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket not found: ------")},
 		{"$this-is-not-valid-too", "obj", "", 1, "", "", "", 0, false, "",
-			fmt.Errorf("%s", "Bucket name invalid: $this-is-not-valid-too")},
-		{"a", "obj", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket name invalid: a")},
+			fmt.Errorf("%s", "Bucket not found: $this-is-not-valid-too")},
+		{"a", "obj", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Bucket not found: a")},
 		// Test case - 5.
 		// Case with invalid object names.
 		{bucket, "", "", 1, "", "", "", 0, false, "", fmt.Errorf("%s", "Object name invalid: minio-bucket#")},
@@ -1103,10 +1103,10 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 		shouldPass bool
 	}{
 		// Test cases with invalid bucket names ( Test number 1-4 ).
-		{".test", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
-		{"Test", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
-		{"---", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
-		{"ad", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		{".test", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: ".test"}, false},
+		{"Test", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "Test"}, false},
+		{"---", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "---"}, false},
+		{"ad", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "ad"}, false},
 		// Valid bucket names, but they donot exist (Test number 5-7).
 		{"volatile-bucket-1", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
 		{"volatile-bucket-2", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
@@ -1404,10 +1404,10 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks
 		shouldPass bool
 	}{
 		// Test cases with invalid bucket names (Test number 1-4).
-		{".test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
-		{"Test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
-		{"---", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
-		{"ad", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		{".test", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: ".test"}, false},
+		{"Test", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "Test"}, false},
+		{"---", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "---"}, false},
+		{"ad", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "ad"}, false},
 		// Test cases for listing uploadID with single part.
 		// Valid bucket names, but they donot exist (Test number 5-7).
 		{"volatile-bucket-1", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
@@ -1642,10 +1642,10 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
 		shouldPass bool
 	}{
 		// Test cases with invalid bucket names (Test number 1-4).
-		{".test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
-		{"Test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
-		{"---", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
-		{"ad", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		{".test", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: ".test"}, false},
+		{"Test", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "Test"}, false},
+		{"---", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "---"}, false},
+		{"ad", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "ad"}, false},
 		// Test cases for listing uploadID with single part.
 		// Valid bucket names, but they donot exist (Test number 5-7).
 		{"volatile-bucket-1", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
@@ -1863,10 +1863,10 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
 		shouldPass bool
 	}{
 		// Test cases with invalid bucket names (Test number 1-4).
-		{".test", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: ".test"}, false},
-		{"Test", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: "Test"}, false},
-		{"---", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: "---"}, false},
-		{"ad", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: "ad"}, false},
+		{".test", "", "", []CompletePart{}, "", BucketNotFound{Bucket: ".test"}, false},
+		{"Test", "", "", []CompletePart{}, "", BucketNotFound{Bucket: "Test"}, false},
+		{"---", "", "", []CompletePart{}, "", BucketNotFound{Bucket: "---"}, false},
+		{"ad", "", "", []CompletePart{}, "", BucketNotFound{Bucket: "ad"}, false},
 		// Test cases for listing uploadID with single part.
 		// Valid bucket names, but they donot exist (Test number 5-7).
 		{"volatile-bucket-1", "", "", []CompletePart{}, "", BucketNotFound{Bucket: "volatile-bucket-1"}, false},

cmd/object-api-putobject_test.go

@@ -80,11 +80,11 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
 	}{
 		// Test case  1-4.
 		// Cases with invalid bucket name.
-		{".test", "obj", []byte(""), nil, "", 0, "", BucketNameInvalid{Bucket: ".test"}},
-		{"------", "obj", []byte(""), nil, "", 0, "", BucketNameInvalid{Bucket: "------"}},
+		{".test", "obj", []byte(""), nil, "", 0, "", BucketNotFound{Bucket: ".test"}},
+		{"------", "obj", []byte(""), nil, "", 0, "", BucketNotFound{Bucket: "------"}},
 		{"$this-is-not-valid-too", "obj", []byte(""), nil, "", 0, "",
-			BucketNameInvalid{Bucket: "$this-is-not-valid-too"}},
-		{"a", "obj", []byte(""), nil, "", 0, "", BucketNameInvalid{Bucket: "a"}},
+			BucketNotFound{Bucket: "$this-is-not-valid-too"}},
+		{"a", "obj", []byte(""), nil, "", 0, "", BucketNotFound{Bucket: "a"}},
 
 		// Test case - 5.
 		// Case with invalid object names.

cmd/object-api-utils.go

@@ -168,7 +168,7 @@ func pathJoin(elem ...string) string {
 func mustGetUUID() string {
 	uuid, err := uuid.New()
 	if err != nil {
-		panic(fmt.Sprintf("Random UUID generation failed. Error: %s", err))
+		logger.CriticalIf(context.Background(), err)
 	}
 
 	return uuid.String()

cmd/object-handlers.go

@@ -31,12 +31,12 @@ import (
 	"sort"
 	"strconv"
 
-	mux "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/ioutil"
+	"github.com/minio/minio/pkg/policy"
 	sha256 "github.com/minio/sha256-simd"
 	"github.com/minio/sio"
 )
@@ -60,23 +60,6 @@ func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) {
 	}
 }
 
-// errAllowableNotFound - For an anon user, return 404 if have ListBucket, 403 otherwise
-// this is in keeping with the permissions sections of the docs of both:
-//   HEAD Object: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html
-//   GET Object: http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
-func errAllowableObjectNotFound(ctx context.Context, bucket string, r *http.Request) APIErrorCode {
-	if getRequestAuthType(r) == authTypeAnonymous {
-		// We care about the bucket as a whole, not a particular resource.
-		resource := "/" + bucket
-		sourceIP := handlers.GetSourceIP(r)
-		if s3Error := enforceBucketPolicy(ctx, bucket, "s3:ListBucket", resource,
-			r.Referer(), sourceIP, r.URL.Query()); s3Error != ErrNone {
-			return ErrAccessDenied
-		}
-	}
-	return ErrNoSuchKey
-}
-
 // GetObjectHandler - GET Object
 // ----------
 // This implementation of the GET operation retrieves object. To use GET,
@@ -96,23 +79,36 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:GetObject", globalServerConfig.GetRegion()); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
-		return
-	}
-
 	getObjectInfo := objectAPI.GetObjectInfo
 	if api.CacheAPI() != nil {
 		getObjectInfo = api.CacheAPI().GetObjectInfo
 	}
 
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object); s3Error != ErrNone {
+		if getRequestAuthType(r) == authTypeAnonymous {
+			// As per "Permission" section in https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
+			// If the object you request does not exist, the error Amazon S3 returns depends on whether you also have the s3:ListBucket permission.
+			// * If you have the s3:ListBucket permission on the bucket, Amazon S3 will return an HTTP status code 404 ("no such key") error.
+			// * if you don’t have the s3:ListBucket permission, Amazon S3 will return an HTTP status code 403 ("access denied") error.`
+			if globalPolicySys.IsAllowed(policy.Args{
+				Action:          policy.ListBucketAction,
+				BucketName:      bucket,
+				ConditionValues: getConditionValues(r, ""),
+				IsOwner:         false,
+			}) {
+				_, err := getObjectInfo(ctx, bucket, object)
+				if toAPIErrorCode(err) == ErrNoSuchKey {
+					s3Error = ErrNoSuchKey
+				}
+			}
+		}
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
+
 	objInfo, err := getObjectInfo(ctx, bucket, object)
 	if err != nil {
-		apiErr := toAPIErrorCode(err)
-		if apiErr == ErrNoSuchKey {
-			apiErr = errAllowableObjectNotFound(ctx, bucket, r)
-		}
-		writeErrorResponse(w, apiErr, r.URL)
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
@@ -232,23 +228,36 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:GetObject", globalServerConfig.GetRegion()); s3Error != ErrNone {
-		writeErrorResponseHeadersOnly(w, s3Error)
-		return
-	}
-
 	getObjectInfo := objectAPI.GetObjectInfo
 	if api.CacheAPI() != nil {
 		getObjectInfo = api.CacheAPI().GetObjectInfo
 	}
 
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object); s3Error != ErrNone {
+		if getRequestAuthType(r) == authTypeAnonymous {
+			// As per "Permission" section in https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html
+			// If the object you request does not exist, the error Amazon S3 returns depends on whether you also have the s3:ListBucket permission.
+			// * If you have the s3:ListBucket permission on the bucket, Amazon S3 will return an HTTP status code 404 ("no such key") error.
+			// * if you don’t have the s3:ListBucket permission, Amazon S3 will return an HTTP status code 403 ("access denied") error.`
+			if globalPolicySys.IsAllowed(policy.Args{
+				Action:          policy.ListBucketAction,
+				BucketName:      bucket,
+				ConditionValues: getConditionValues(r, ""),
+				IsOwner:         false,
+			}) {
+				_, err := getObjectInfo(ctx, bucket, object)
+				if toAPIErrorCode(err) == ErrNoSuchKey {
+					s3Error = ErrNoSuchKey
+				}
+			}
+		}
+		writeErrorResponseHeadersOnly(w, s3Error)
+		return
+	}
+
 	objInfo, err := getObjectInfo(ctx, bucket, object)
 	if err != nil {
-		apiErr := toAPIErrorCode(err)
-		if apiErr == ErrNoSuchKey {
-			apiErr = errAllowableObjectNotFound(ctx, bucket, r)
-		}
-		writeErrorResponseHeadersOnly(w, apiErr)
+		writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
 		return
 	}
 
@@ -340,7 +349,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, dstBucket, "s3:PutObject", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, dstBucket, dstObject); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -666,10 +675,14 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(w, ErrAccessDenied, r.URL)
 		return
 	case authTypeAnonymous:
-		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		sourceIP := handlers.GetSourceIP(r)
-		if s3Err = enforceBucketPolicy(ctx, bucket, "s3:PutObject", r.URL.Path, r.Referer(), sourceIP, r.URL.Query()); s3Err != ErrNone {
-			writeErrorResponse(w, s3Err, r.URL)
+		if !globalPolicySys.IsAllowed(policy.Args{
+			Action:          policy.PutObjectAction,
+			BucketName:      bucket,
+			ConditionValues: getConditionValues(r, ""),
+			IsOwner:         false,
+			ObjectName:      object,
+		}) {
+			writeErrorResponse(w, ErrAccessDenied, r.URL)
 			return
 		}
 	case authTypeStreamingSigned:
@@ -781,7 +794,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:PutObject", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, bucket, object); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -867,7 +880,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, dstBucket, "s3:PutObject", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, dstBucket, dstObject); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -1133,10 +1146,14 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		writeErrorResponse(w, ErrAccessDenied, r.URL)
 		return
 	case authTypeAnonymous:
-		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
-		if s3Error := enforceBucketPolicy(ctx, bucket, "s3:PutObject", r.URL.Path,
-			r.Referer(), handlers.GetSourceIP(r), r.URL.Query()); s3Error != ErrNone {
-			writeErrorResponse(w, s3Error, r.URL)
+		if !globalPolicySys.IsAllowed(policy.Args{
+			Action:          policy.PutObjectAction,
+			BucketName:      bucket,
+			ConditionValues: getConditionValues(r, ""),
+			IsOwner:         false,
+			ObjectName:      object,
+		}) {
+			writeErrorResponse(w, ErrAccessDenied, r.URL)
 			return
 		}
 	case authTypeStreamingSigned:
@@ -1262,7 +1279,8 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 	if api.CacheAPI() != nil {
 		abortMultipartUpload = api.CacheAPI().AbortMultipartUpload
 	}
-	if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:AbortMultipartUpload", globalServerConfig.GetRegion()); s3Error != ErrNone {
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.AbortMultipartUploadAction, bucket, object); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -1297,7 +1315,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:ListMultipartUploadParts", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListMultipartUploadPartsAction, bucket, object); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -1337,7 +1355,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:PutObject", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, bucket, object); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -1446,7 +1464,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, bucket, "s3:DeleteObject", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, object); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}

cmd/object-handlers_test.go

@@ -172,7 +172,7 @@ func testAPIHeadObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestAPIHeadObjectHandler", bucketName, objectName, instanceType, apiRouter, anonReq, getReadOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestAPIHeadObjectHandler", bucketName, objectName, instanceType, apiRouter, anonReq, getAnonReadOnlyObjectPolicy(bucketName, objectName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -445,7 +445,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestAPIGetObjectHandler", bucketName, objectName, instanceType, apiRouter, anonReq, getReadOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestAPIGetObjectHandler", bucketName, objectName, instanceType, apiRouter, anonReq, getAnonReadOnlyObjectPolicy(bucketName, objectName))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -1001,7 +1001,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestAPIPutObjectHandler", bucketName, objectName, instanceType, apiRouter, anonReq, getWriteOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestAPIPutObjectHandler", bucketName, objectName, instanceType, apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, objectName))
 
 	// HTTP request to test the case of `objectLayer` being set to `nil`.
 	// There is no need to use an existing bucket or valid input for creating the request,
@@ -1847,7 +1847,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestAPICopyObjectHandler", bucketName, newCopyAnonObject, instanceType, apiRouter, anonReq, getWriteOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestAPICopyObjectHandler", bucketName, newCopyAnonObject, instanceType, apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, newCopyAnonObject))
 
 	// HTTP request to test the case of `objectLayer` being set to `nil`.
 	// There is no need to use an existing bucket or valid input for creating the request,
@@ -1998,7 +1998,7 @@ func testAPINewMultipartHandler(obj ObjectLayer, instanceType, bucketName string
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestAPINewMultipartHandler", bucketName, objectName, instanceType, apiRouter, anonReq, getWriteOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestAPINewMultipartHandler", bucketName, objectName, instanceType, apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, objectName))
 
 	// HTTP request to test the case of `objectLayer` being set to `nil`.
 	// There is no need to use an existing bucket or valid input for creating the request,
@@ -2409,7 +2409,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
 	ExecObjectLayerAPIAnonTest(t, obj, "TestAPICompleteMultipartHandler", bucketName, objectName, instanceType,
-		apiRouter, anonReq, getWriteOnlyObjectStatement)
+		apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, objectName))
 
 	// HTTP request to test the case of `objectLayer` being set to `nil`.
 	// There is no need to use an existing bucket or valid input for creating the request,
@@ -2572,7 +2572,7 @@ func testAPIAbortMultipartHandler(obj ObjectLayer, instanceType, bucketName stri
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
 	ExecObjectLayerAPIAnonTest(t, obj, "TestAPIAbortMultipartHandler", bucketName, objectName, instanceType,
-		apiRouter, anonReq, getWriteOnlyObjectStatement)
+		apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, objectName))
 
 	// HTTP request to test the case of `objectLayer` being set to `nil`.
 	// There is no need to use an existing bucket or valid input for creating the request,
@@ -2734,7 +2734,7 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestAPIDeleteObjectHandler", bucketName, anonObjectName, instanceType, apiRouter, anonReq, getWriteOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestAPIDeleteObjectHandler", bucketName, anonObjectName, instanceType, apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, anonObjectName))
 
 	// HTTP request to test the case of `objectLayer` being set to `nil`.
 	// There is no need to use an existing bucket or valid input for creating the request,
@@ -3205,7 +3205,7 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestAPIPutObjectPartHandler", bucketName, testObject, instanceType, apiRouter, anonReq, getWriteOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestAPIPutObjectPartHandler", bucketName, testObject, instanceType, apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, testObject))
 
 	// HTTP request for testing when `ObjectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request
@@ -3508,7 +3508,7 @@ func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName str
 	// ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse,
 	// sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the
 	// unsigned request goes through and its validated again.
-	ExecObjectLayerAPIAnonTest(t, obj, "TestAPIListObjectPartsHandler", bucketName, testObject, instanceType, apiRouter, anonReq, getWriteOnlyObjectStatement)
+	ExecObjectLayerAPIAnonTest(t, obj, "TestAPIListObjectPartsHandler", bucketName, testObject, instanceType, apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, testObject))
 
 	// HTTP request for testing when `objectLayer` is set to `nil`.
 	// There is no need to use an existing bucket and valid input for creating the request

cmd/peer-rpc.go

@@ -25,6 +25,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
 	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/policy"
 )
 
 const s3Path = "/s3/remote"
@@ -43,23 +44,33 @@ type DeleteBucketArgs struct {
 // DeleteBucket - handles delete bucket RPC call which removes all values of given bucket in global NotificationSys object.
 func (receiver *PeerRPCReceiver) DeleteBucket(args *DeleteBucketArgs, reply *AuthRPCArgs) error {
 	globalNotificationSys.RemoveNotification(args.BucketName)
+	globalPolicySys.Remove(args.BucketName)
 	return nil
 }
 
-// UpdateBucketPolicyArgs - update bucket policy RPC arguments.
-type UpdateBucketPolicyArgs struct {
+// SetBucketPolicyArgs - set bucket policy RPC arguments.
+type SetBucketPolicyArgs struct {
+	AuthRPCArgs
+	BucketName string
+	Policy     policy.Policy
+}
+
+// SetBucketPolicy - handles set bucket policy RPC call which adds bucket policy to globalPolicySys.
+func (receiver *PeerRPCReceiver) SetBucketPolicy(args *SetBucketPolicyArgs, reply *AuthRPCArgs) error {
+	globalPolicySys.Set(args.BucketName, args.Policy)
+	return nil
+}
+
+// RemoveBucketPolicyArgs - delete bucket policy RPC arguments.
+type RemoveBucketPolicyArgs struct {
 	AuthRPCArgs
 	BucketName string
 }
 
-// UpdateBucketPolicy - handles update bucket policy RPC call which sets bucket policies to given bucket in global BucketPolicies object.
-func (receiver *PeerRPCReceiver) UpdateBucketPolicy(args *UpdateBucketPolicyArgs, reply *AuthRPCArgs) error {
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		// If the object layer is just coming up then it will load the policy from the disk.
-		return nil
-	}
-	return objectAPI.RefreshBucketPolicy(context.Background(), args.BucketName)
+// RemoveBucketPolicy - handles delete bucket policy RPC call which removes bucket policy to globalPolicySys.
+func (receiver *PeerRPCReceiver) RemoveBucketPolicy(args *RemoveBucketPolicyArgs, reply *AuthRPCArgs) error {
+	globalPolicySys.Remove(args.BucketName)
+	return nil
 }
 
 // PutBucketNotificationArgs - put bucket notification RPC arguments.
@@ -104,7 +115,7 @@ func (receiver *PeerRPCReceiver) ListenBucketNotification(args *ListenBucketNoti
 	rulesMap := event.NewRulesMap(args.EventNames, args.Pattern, target.ID())
 	if err := globalNotificationSys.AddRemoteTarget(args.BucketName, target, rulesMap); err != nil {
 		reqInfo := &logger.ReqInfo{BucketName: target.bucketName}
-		reqInfo.AppendTags("target", target.id.Name)
+		reqInfo.AppendTags("targetName", target.id.Name)
 		ctx := logger.SetReqInfo(context.Background(), reqInfo)
 		logger.LogIf(ctx, err)
 		return err
@@ -152,19 +163,12 @@ func (receiver *PeerRPCReceiver) SendEvent(args *SendEventArgs, reply *SendEvent
 	}
 
 	var err error
-	if errMap := globalNotificationSys.send(args.BucketName, args.Event, args.TargetID); len(errMap) != 0 {
-		var found bool
-		if err, found = errMap[args.TargetID]; !found {
-			// errMap must be zero or one element map because we sent to only one target ID.
-			panic(fmt.Errorf("error for target %v not found in error map %+v", args.TargetID, errMap))
-		}
-	}
-
-	if err != nil {
+	for _, terr := range globalNotificationSys.send(args.BucketName, args.Event, args.TargetID) {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("Event", args.Event.EventName.String())
-		reqInfo.AppendTags("target", args.TargetID.Name)
+		reqInfo.AppendTags("targetName", args.TargetID.Name)
 		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogIf(ctx, err)
+		logger.LogIf(ctx, terr.Err)
+		err = terr.Err
 	}
 
 	reply.Error = err
@@ -179,7 +183,7 @@ func registerS3PeerRPCRouter(router *mux.Router) error {
 		return err
 	}
 
-	subrouter := router.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
+	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
 	subrouter.Path(s3Path).Handler(peerRPCServer)
 	return nil
 }
@@ -196,13 +200,23 @@ func (rpcClient *PeerRPCClient) DeleteBucket(bucketName string) error {
 	return rpcClient.Call("Peer.DeleteBucket", &args, &reply)
 }
 
-// UpdateBucketPolicy - calls update bucket policy RPC.
-func (rpcClient *PeerRPCClient) UpdateBucketPolicy(bucketName string) error {
-	args := UpdateBucketPolicyArgs{
+// SetBucketPolicy - calls set bucket policy RPC.
+func (rpcClient *PeerRPCClient) SetBucketPolicy(bucketName string, bucketPolicy *policy.Policy) error {
+	args := SetBucketPolicyArgs{
+		BucketName: bucketName,
+		Policy:     *bucketPolicy,
+	}
+	reply := AuthRPCReply{}
+	return rpcClient.Call("Peer.SetBucketPolicy", &args, &reply)
+}
+
+// RemoveBucketPolicy - calls remove bucket policy RPC.
+func (rpcClient *PeerRPCClient) RemoveBucketPolicy(bucketName string) error {
+	args := RemoveBucketPolicyArgs{
 		BucketName: bucketName,
 	}
 	reply := AuthRPCReply{}
-	return rpcClient.Call("Peer.UpdateBucketPolicy", &args, &reply)
+	return rpcClient.Call("Peer.RemoveBucketPolicy", &args, &reply)
 }
 
 // PutBucketNotification - calls put bukcet notification RPC.
@@ -275,7 +289,8 @@ func makeRemoteRPCClients(endpoints EndpointList) map[xnet.Host]*PeerRPCClient {
 	cred := globalServerConfig.GetCredential()
 	serviceEndpoint := path.Join(minioReservedBucketPath, s3Path)
 	for _, hostStr := range GetRemotePeers(endpoints) {
-		host := xnet.MustParseHost(hostStr)
+		host, err := xnet.ParseHost(hostStr)
+		logger.CriticalIf(context.Background(), err)
 		peerRPCClientMap[*host] = &PeerRPCClient{newAuthRPCClient(authConfig{
 			accessKey:       cred.AccessKey,
 			secretKey:       cred.SecretKey,

cmd/policy.go

@@ -0,0 +1,228 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"path"
+	"sync"
+
+	miniogopolicy "github.com/minio/minio-go/pkg/policy"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/handlers"
+	"github.com/minio/minio/pkg/policy"
+)
+
+// PolicySys - policy system.
+type PolicySys struct {
+	sync.RWMutex
+	bucketPolicyMap map[string]policy.Policy
+}
+
+// Set - sets policy to given bucket name.  If policy is empty, existing policy is removed.
+func (sys *PolicySys) Set(bucketName string, policy policy.Policy) {
+	sys.Lock()
+	defer sys.Unlock()
+
+	if policy.IsEmpty() {
+		delete(sys.bucketPolicyMap, bucketName)
+	} else {
+		sys.bucketPolicyMap[bucketName] = policy
+	}
+}
+
+// Remove - removes policy for given bucket name.
+func (sys *PolicySys) Remove(bucketName string) {
+	sys.Lock()
+	defer sys.Unlock()
+
+	delete(sys.bucketPolicyMap, bucketName)
+}
+
+// IsAllowed - checks given policy args is allowed to continue the Rest API.
+func (sys *PolicySys) IsAllowed(args policy.Args) bool {
+	sys.RLock()
+	defer sys.RUnlock()
+
+	// If policy is available for given bucket, check the policy.
+	if p, found := sys.bucketPolicyMap[args.BucketName]; found {
+		return p.IsAllowed(args)
+	}
+
+	// As policy is not available for given bucket name, returns IsOwner i.e.
+	// operation is allowed only for owner.
+	return args.IsOwner
+}
+
+// Init - initializes policy system from policy.json of all buckets.
+func (sys *PolicySys) Init(objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errInvalidArgument
+	}
+
+	buckets, err := objAPI.ListBuckets(context.Background())
+	if err != nil {
+		return err
+	}
+
+	for _, bucket := range buckets {
+		config, err := GetPolicyConfig(objAPI, bucket.Name)
+		if err != nil {
+			if _, ok := err.(BucketPolicyNotFound); !ok {
+				return err
+			}
+		} else {
+			// This part is specifically written to handle migration
+			// when the Version string is empty, this was allowed
+			// in all previous minio releases but we need to migrate
+			// those policies by properly setting the Version string
+			// from now on.
+			if config.Version == "" {
+				logger.Info("Found in-consistent bucket policies, Migrating them for Bucket: (%s)", bucket.Name)
+				config.Version = policy.DefaultVersion
+
+				if err = savePolicyConfig(objAPI, bucket.Name, config); err != nil {
+					return err
+				}
+			}
+
+			sys.Set(bucket.Name, *config)
+		}
+	}
+
+	return nil
+}
+
+// NewPolicySys - creates new policy system.
+func NewPolicySys() *PolicySys {
+	return &PolicySys{
+		bucketPolicyMap: make(map[string]policy.Policy),
+	}
+}
+
+func getConditionValues(request *http.Request, locationConstraint string) map[string][]string {
+	args := make(map[string][]string)
+
+	for key, values := range request.Header {
+		if existingValues, found := args[key]; found {
+			args[key] = append(existingValues, values...)
+		} else {
+			args[key] = values
+		}
+	}
+
+	for key, values := range request.URL.Query() {
+		if existingValues, found := args[key]; found {
+			args[key] = append(existingValues, values...)
+		} else {
+			args[key] = values
+		}
+	}
+
+	args["SourceIp"] = []string{handlers.GetSourceIP(request)}
+
+	if locationConstraint != "" {
+		args["LocationConstraint"] = []string{locationConstraint}
+	}
+
+	return args
+}
+
+// GetPolicyConfig - get policy config for given bucket name.
+func GetPolicyConfig(objAPI ObjectLayer, bucketName string) (*policy.Policy, error) {
+	// Construct path to policy.json for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
+
+	reader, err := readConfig(context.Background(), objAPI, configFile)
+	if err != nil {
+		if err == errConfigNotFound {
+			err = BucketPolicyNotFound{Bucket: bucketName}
+		}
+
+		return nil, err
+	}
+
+	return policy.ParseConfig(reader, bucketName)
+}
+
+func savePolicyConfig(objAPI ObjectLayer, bucketName string, bucketPolicy *policy.Policy) error {
+	data, err := json.Marshal(bucketPolicy)
+	if err != nil {
+		return err
+	}
+
+	// Construct path to policy.json for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
+
+	return saveConfig(objAPI, configFile, data)
+}
+
+func removePolicyConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) error {
+	// Construct path to policy.json for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
+
+	if err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile); err != nil {
+		if _, ok := err.(ObjectNotFound); ok {
+			return BucketPolicyNotFound{Bucket: bucketName}
+		}
+
+		return err
+	}
+
+	return nil
+}
+
+// PolicyToBucketAccessPolicy - converts policy.Policy to minio-go/policy.BucketAccessPolicy.
+func PolicyToBucketAccessPolicy(bucketPolicy *policy.Policy) (*miniogopolicy.BucketAccessPolicy, error) {
+	// Return empty BucketAccessPolicy for empty bucket policy.
+	if bucketPolicy == nil {
+		return &miniogopolicy.BucketAccessPolicy{Version: policy.DefaultVersion}, nil
+	}
+
+	data, err := json.Marshal(bucketPolicy)
+	if err != nil {
+		// This should not happen because bucketPolicy is valid to convert to JSON data.
+		return nil, err
+	}
+
+	var policyInfo miniogopolicy.BucketAccessPolicy
+	if err = json.Unmarshal(data, &policyInfo); err != nil {
+		// This should not happen because data is valid to JSON data.
+		return nil, err
+	}
+
+	return &policyInfo, nil
+}
+
+// BucketAccessPolicyToPolicy - converts minio-go/policy.BucketAccessPolicy to policy.Policy.
+func BucketAccessPolicyToPolicy(policyInfo *miniogopolicy.BucketAccessPolicy) (*policy.Policy, error) {
+	data, err := json.Marshal(policyInfo)
+	if err != nil {
+		// This should not happen because policyInfo is valid to convert to JSON data.
+		return nil, err
+	}
+
+	var bucketPolicy policy.Policy
+	if err = json.Unmarshal(data, &bucketPolicy); err != nil {
+		// This should not happen because data is valid to JSON data.
+		return nil, err
+	}
+
+	return &bucketPolicy, nil
+}

cmd/policy_test.go

@@ -0,0 +1,424 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"reflect"
+	"testing"
+
+	miniogopolicy "github.com/minio/minio-go/pkg/policy"
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/policy/condition"
+)
+
+func TestPolicySysSet(t *testing.T) {
+	case1PolicySys := NewPolicySys()
+	case1Policy := policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(policy.PutObjectAction),
+				policy.NewResourceSet(policy.NewResource("mybucket", "/myobject*")),
+				condition.NewFunctions(),
+			),
+		},
+	}
+	case1Result := NewPolicySys()
+	case1Result.bucketPolicyMap["mybucket"] = case1Policy
+
+	case2PolicySys := NewPolicySys()
+	case2PolicySys.bucketPolicyMap["mybucket"] = case1Policy
+	case2Policy := policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(policy.GetObjectAction),
+				policy.NewResourceSet(policy.NewResource("mybucket", "/myobject*")),
+				condition.NewFunctions(),
+			),
+		},
+	}
+	case2Result := NewPolicySys()
+	case2Result.bucketPolicyMap["mybucket"] = case2Policy
+
+	case3PolicySys := NewPolicySys()
+	case3PolicySys.bucketPolicyMap["mybucket"] = case2Policy
+	case3Policy := policy.Policy{
+		ID:      "MyPolicyForMyBucket",
+		Version: policy.DefaultVersion,
+	}
+	case3Result := NewPolicySys()
+
+	testCases := []struct {
+		policySys      *PolicySys
+		bucketName     string
+		bucketPolicy   policy.Policy
+		expectedResult *PolicySys
+	}{
+		{case1PolicySys, "mybucket", case1Policy, case1Result},
+		{case2PolicySys, "mybucket", case2Policy, case2Result},
+		{case3PolicySys, "mybucket", case3Policy, case3Result},
+	}
+
+	for i, testCase := range testCases {
+		result := testCase.policySys
+		result.Set(testCase.bucketName, testCase.bucketPolicy)
+
+		if !reflect.DeepEqual(result, testCase.expectedResult) {
+			t.Fatalf("case %v: expected: %v, got: %v\n", i+1, testCase.expectedResult, result)
+		}
+	}
+}
+
+func TestPolicySysRemove(t *testing.T) {
+	case1Policy := policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(policy.PutObjectAction),
+				policy.NewResourceSet(policy.NewResource("mybucket", "/myobject*")),
+				condition.NewFunctions(),
+			),
+		},
+	}
+	case1PolicySys := NewPolicySys()
+	case1PolicySys.bucketPolicyMap["mybucket"] = case1Policy
+	case1Result := NewPolicySys()
+
+	case2Policy := policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(policy.GetObjectAction),
+				policy.NewResourceSet(policy.NewResource("mybucket", "/myobject*")),
+				condition.NewFunctions(),
+			),
+		},
+	}
+	case2PolicySys := NewPolicySys()
+	case2PolicySys.bucketPolicyMap["mybucket"] = case2Policy
+	case2Result := NewPolicySys()
+	case2Result.bucketPolicyMap["mybucket"] = case2Policy
+
+	case3PolicySys := NewPolicySys()
+	case3Result := NewPolicySys()
+
+	testCases := []struct {
+		policySys      *PolicySys
+		bucketName     string
+		expectedResult *PolicySys
+	}{
+		{case1PolicySys, "mybucket", case1Result},
+		{case2PolicySys, "yourbucket", case2Result},
+		{case3PolicySys, "mybucket", case3Result},
+	}
+
+	for i, testCase := range testCases {
+		result := testCase.policySys
+		result.Remove(testCase.bucketName)
+
+		if !reflect.DeepEqual(result, testCase.expectedResult) {
+			t.Fatalf("case %v: expected: %v, got: %v\n", i+1, testCase.expectedResult, result)
+		}
+	}
+}
+
+func TestPolicySysIsAllowed(t *testing.T) {
+	policySys := NewPolicySys()
+	policySys.Set("mybucket", policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(policy.GetBucketLocationAction),
+				policy.NewResourceSet(policy.NewResource("mybucket", "")),
+				condition.NewFunctions(),
+			),
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(policy.PutObjectAction),
+				policy.NewResourceSet(policy.NewResource("mybucket", "/myobject*")),
+				condition.NewFunctions(),
+			),
+		},
+	})
+
+	anonGetBucketLocationArgs := policy.Args{
+		AccountName:     "Q3AM3UQ867SPQQA43P2F",
+		Action:          policy.GetBucketLocationAction,
+		BucketName:      "mybucket",
+		ConditionValues: map[string][]string{},
+	}
+
+	anonPutObjectActionArgs := policy.Args{
+		AccountName: "Q3AM3UQ867SPQQA43P2F",
+		Action:      policy.PutObjectAction,
+		BucketName:  "mybucket",
+		ConditionValues: map[string][]string{
+			"x-amz-copy-source": {"mybucket/myobject"},
+			"SourceIp":          {"192.168.1.10"},
+		},
+		ObjectName: "myobject",
+	}
+
+	anonGetObjectActionArgs := policy.Args{
+		AccountName:     "Q3AM3UQ867SPQQA43P2F",
+		Action:          policy.GetObjectAction,
+		BucketName:      "mybucket",
+		ConditionValues: map[string][]string{},
+		ObjectName:      "myobject",
+	}
+
+	getBucketLocationArgs := policy.Args{
+		AccountName:     "Q3AM3UQ867SPQQA43P2F",
+		Action:          policy.GetBucketLocationAction,
+		BucketName:      "mybucket",
+		ConditionValues: map[string][]string{},
+		IsOwner:         true,
+	}
+
+	putObjectActionArgs := policy.Args{
+		AccountName: "Q3AM3UQ867SPQQA43P2F",
+		Action:      policy.PutObjectAction,
+		BucketName:  "mybucket",
+		ConditionValues: map[string][]string{
+			"x-amz-copy-source": {"mybucket/myobject"},
+			"SourceIp":          {"192.168.1.10"},
+		},
+		IsOwner:    true,
+		ObjectName: "myobject",
+	}
+
+	getObjectActionArgs := policy.Args{
+		AccountName:     "Q3AM3UQ867SPQQA43P2F",
+		Action:          policy.GetObjectAction,
+		BucketName:      "mybucket",
+		ConditionValues: map[string][]string{},
+		IsOwner:         true,
+		ObjectName:      "myobject",
+	}
+
+	yourbucketAnonGetObjectActionArgs := policy.Args{
+		AccountName:     "Q3AM3UQ867SPQQA43P2F",
+		Action:          policy.GetObjectAction,
+		BucketName:      "yourbucket",
+		ConditionValues: map[string][]string{},
+		ObjectName:      "yourobject",
+	}
+
+	yourbucketGetObjectActionArgs := policy.Args{
+		AccountName:     "Q3AM3UQ867SPQQA43P2F",
+		Action:          policy.GetObjectAction,
+		BucketName:      "yourbucket",
+		ConditionValues: map[string][]string{},
+		IsOwner:         true,
+		ObjectName:      "yourobject",
+	}
+
+	testCases := []struct {
+		policySys      *PolicySys
+		args           policy.Args
+		expectedResult bool
+	}{
+		{policySys, anonGetBucketLocationArgs, true},
+		{policySys, anonPutObjectActionArgs, true},
+		{policySys, anonGetObjectActionArgs, false},
+		{policySys, getBucketLocationArgs, true},
+		{policySys, putObjectActionArgs, true},
+		{policySys, getObjectActionArgs, true},
+		{policySys, yourbucketAnonGetObjectActionArgs, false},
+		{policySys, yourbucketGetObjectActionArgs, true},
+	}
+
+	for i, testCase := range testCases {
+		result := testCase.policySys.IsAllowed(testCase.args)
+
+		if result != testCase.expectedResult {
+			t.Fatalf("case %v: expected: %v, got: %v\n", i+1, testCase.expectedResult, result)
+		}
+	}
+}
+
+func getReadOnlyStatement(bucketName, prefix string) []miniogopolicy.Statement {
+	return []miniogopolicy.Statement{
+		{
+			Effect:    string(policy.Allow),
+			Principal: miniogopolicy.User{AWS: set.CreateStringSet("*")},
+			Resources: set.CreateStringSet(policy.NewResource(bucketName, "").String()),
+			Actions:   set.CreateStringSet("s3:GetBucketLocation", "s3:ListBucket"),
+		},
+		{
+			Effect:    string(policy.Allow),
+			Principal: miniogopolicy.User{AWS: set.CreateStringSet("*")},
+			Resources: set.CreateStringSet(policy.NewResource(bucketName, prefix).String()),
+			Actions:   set.CreateStringSet("s3:GetObject"),
+		},
+	}
+}
+
+func TestPolicyToBucketAccessPolicy(t *testing.T) {
+	case1Policy := &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
+				policy.NewResourceSet(policy.NewResource("mybucket", "")),
+				condition.NewFunctions(),
+			),
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(policy.GetObjectAction),
+				policy.NewResourceSet(policy.NewResource("mybucket", "/myobject*")),
+				condition.NewFunctions(),
+			),
+		},
+	}
+
+	case1Result := &miniogopolicy.BucketAccessPolicy{
+		Version:    policy.DefaultVersion,
+		Statements: getReadOnlyStatement("mybucket", "/myobject*"),
+	}
+
+	case2Policy := &policy.Policy{
+		Version:    policy.DefaultVersion,
+		Statements: []policy.Statement{},
+	}
+
+	case2Result := &miniogopolicy.BucketAccessPolicy{
+		Version:    policy.DefaultVersion,
+		Statements: []miniogopolicy.Statement{},
+	}
+
+	case3Policy := &policy.Policy{
+		Version: "12-10-2012",
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(policy.PutObjectAction),
+				policy.NewResourceSet(policy.NewResource("mybucket", "/myobject*")),
+				condition.NewFunctions(),
+			),
+		},
+	}
+
+	testCases := []struct {
+		bucketPolicy   *policy.Policy
+		expectedResult *miniogopolicy.BucketAccessPolicy
+		expectErr      bool
+	}{
+		{case1Policy, case1Result, false},
+		{case2Policy, case2Result, false},
+		{case3Policy, nil, true},
+	}
+
+	for i, testCase := range testCases {
+		result, err := PolicyToBucketAccessPolicy(testCase.bucketPolicy)
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: error: expected: %v, got: %v\n", i+1, testCase.expectErr, expectErr)
+		}
+
+		if !testCase.expectErr {
+			if !reflect.DeepEqual(result, testCase.expectedResult) {
+				t.Fatalf("case %v: result: expected: %+v, got: %+v\n", i+1, testCase.expectedResult, result)
+			}
+		}
+	}
+}
+
+func TestBucketAccessPolicyToPolicy(t *testing.T) {
+	case1PolicyInfo := &miniogopolicy.BucketAccessPolicy{
+		Version:    policy.DefaultVersion,
+		Statements: getReadOnlyStatement("mybucket", "/myobject*"),
+	}
+
+	case1Result := &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
+				policy.NewResourceSet(policy.NewResource("mybucket", "")),
+				condition.NewFunctions(),
+			),
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(policy.GetObjectAction),
+				policy.NewResourceSet(policy.NewResource("mybucket", "/myobject*")),
+				condition.NewFunctions(),
+			),
+		},
+	}
+
+	case2PolicyInfo := &miniogopolicy.BucketAccessPolicy{
+		Version:    policy.DefaultVersion,
+		Statements: []miniogopolicy.Statement{},
+	}
+
+	case2Result := &policy.Policy{
+		Version:    policy.DefaultVersion,
+		Statements: []policy.Statement{},
+	}
+
+	case3PolicyInfo := &miniogopolicy.BucketAccessPolicy{
+		Version:    "12-10-2012",
+		Statements: getReadOnlyStatement("mybucket", "/myobject*"),
+	}
+
+	testCases := []struct {
+		policyInfo     *miniogopolicy.BucketAccessPolicy
+		expectedResult *policy.Policy
+		expectErr      bool
+	}{
+		{case1PolicyInfo, case1Result, false},
+		{case2PolicyInfo, case2Result, false},
+		{case3PolicyInfo, nil, true},
+	}
+
+	for i, testCase := range testCases {
+		result, err := BucketAccessPolicyToPolicy(testCase.policyInfo)
+		expectErr := (err != nil)
+
+		if expectErr != testCase.expectErr {
+			t.Fatalf("case %v: error: expected: %v, got: %v\n", i+1, testCase.expectErr, expectErr)
+		}
+
+		if !testCase.expectErr {
+			if !reflect.DeepEqual(result, testCase.expectedResult) {
+				t.Fatalf("case %v: result: expected: %+v, got: %+v\n", i+1, testCase.expectedResult, result)
+			}
+		}
+	}
+}

cmd/posix-list-dir-nix.go

@@ -112,6 +112,12 @@ var readDirBufPool = sync.Pool{
 
 // Return all the entries at the directory dirPath.
 func readDir(dirPath string) (entries []string, err error) {
+	return readDirN(dirPath, -1)
+}
+
+// Return count entries at the directory dirPath and all entries
+// if count is set to -1
+func readDirN(dirPath string, count int) (entries []string, err error) {
 	bufp := readDirBufPool.Get().(*[]byte)
 	buf := *bufp
 	defer readDirBufPool.Put(bufp)
@@ -135,7 +141,11 @@ func readDir(dirPath string) (entries []string, err error) {
 	defer d.Close()
 
 	fd := int(d.Fd())
-	for {
+
+	remaining := count
+	done := false
+
+	for !done {
 		nbuf, err := syscall.ReadDirent(fd, buf)
 		if err != nil {
 			return nil, err
@@ -147,6 +157,13 @@ func readDir(dirPath string) (entries []string, err error) {
 		if tmpEntries, err = parseDirents(dirPath, buf[:nbuf]); err != nil {
 			return nil, err
 		}
+		if count > 0 {
+			if remaining <= len(tmpEntries) {
+				tmpEntries = tmpEntries[:remaining]
+				done = true
+			}
+			remaining -= len(tmpEntries)
+		}
 		entries = append(entries, tmpEntries...)
 	}
 	return

cmd/posix-list-dir-others.go

@@ -30,7 +30,12 @@ import (
 
 // Return all the entries at the directory dirPath.
 func readDir(dirPath string) (entries []string, err error) {
-	d, err := os.Open((dirPath))
+	return readDirN(dirPath, -1)
+}
+
+// Return N entries at the directory dirPath. If count is -1, return all entries
+func readDirN(dirPath string, count int) (entries []string, err error) {
+	d, err := os.Open(dirPath)
 	if err != nil {
 		// File is really not found.
 		if os.IsNotExist(err) {
@@ -45,20 +50,34 @@ func readDir(dirPath string) (entries []string, err error) {
 	}
 	defer d.Close()
 
-	for {
-		// Read 1000 entries.
-		fis, err := d.Readdir(1000)
+	maxEntries := 1000
+	if count > 0 && count < maxEntries {
+		maxEntries = count
+	}
+
+	done := false
+	remaining := count
+
+	for !done {
+		// Read up to max number of entries.
+		fis, err := d.Readdir(maxEntries)
 		if err != nil {
 			if err == io.EOF {
 				break
 			}
 			return nil, err
 		}
+		if count > 0 {
+			if remaining <= len(fis) {
+				fis = fis[:remaining]
+				done = true
+			}
+		}
 		for _, fi := range fis {
 			// Stat symbolic link and follow to get the final value.
 			if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
 				var st os.FileInfo
-				st, err = os.Stat((path.Join(dirPath, fi.Name())))
+				st, err = os.Stat(path.Join(dirPath, fi.Name()))
 				if err != nil {
 					reqInfo := (&logger.ReqInfo{}).AppendTags("path", path.Join(dirPath, fi.Name()))
 					ctx := logger.SetReqInfo(context.Background(), reqInfo)
@@ -71,6 +90,9 @@ func readDir(dirPath string) (entries []string, err error) {
 				} else if st.Mode().IsRegular() {
 					entries = append(entries, fi.Name())
 				}
+				if count > 0 {
+					remaining--
+				}
 				continue
 			}
 			if fi.Mode().IsDir() {
@@ -79,6 +101,9 @@ func readDir(dirPath string) (entries []string, err error) {
 			} else if fi.Mode().IsRegular() {
 				entries = append(entries, fi.Name())
 			}
+			if count > 0 {
+				remaining--
+			}
 		}
 	}
 	return entries, nil

cmd/posix-list-dir_test.go

@@ -201,3 +201,41 @@ func TestReadDir(t *testing.T) {
 		}
 	}
 }
+
+func TestReadDirN(t *testing.T) {
+	testCases := []struct {
+		numFiles    int
+		n           int
+		expectedNum int
+	}{
+		{0, 0, 0},
+		{0, 1, 0},
+		{1, 0, 1},
+		{0, -1, 0},
+		{1, -1, 1},
+		{10, -1, 10},
+		{1, 1, 1},
+		{2, 1, 1},
+		{10, 9, 9},
+		{10, 10, 10},
+		{10, 11, 10},
+	}
+
+	for i, testCase := range testCases {
+		dir := mustSetupDir(t)
+		defer os.RemoveAll(dir)
+
+		for c := 1; c <= testCase.numFiles; c++ {
+			if err := ioutil.WriteFile(filepath.Join(dir, fmt.Sprintf("%d", c)), []byte{}, os.ModePerm); err != nil {
+				t.Fatalf("Unable to create a file, %s", err)
+			}
+		}
+		entries, err := readDirN(dir, testCase.n)
+		if err != nil {
+			t.Fatalf("Unable to read entries, %s", err)
+		}
+		if len(entries) != testCase.expectedNum {
+			t.Fatalf("Test %d: unexpected number of entries, waiting for %d, but found %d", i+1, testCase.expectedNum, len(entries))
+		}
+	}
+}

cmd/posix.go

@@ -261,8 +261,8 @@ func (s *posix) DiskInfo() (info disk.Info, err error) {
 // compatible way for all operating systems. If volume is not found
 // an error is generated.
 func (s *posix) getVolDir(volume string) (string, error) {
-	if !isValidVolname(volume) {
-		return "", errInvalidArgument
+	if volume == "" || volume == "." || volume == ".." {
+		return "", errVolumeNotFound
 	}
 	volumeDir := pathJoin(s.diskPath, volume)
 	return volumeDir, nil
@@ -301,6 +301,10 @@ func (s *posix) MakeVol(volume string) (err error) {
 		return err
 	}
 
+	if !isValidVolname(volume) {
+		return errInvalidArgument
+	}
+
 	volumeDir, err := s.getVolDir(volume)
 	if err != nil {
 		return err
@@ -463,7 +467,7 @@ func (s *posix) DeleteVol(volume string) (err error) {
 
 // ListDir - return all the entries at the given directory path.
 // If an entry is a directory it will be returned with a trailing "/".
-func (s *posix) ListDir(volume, dirPath string) (entries []string, err error) {
+func (s *posix) ListDir(volume, dirPath string, count int) (entries []string, err error) {
 	defer func() {
 		if err == syscall.EIO {
 			atomic.AddInt32(&s.ioErrCount, 1)
@@ -491,7 +495,12 @@ func (s *posix) ListDir(volume, dirPath string) (entries []string, err error) {
 		}
 		return nil, err
 	}
-	return readDir(pathJoin(volumeDir, dirPath))
+
+	dirPath = pathJoin(volumeDir, dirPath)
+	if count > 0 {
+		return readDirN(dirPath, count)
+	}
+	return readDir(dirPath)
 }
 
 // ReadAll reads from r until an error or EOF and returns the data it read.

cmd/posix_test.go

@@ -231,7 +231,7 @@ func TestPosixReadAll(t *testing.T) {
 		{
 			volume: "ab",
 			path:   "as-file",
-			err:    errInvalidArgument,
+			err:    errVolumeNotFound,
 		},
 	}
 
@@ -477,7 +477,7 @@ func TestPosixDeleteVol(t *testing.T) {
 		{
 			volName:     "ab",
 			ioErrCount:  0,
-			expectedErr: errInvalidArgument,
+			expectedErr: errVolumeNotFound,
 		},
 	}
 
@@ -588,7 +588,7 @@ func TestPosixStatVol(t *testing.T) {
 		{
 			volName:     "ab",
 			ioErrCount:  0,
-			expectedErr: errInvalidArgument,
+			expectedErr: errVolumeNotFound,
 		},
 	}
 
@@ -756,7 +756,7 @@ func TestPosixPosixListDir(t *testing.T) {
 			srcVol:      "ab",
 			srcPath:     "success-file",
 			ioErrCnt:    0,
-			expectedErr: errInvalidArgument,
+			expectedErr: errVolumeNotFound,
 		},
 		// TestPosix case - 4.
 		// TestPosix case with io error count > max limit.
@@ -785,7 +785,7 @@ func TestPosixPosixListDir(t *testing.T) {
 		} else {
 			t.Errorf("Expected the StorageAPI to be of type *posix")
 		}
-		dirList, err = posixStorage.ListDir(testCase.srcVol, testCase.srcPath)
+		dirList, err = posixStorage.ListDir(testCase.srcVol, testCase.srcPath, -1)
 		if err != testCase.expectedErr {
 			t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}
@@ -902,7 +902,7 @@ func TestPosixDeleteFile(t *testing.T) {
 			srcVol:      "my",
 			srcPath:     "success-file",
 			ioErrCnt:    0,
-			expectedErr: errInvalidArgument,
+			expectedErr: errVolumeNotFound,
 		},
 		// TestPosix case - 5.
 		// TestPosix case with non-existent volume.
@@ -1068,7 +1068,7 @@ func TestPosixReadFile(t *testing.T) {
 		},
 		// Empty volume name. - 12
 		{
-			"", "myobject", 14, 1, nil, errInvalidArgument,
+			"", "myobject", 14, 1, nil, errVolumeNotFound,
 		},
 		// Empty filename name. - 13
 		{
@@ -1374,7 +1374,7 @@ func TestPosixAppendFile(t *testing.T) {
 	// TestPosix case with invalid volume name.
 	// A valid volume name should be atleast of size 3.
 	err = posixStorage.AppendFile("bn", "yes", []byte("hello, world"))
-	if err != errInvalidArgument {
+	if err != errVolumeNotFound {
 		t.Fatalf("expected: \"Invalid argument error\", got: \"%s\"", err)
 	}
 
@@ -1472,19 +1472,19 @@ func TestPosixPrepareFile(t *testing.T) {
 		}
 	}
 
-	// TestPosix case with invalid file size which should be strictly positive
-	err = posixStorage.PrepareFile("bn", "yes", -3)
-	if err != errInvalidArgument {
-		t.Fatalf("should fail: %v", err)
-	}
-
 	// TestPosix case with invalid volume name.
 	// A valid volume name should be atleast of size 3.
 	err = posixStorage.PrepareFile("bn", "yes", 16)
-	if err != errInvalidArgument {
+	if err != errVolumeNotFound {
 		t.Fatalf("expected: \"Invalid argument error\", got: \"%s\"", err)
 	}
 
+	// TestPosix case with invalid file size which should be strictly positive
+	err = posixStorage.PrepareFile("success-vol", "yes", -3)
+	if err != errInvalidArgument {
+		t.Fatalf("should fail: %v", err)
+	}
+
 	// TestPosix case with IO error count > max limit.
 
 	// setting ioErrCnt to 6.
@@ -1681,7 +1681,7 @@ func TestPosixRenameFile(t *testing.T) {
 			srcPath:     "file4",
 			destPath:    "new-path/",
 			ioErrCnt:    0,
-			expectedErr: errInvalidArgument,
+			expectedErr: errVolumeNotFound,
 		},
 		// TestPosix case - 14.
 		// TestPosix case with invalid destination volume name. Length should be atleast 3.
@@ -1692,7 +1692,7 @@ func TestPosixRenameFile(t *testing.T) {
 			srcPath:     "file4",
 			destPath:    "new-path/",
 			ioErrCnt:    0,
-			expectedErr: errInvalidArgument,
+			expectedErr: errVolumeNotFound,
 		},
 		// TestPosix case - 15.
 		// TestPosix case with invalid destination volume name. Length should be atleast 3.
@@ -1703,7 +1703,7 @@ func TestPosixRenameFile(t *testing.T) {
 			srcPath:     "file4",
 			destPath:    "new-path/",
 			ioErrCnt:    0,
-			expectedErr: errInvalidArgument,
+			expectedErr: errVolumeNotFound,
 		},
 		// TestPosix case - 16.
 		// TestPosix case with the parent of the destination being a file.

cmd/routers.go

@@ -19,7 +19,7 @@ package cmd
 import (
 	"net/http"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 )
 
 func newObjectLayerFn() (layer ObjectLayer) {
@@ -34,27 +34,27 @@ func newCacheObjectsFn() CacheObjectLayer {
 }
 
 // Composed function registering routers for only distributed XL setup.
-func registerDistXLRouters(mux *router.Router, endpoints EndpointList) error {
+func registerDistXLRouters(router *mux.Router, endpoints EndpointList) error {
 	// Register storage rpc router only if its a distributed setup.
-	err := registerStorageRPCRouters(mux, endpoints)
+	err := registerStorageRPCRouters(router, endpoints)
 	if err != nil {
 		return err
 	}
 
 	// Register distributed namespace lock.
-	err = registerDistNSLockRouter(mux, endpoints)
+	err = registerDistNSLockRouter(router, endpoints)
 	if err != nil {
 		return err
 	}
 
 	// Register S3 peer communication router.
-	err = registerS3PeerRPCRouter(mux)
+	err = registerS3PeerRPCRouter(router)
 	if err != nil {
 		return err
 	}
 
 	// Register RPC router for web related calls.
-	return registerBrowserPeerRPCRouter(mux)
+	return registerBrowserPeerRPCRouter(router)
 }
 
 // List of some generic handlers which are applied for all incoming requests.
@@ -100,38 +100,38 @@ var globalHandlers = []HandlerFunc{
 func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
 	// Initialize router. `SkipClean(true)` stops gorilla/mux from
 	// normalizing URL path minio/minio#3256
-	mux := router.NewRouter().SkipClean(true)
+	router := mux.NewRouter().SkipClean(true)
 
 	// Initialize distributed NS lock.
 	if globalIsDistXL {
-		registerDistXLRouters(mux, endpoints)
+		registerDistXLRouters(router, endpoints)
 	}
 
 	// Add Admin RPC router
-	err := registerAdminRPCRouter(mux)
+	err := registerAdminRPCRouter(router)
 	if err != nil {
 		return nil, err
 	}
 
 	// Add Admin router.
-	registerAdminRouter(mux)
+	registerAdminRouter(router)
 
 	// Add healthcheck router
-	registerHealthCheckRouter(mux)
+	registerHealthCheckRouter(router)
 
 	// Add server metrics router
-	registerMetricsRouter(mux)
+	registerMetricsRouter(router)
 
 	// Register web router when its enabled.
 	if globalIsBrowserEnabled {
-		if err := registerWebRouter(mux); err != nil {
+		if err := registerWebRouter(router); err != nil {
 			return nil, err
 		}
 	}
 
 	// Add API router.
-	registerAPIRouter(mux)
+	registerAPIRouter(router)
 
 	// Register rest of the handlers.
-	return registerHandlers(mux, globalHandlers...), nil
+	return registerHandlers(router, globalHandlers...), nil
 }

cmd/rpc-server_test.go

@@ -22,7 +22,7 @@ import (
 	"os"
 	"testing"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 )
 
 type ArithArgs struct {
@@ -49,10 +49,10 @@ func TestGoHTTPRPC(t *testing.T) {
 		AuthRPCServer: AuthRPCServer{},
 	})
 
-	mux := router.NewRouter().SkipClean(true)
-	mux.Path("/foo").Handler(newServer)
+	router := mux.NewRouter().SkipClean(true)
+	router.Path("/foo").Handler(newServer)
 
-	httpServer := httptest.NewServer(mux)
+	httpServer := httptest.NewServer(router)
 	defer httpServer.Close()
 
 	rootPath, err := newTestConfig("us-east-1")

cmd/server-main.go

@@ -18,15 +18,15 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"os"
 	"os/signal"
-	"runtime"
 	"syscall"
 
 	"github.com/minio/cli"
 	"github.com/minio/dsync"
-	miniohttp "github.com/minio/minio/cmd/http"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 )
 
@@ -118,26 +118,26 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 
 	// Server address.
 	serverAddr := ctx.String("address")
-	logger.FatalIf(CheckLocalServerAddr(serverAddr), "Invalid address ‘%s’ in command line argument.", serverAddr)
+	logger.FatalIf(CheckLocalServerAddr(serverAddr), "Unable to validate passed arguments")
 
 	var setupType SetupType
 	var err error
 
 	if len(ctx.Args()) > serverCommandLineArgsMax {
-		logger.FatalIf(errInvalidArgument, "Invalid total number of arguments (%d) passed, supported upto 32 unique arguments", len(ctx.Args()))
+		uErr := uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Invalid total number of endpoints (%d) passed, supported upto 32 unique arguments", len(ctx.Args())))
+		logger.FatalIf(uErr, "Unable to validate passed endpoints")
 	}
 
 	globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(serverAddr, ctx.Args()...)
-	logger.FatalIf(err, "Invalid command line arguments server=‘%s’, args=%s", serverAddr, ctx.Args())
+	logger.FatalIf(err, "Invalid command line arguments")
 
 	globalMinioHost, globalMinioPort = mustSplitHostPort(globalMinioAddr)
-	if runtime.GOOS == "darwin" {
-		// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
-		// to IPv6 address ie minio will start listening on IPv6 address whereas another
-		// (non-)minio process is listening on IPv4 of given port.
-		// To avoid this error sutiation we check for port availability only for macOS.
-		logger.FatalIf(checkPortAvailability(globalMinioPort), "Port %d already in use", globalMinioPort)
-	}
+
+	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
+	// to IPv6 address ie minio will start listening on IPv6 address whereas another
+	// (non-)minio process is listening on IPv4 of given port.
+	// To avoid this error sutiation we check for port availability.
+	logger.FatalIf(checkPortAvailability(globalMinioPort), "Unable to start the server")
 
 	globalIsXL = (setupType == XLSetupType)
 	globalIsDistXL = (setupType == DistXLSetupType)
@@ -168,6 +168,10 @@ func serverMain(ctx *cli.Context) {
 		cli.ShowCommandHelpAndExit(ctx, "server", 1)
 	}
 
+	// Disable logging until server initialization is complete, any
+	// error during initialization will be shown as a fatal message
+	logger.Disable = true
+
 	// Get "json" flag from command line argument and
 	// enable json and quite modes if jason flag is turned on.
 	jsonFlag := ctx.IsSet("json") || ctx.GlobalIsSet("json")
@@ -181,6 +185,8 @@ func serverMain(ctx *cli.Context) {
 		logger.EnableQuiet()
 	}
 
+	logger.RegisterUIError(fmtError)
+
 	// Handle all server command args.
 	serverHandleCmdArgs(ctx)
 
@@ -188,7 +194,7 @@ func serverMain(ctx *cli.Context) {
 	serverHandleEnvVars()
 
 	// Create certs path.
-	logger.FatalIf(createConfigDir(), "Unable to create configuration directories.")
+	logger.FatalIf(createConfigDir(), "Unable to initialize configuration files")
 
 	// Initialize server config.
 	initConfig()
@@ -196,15 +202,15 @@ func serverMain(ctx *cli.Context) {
 	// Check and load SSL certificates.
 	var err error
 	globalPublicCerts, globalRootCAs, globalTLSCertificate, globalIsSSL, err = getSSLConfig()
-	logger.FatalIf(err, "Invalid SSL certificate file")
+	logger.FatalIf(err, "Unable to load the TLS configuration")
 
 	// Is distributed setup, error out if no certificates are found for HTTPS endpoints.
 	if globalIsDistXL {
 		if globalEndpoints.IsHTTPS() && !globalIsSSL {
-			logger.FatalIf(errInvalidArgument, "No certificates found, use HTTP endpoints (%s)", globalEndpoints)
+			logger.Fatal(uiErrNoCertsAndHTTPSEndpoints(nil), "Unable to start the server")
 		}
 		if !globalEndpoints.IsHTTPS() && globalIsSSL {
-			logger.FatalIf(errInvalidArgument, "TLS Certificates found, use HTTPS endpoints (%s)", globalEndpoints)
+			logger.Fatal(uiErrCertsAndHTTPEndpoints(nil), "Unable to start the server")
 		}
 	}
 
@@ -225,7 +231,9 @@ func serverMain(ctx *cli.Context) {
 	// Set nodes for dsync for distributed setup.
 	if globalIsDistXL {
 		globalDsync, err = dsync.New(newDsyncNodes(globalEndpoints))
-		logger.FatalIf(err, "Unable to initialize distributed locking on %s", globalEndpoints)
+		if err != nil {
+			logger.Fatal(err, "Unable to initialize distributed locking on %s", globalEndpoints)
+		}
 	}
 
 	// Initialize name space lock.
@@ -237,16 +245,23 @@ func serverMain(ctx *cli.Context) {
 	// Configure server.
 	var handler http.Handler
 	handler, err = configureServerHandler(globalEndpoints)
-	logger.FatalIf(err, "Unable to configure one of server's RPC services.")
+	if err != nil {
+		logger.Fatal(uiErrUnexpectedError(err), "Unable to configure one of server's RPC services")
+	}
 
-	// Initialize notification system.
+	// Create new notification system.
 	globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
-	logger.FatalIf(err, "Unable to initialize notification system.")
+	if err != nil {
+		logger.Fatal(err, "Unable to initialize the notification system")
+	}
+
+	// Create new policy system.
+	globalPolicySys = NewPolicySys()
 
 	// Initialize Admin Peers inter-node communication only in distributed setup.
 	initGlobalAdminPeers(globalEndpoints)
 
-	globalHTTPServer = miniohttp.NewServer([]string{globalMinioAddr}, handler, globalTLSCertificate)
+	globalHTTPServer = xhttp.NewServer([]string{globalMinioAddr}, handler, globalTLSCertificate)
 	globalHTTPServer.ReadTimeout = globalConnReadTimeout
 	globalHTTPServer.WriteTimeout = globalConnWriteTimeout
 	globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
@@ -259,10 +274,8 @@ func serverMain(ctx *cli.Context) {
 
 	newObject, err := newObjectLayer(globalEndpoints)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
-		err = globalHTTPServer.Shutdown()
-		logger.LogIf(context.Background(), err)
-		os.Exit(1)
+		globalHTTPServer.Shutdown()
+		logger.FatalIf(err, "Unable to initialize backend")
 	}
 
 	globalObjLayerMutex.Lock()
@@ -276,6 +289,9 @@ func serverMain(ctx *cli.Context) {
 	// Set uptime time after object layer has initialized.
 	globalBootTime = UTCNow()
 
+	// Re-enable logging
+	logger.Disable = false
+
 	handleSignals()
 }
 

cmd/server_test.go

@@ -36,6 +36,7 @@ import (
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/pkg/policy"
 )
 
 // API suite container common to both FS and XL.
@@ -319,7 +320,7 @@ func (s *TestSuiteCommon) TestBucketSQSNotificationAMQP(c *check) {
 // Deletes the policy and verifies the deletion by fetching it back.
 func (s *TestSuiteCommon) TestBucketPolicy(c *check) {
 	// Sample bucket policy.
-	bucketPolicyBuf := `{"Version":"2012-10-17","Statement":[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s"],"Sid":""},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s/this*"],"Sid":""}]}`
+	bucketPolicyBuf := `{"Version":"2012-10-17","Statement":[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s"]},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s/this*"]}]}`
 
 	// generate a random bucket Name.
 	bucketName := getRandomBucketName()
@@ -361,7 +362,11 @@ func (s *TestSuiteCommon) TestBucketPolicy(c *check) {
 	bucketPolicyReadBuf, err := ioutil.ReadAll(response.Body)
 	c.Assert(err, nil)
 	// Verify if downloaded policy matches with previousy uploaded.
-	c.Assert(bytes.Equal([]byte(bucketPolicyStr), bucketPolicyReadBuf), true)
+	expectedPolicy, err := policy.ParseConfig(strings.NewReader(bucketPolicyStr), bucketName)
+	c.Assert(err, nil)
+	gotPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), bucketName)
+	c.Assert(err, nil)
+	c.Assert(reflect.DeepEqual(expectedPolicy, gotPolicy), true)
 
 	// Delete policy.
 	request, err = newTestSignedRequest("DELETE", getDeletePolicyURL(s.endPoint, bucketName), 0, nil,

cmd/storage-class.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"strconv"
 	"strings"
@@ -104,20 +103,20 @@ func parseStorageClass(storageClassEnv string) (sc storageClass, err error) {
 
 	// only two elements allowed in the string - "scheme" and "number of parity disks"
 	if len(s) > 2 {
-		return storageClass{}, errors.New("Too many sections in " + storageClassEnv)
+		return storageClass{}, uiErrStorageClassValue(nil).Msg("Too many sections in " + storageClassEnv)
 	} else if len(s) < 2 {
-		return storageClass{}, errors.New("Too few sections in " + storageClassEnv)
+		return storageClass{}, uiErrStorageClassValue(nil).Msg("Too few sections in " + storageClassEnv)
 	}
 
 	// only allowed scheme is "EC"
 	if s[0] != supportedStorageClassScheme {
-		return storageClass{}, errors.New("Unsupported scheme " + s[0] + ". Supported scheme is EC")
+		return storageClass{}, uiErrStorageClassValue(nil).Msg("Unsupported scheme " + s[0] + ". Supported scheme is EC")
 	}
 
 	// Number of parity disks should be integer
 	parityDisks, err := strconv.Atoi(s[1])
 	if err != nil {
-		return storageClass{}, err
+		return storageClass{}, uiErrStorageClassValue(err)
 	}
 
 	sc = storageClass{

cmd/storage-class_test.go

@@ -69,8 +69,8 @@ func testParseStorageClass(obj ObjectLayer, instanceType string, t TestErrHandle
 			t.Errorf("Test %d, Expected %v, got %v", i+1, tt.wantSc, gotSc)
 			return
 		}
-		if tt.expectedError != nil && !reflect.DeepEqual(err, tt.expectedError) {
-			t.Errorf("Test %d, Expected %v, got %v", i+1, tt.expectedError, err)
+		if tt.expectedError != nil && err.Error() != tt.expectedError.Error() {
+			t.Errorf("Test %d, Expected `%v`, got `%v`", i+1, tt.expectedError, err)
 		}
 	}
 }

cmd/storage-interface.go

@@ -39,7 +39,7 @@ type StorageAPI interface {
 	DeleteVol(volume string) (err error)
 
 	// File operations.
-	ListDir(volume, dirPath string) ([]string, error)
+	ListDir(volume, dirPath string, count int) ([]string, error)
 	ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error)
 	PrepareFile(volume string, path string, len int64) (err error)
 	AppendFile(volume string, path string, buf []byte) (err error)

cmd/storage-rpc-client.go

@@ -283,10 +283,11 @@ func (n *networkStorage) ReadFile(volume string, path string, offset int64, buff
 }
 
 // ListDir - list all entries at prefix.
-func (n *networkStorage) ListDir(volume, path string) (entries []string, err error) {
+func (n *networkStorage) ListDir(volume, path string, count int) (entries []string, err error) {
 	if err = n.call("Storage.ListDirHandler", &ListDirArgs{
-		Vol:  volume,
-		Path: path,
+		Vol:   volume,
+		Path:  path,
+		Count: count,
 	}, &entries); err != nil {
 		return nil, err
 	}

cmd/storage-rpc-client_test.go

@@ -435,7 +435,7 @@ func (s *TestRPCStorageSuite) testRPCStorageListDir(t *testing.T) {
 				t.Error("Unable to initiate MakeVol", err)
 			}
 		}
-		dirs, err := storageDisk.ListDir("myvol", "")
+		dirs, err := storageDisk.ListDir("myvol", "", -1)
 		if err != nil {
 			t.Error(err)
 		}
@@ -448,7 +448,7 @@ func (s *TestRPCStorageSuite) testRPCStorageListDir(t *testing.T) {
 				t.Error("Unable to initiate DeleteVol", err)
 			}
 		}
-		dirs, err = storageDisk.ListDir("myvol", "")
+		dirs, err = storageDisk.ListDir("myvol", "", -1)
 		if err != nil {
 			t.Error(err)
 		}

cmd/storage-rpc-server-datatypes.go

@@ -134,6 +134,9 @@ type ListDirArgs struct {
 
 	// Name of the path.
 	Path string
+
+	// Number of wanted results
+	Count int
 }
 
 // RenameFileArgs represents rename file RPC arguments.

cmd/storage-rpc-server.go

@@ -22,7 +22,7 @@ import (
 	"path"
 	"time"
 
-	router "github.com/gorilla/mux"
+	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/disk"
 )
@@ -120,7 +120,7 @@ func (s *storageServer) ListDirHandler(args *ListDirArgs, reply *[]string) error
 		return err
 	}
 
-	entries, err := s.storage.ListDir(args.Vol, args.Path)
+	entries, err := s.storage.ListDir(args.Vol, args.Path, args.Count)
 	if err != nil {
 		return err
 	}
@@ -221,7 +221,7 @@ func newStorageRPCServer(endpoints EndpointList) (servers []*storageServer, err
 }
 
 // registerStorageRPCRouter - register storage rpc router.
-func registerStorageRPCRouters(mux *router.Router, endpoints EndpointList) error {
+func registerStorageRPCRouters(router *mux.Router, endpoints EndpointList) error {
 	// Initialize storage rpc servers for every disk that is hosted on this node.
 	storageRPCs, err := newStorageRPCServer(endpoints)
 	if err != nil {
@@ -238,7 +238,7 @@ func registerStorageRPCRouters(mux *router.Router, endpoints EndpointList) error
 			return err
 		}
 		// Add minio storage routes.
-		storageRouter := mux.PathPrefix(minioReservedBucketPath).Subrouter()
+		storageRouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
 		storageRouter.Path(path.Join(storageRPCPath, stServer.path)).Handler(storageRPCServer)
 	}
 	return nil

cmd/test-utils_test.go

@@ -52,13 +52,13 @@ import (
 	"time"
 
 	"github.com/fatih/color"
-	router "github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/policy"
+	"github.com/gorilla/mux"
 	"github.com/minio/minio-go/pkg/s3signer"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/bpool"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/policy"
 )
 
 // Tests should initNSLock only once.
@@ -75,7 +75,7 @@ func init() {
 	// Set system resources to maximum.
 	setMaxResources()
 
-	logger.EnableQuiet()
+	logger.Disable = true
 }
 
 // concurreny level for certain parallel tests.
@@ -354,9 +354,12 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
 	globalMinioAddr = getEndpointsLocalAddr(testServer.Disks)
 	globalNotificationSys, err = NewNotificationSys(globalServerConfig, testServer.Disks)
 	if err != nil {
-		t.Fatalf("Unable to initialize queue configuration")
+		t.Fatalf("Unable to create new notification system. %v", err)
 	}
 
+	// Create new policy system.
+	globalPolicySys = NewPolicySys()
+
 	return testServer
 }
 
@@ -398,7 +401,7 @@ func StartTestServer(t TestErrHandler, instanceType string) TestServer {
 // The object Layer will be a temp back used for testing purpose.
 func initTestStorageRPCEndPoint(endpoints EndpointList) http.Handler {
 	// Initialize router.
-	muxRouter := router.NewRouter().SkipClean(true)
+	muxRouter := mux.NewRouter().SkipClean(true)
 	registerStorageRPCRouters(muxRouter, endpoints)
 	return muxRouter
 }
@@ -469,16 +472,16 @@ func StartTestPeersRPCServer(t TestErrHandler, instanceType string) TestServer {
 	testRPCServer.Obj = objLayer
 	globalObjLayerMutex.Unlock()
 
-	mux := router.NewRouter().SkipClean(true)
+	router := mux.NewRouter().SkipClean(true)
 	// need storage layer for bucket config storage.
-	registerStorageRPCRouters(mux, endpoints)
+	registerStorageRPCRouters(router, endpoints)
 	// need API layer to send requests, etc.
-	registerAPIRouter(mux)
+	registerAPIRouter(router)
 	// module being tested is Peer RPCs router.
-	registerS3PeerRPCRouter(mux)
+	registerS3PeerRPCRouter(router)
 
 	// Run TestServer.
-	testRPCServer.Server = httptest.NewServer(mux)
+	testRPCServer.Server = httptest.NewServer(router)
 
 	// initialize remainder of serverCmdConfig
 	testRPCServer.endpoints = endpoints
@@ -1715,17 +1718,14 @@ func newTestObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err erro
 		return xl.storageDisks
 	}
 
-	// Initialize and load bucket policies.
-	xl.bucketPolicies, err = initBucketPolicies(xl)
-	if err != nil {
-		return nil, err
-	}
-
-	// Initialize a new event notifier.
+	// Create new notification system.
 	if globalNotificationSys, err = NewNotificationSys(globalServerConfig, endpoints); err != nil {
 		return nil, err
 	}
 
+	// Create new policy system.
+	globalPolicySys = NewPolicySys()
+
 	return xl, nil
 }
 
@@ -1821,7 +1821,7 @@ func prepareTestBackend(instanceType string) (ObjectLayer, []string, error) {
 //   STEP 2: Set the policy to allow the unsigned request, use the policyFunc to obtain the relevant statement and call
 //           the handler again to verify its success.
 func ExecObjectLayerAPIAnonTest(t *testing.T, obj ObjectLayer, testName, bucketName, objectName, instanceType string, apiRouter http.Handler,
-	anonReq *http.Request, policyFunc func(string, string) policy.Statement) {
+	anonReq *http.Request, bucketPolicy *policy.Policy) {
 
 	anonTestStr := "Anonymous HTTP request test"
 	unknownSignTestStr := "Unknown HTTP signature test"
@@ -1863,7 +1863,8 @@ func ExecObjectLayerAPIAnonTest(t *testing.T, obj ObjectLayer, testName, bucketN
 	// HEAD HTTTP request doesn't contain response body.
 	if anonReq.Method != "HEAD" {
 		// read the response body.
-		actualContent, err := ioutil.ReadAll(rec.Body)
+		var actualContent []byte
+		actualContent, err = ioutil.ReadAll(rec.Body)
 		if err != nil {
 			t.Fatal(failTestStr(anonTestStr, fmt.Sprintf("Failed parsing response body: <ERROR> %v", err)))
 		}
@@ -1872,13 +1873,13 @@ func ExecObjectLayerAPIAnonTest(t *testing.T, obj ObjectLayer, testName, bucketN
 			t.Fatal(failTestStr(anonTestStr, "error response content differs from expected value"))
 		}
 	}
-	// Set write only policy on bucket to allow anonymous HTTP request for the operation under test.
-	// request to go through.
-	bp := policy.BucketAccessPolicy{
-		Version:    "1.0",
-		Statements: []policy.Statement{policyFunc(bucketName, "")},
+
+	if err := obj.SetBucketPolicy(context.Background(), bucketName, bucketPolicy); err != nil {
+		t.Fatalf("unexpected error. %v", err)
 	}
-	obj.SetBucketPolicy(context.Background(), bucketName, bp)
+	globalPolicySys.Set(bucketName, *bucketPolicy)
+	defer globalPolicySys.Remove(bucketName)
+
 	// now call the handler again with the unsigned/anonymous request, it should be accepted.
 	rec = httptest.NewRecorder()
 
@@ -2130,7 +2131,7 @@ func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType)
 	defer removeRoots(erasureDisks)
 }
 
-func registerBucketLevelFunc(bucket *router.Router, api objectAPIHandlers, apiFunctions ...string) {
+func registerBucketLevelFunc(bucket *mux.Router, api objectAPIHandlers, apiFunctions ...string) {
 	for _, apiFunction := range apiFunctions {
 		switch apiFunction {
 		case "PostPolicy":
@@ -2204,14 +2205,14 @@ func registerBucketLevelFunc(bucket *router.Router, api objectAPIHandlers, apiFu
 }
 
 // registerAPIFunctions helper function to add API functions identified by name to the routers.
-func registerAPIFunctions(muxRouter *router.Router, objLayer ObjectLayer, apiFunctions ...string) {
+func registerAPIFunctions(muxRouter *mux.Router, objLayer ObjectLayer, apiFunctions ...string) {
 	if len(apiFunctions) == 0 {
 		// Register all api endpoints by default.
 		registerAPIRouter(muxRouter)
 		return
 	}
 	// API Router.
-	apiRouter := muxRouter.NewRoute().PathPrefix("/").Subrouter()
+	apiRouter := muxRouter.PathPrefix("/").Subrouter()
 	// Bucket router.
 	bucketRouter := apiRouter.PathPrefix("/{bucket}").Subrouter()
 
@@ -2241,7 +2242,7 @@ func registerAPIFunctions(muxRouter *router.Router, objLayer ObjectLayer, apiFun
 func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Handler {
 	// initialize a new mux router.
 	// goriilla/mux is the library used to register all the routes and handle them.
-	muxRouter := router.NewRouter().SkipClean(true)
+	muxRouter := mux.NewRouter().SkipClean(true)
 	if len(apiFunctions) > 0 {
 		// Iterate the list of API functions requested for and register them in mux HTTP handler.
 		registerAPIFunctions(muxRouter, objLayer, apiFunctions...)
@@ -2258,7 +2259,7 @@ func initTestWebRPCEndPoint(objLayer ObjectLayer) http.Handler {
 	globalObjLayerMutex.Unlock()
 
 	// Initialize router.
-	muxRouter := router.NewRouter().SkipClean(true)
+	muxRouter := mux.NewRouter().SkipClean(true)
 	registerWebRouter(muxRouter)
 	return muxRouter
 }
@@ -2266,7 +2267,7 @@ func initTestWebRPCEndPoint(objLayer ObjectLayer) http.Handler {
 // Initialize browser RPC endpoint.
 func initTestBrowserPeerRPCEndPoint() http.Handler {
 	// Initialize router.
-	muxRouter := router.NewRouter().SkipClean(true)
+	muxRouter := mux.NewRouter().SkipClean(true)
 	registerBrowserPeerRPCRouter(muxRouter)
 	return muxRouter
 }
@@ -2320,7 +2321,7 @@ func StartTestS3PeerRPCServer(t TestErrHandler) (TestServer, []string) {
 	globalObjLayerMutex.Unlock()
 
 	// Register router on a new mux
-	muxRouter := router.NewRouter().SkipClean(true)
+	muxRouter := mux.NewRouter().SkipClean(true)
 	err = registerS3PeerRPCRouter(muxRouter)
 	if err != nil {
 		t.Fatalf("%s", err)

cmd/tree-walk.go

@@ -98,6 +98,9 @@ type listDirFunc func(bucket, prefixDir, prefixEntry string) (entries []string,
 // 4. XL backend multipart listing - isLeaf is true if the entry is a directory and contains uploads.json
 type isLeafFunc func(string, string) bool
 
+// A function isLeafDir of type isLeafDirFunc is used to detect if an entry represents an empty directory.
+type isLeafDirFunc func(string, string) bool
+
 func filterListEntries(bucket, prefixDir string, entries []string, prefixEntry string, isLeaf isLeafFunc) ([]string, bool) {
 	// Listing needs to be sorted.
 	sort.Strings(entries)
@@ -125,7 +128,7 @@ func filterListEntries(bucket, prefixDir string, entries []string, prefixEntry s
 }
 
 // treeWalk walks directory tree recursively pushing treeWalkResult into the channel as and when it encounters files.
-func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker string, recursive bool, listDir listDirFunc, isLeaf isLeafFunc, resultCh chan treeWalkResult, endWalkCh chan struct{}, isEnd bool) error {
+func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker string, recursive bool, listDir listDirFunc, isLeaf isLeafFunc, isLeafDir isLeafDirFunc, resultCh chan treeWalkResult, endWalkCh chan struct{}, isEnd bool) error {
 	// Example:
 	// if prefixDir="one/two/three/" and marker="four/five.txt" treeWalk is recursively
 	// called with prefixDir="one/two/three/four/" and marker="five.txt"
@@ -167,17 +170,30 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
 		return nil
 	}
 	for i, entry := range entries {
+		var leaf, leafDir bool
+
 		// Decision to do isLeaf check was pushed from listDir() to here.
-		if delayIsLeaf && isLeaf(bucket, pathJoin(prefixDir, entry)) {
-			entry = strings.TrimSuffix(entry, slashSeparator)
+		if delayIsLeaf {
+			leaf = isLeaf(bucket, pathJoin(prefixDir, entry))
+			if leaf {
+				entry = strings.TrimSuffix(entry, slashSeparator)
+			}
+		} else {
+			leaf = !strings.HasSuffix(entry, slashSeparator)
 		}
 
+		if strings.HasSuffix(entry, slashSeparator) {
+			leafDir = isLeafDir(bucket, pathJoin(prefixDir, entry))
+		}
+
+		isDir := !leafDir && !leaf
+
 		if i == 0 && markerDir == entry {
 			if !recursive {
 				// Skip as the marker would already be listed in the previous listing.
 				continue
 			}
-			if recursive && !hasSuffix(entry, slashSeparator) {
+			if recursive && !isDir {
 				// We should not skip for recursive listing and if markerDir is a directory
 				// for ex. if marker is "four/five.txt" markerDir will be "four/" which
 				// should not be skipped, instead it will need to be treeWalk()'ed into.
@@ -186,7 +202,7 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
 				continue
 			}
 		}
-		if recursive && hasSuffix(entry, slashSeparator) {
+		if recursive && isDir {
 			// If the entry is a directory, we will need recurse into it.
 			markerArg := ""
 			if entry == markerDir {
@@ -198,7 +214,7 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
 			// markIsEnd is passed to this entry's treeWalk() so that treeWalker.end can be marked
 			// true at the end of the treeWalk stream.
 			markIsEnd := i == len(entries)-1 && isEnd
-			if tErr := doTreeWalk(ctx, bucket, pathJoin(prefixDir, entry), prefixMatch, markerArg, recursive, listDir, isLeaf, resultCh, endWalkCh, markIsEnd); tErr != nil {
+			if tErr := doTreeWalk(ctx, bucket, pathJoin(prefixDir, entry), prefixMatch, markerArg, recursive, listDir, isLeaf, isLeafDir, resultCh, endWalkCh, markIsEnd); tErr != nil {
 				return tErr
 			}
 			continue
@@ -218,7 +234,7 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker
 }
 
 // Initiate a new treeWalk in a goroutine.
-func startTreeWalk(ctx context.Context, bucket, prefix, marker string, recursive bool, listDir listDirFunc, isLeaf isLeafFunc, endWalkCh chan struct{}) chan treeWalkResult {
+func startTreeWalk(ctx context.Context, bucket, prefix, marker string, recursive bool, listDir listDirFunc, isLeaf isLeafFunc, isLeafDir isLeafDirFunc, endWalkCh chan struct{}) chan treeWalkResult {
 	// Example 1
 	// If prefix is "one/two/three/" and marker is "one/two/three/four/five.txt"
 	// treeWalk is called with prefixDir="one/two/three/" and marker="four/five.txt"
@@ -240,7 +256,7 @@ func startTreeWalk(ctx context.Context, bucket, prefix, marker string, recursive
 	marker = strings.TrimPrefix(marker, prefixDir)
 	go func() {
 		isEnd := true // Indication to start walking the tree with end as true.
-		doTreeWalk(ctx, bucket, prefixDir, entryPrefixMatch, marker, recursive, listDir, isLeaf, resultCh, endWalkCh, isEnd)
+		doTreeWalk(ctx, bucket, prefixDir, entryPrefixMatch, marker, recursive, listDir, isLeaf, isLeafDir, resultCh, endWalkCh, isEnd)
 		close(resultCh)
 	}()
 	return resultCh

cmd/tree-walk_test.go

@@ -128,11 +128,11 @@ func createNamespace(disk StorageAPI, volume string, files []string) error {
 
 // Test if tree walker returns entries matching prefix alone are received
 // when a non empty prefix is supplied.
-func testTreeWalkPrefix(t *testing.T, listDir listDirFunc, isLeaf isLeafFunc) {
+func testTreeWalkPrefix(t *testing.T, listDir listDirFunc, isLeaf isLeafFunc, isLeafDir isLeafDirFunc) {
 	// Start the tree walk go-routine.
 	prefix := "d/"
 	endWalkCh := make(chan struct{})
-	twResultCh := startTreeWalk(context.Background(), volume, prefix, "", true, listDir, isLeaf, endWalkCh)
+	twResultCh := startTreeWalk(context.Background(), volume, prefix, "", true, listDir, isLeaf, isLeafDir, endWalkCh)
 
 	// Check if all entries received on the channel match the prefix.
 	for res := range twResultCh {
@@ -143,11 +143,11 @@ func testTreeWalkPrefix(t *testing.T, listDir listDirFunc, isLeaf isLeafFunc) {
 }
 
 // Test if entries received on tree walk's channel appear after the supplied marker.
-func testTreeWalkMarker(t *testing.T, listDir listDirFunc, isLeaf isLeafFunc) {
+func testTreeWalkMarker(t *testing.T, listDir listDirFunc, isLeaf isLeafFunc, isLeafDir isLeafDirFunc) {
 	// Start the tree walk go-routine.
 	prefix := ""
 	endWalkCh := make(chan struct{})
-	twResultCh := startTreeWalk(context.Background(), volume, prefix, "d/g", true, listDir, isLeaf, endWalkCh)
+	twResultCh := startTreeWalk(context.Background(), volume, prefix, "d/g", true, listDir, isLeaf, isLeafDir, endWalkCh)
 
 	// Check if only 3 entries, namely d/g/h, i/j/k, lmn are received on the channel.
 	expectedCount := 3
@@ -187,11 +187,20 @@ func TestTreeWalk(t *testing.T) {
 	isLeaf := func(volume, prefix string) bool {
 		return !hasSuffix(prefix, slashSeparator)
 	}
+
+	isLeafDir := func(volume, prefix string) bool {
+		entries, listErr := disk.ListDir(volume, prefix, 1)
+		if listErr != nil {
+			return false
+		}
+		return len(entries) == 0
+	}
+
 	listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk)
 	// Simple test for prefix based walk.
-	testTreeWalkPrefix(t, listDir, isLeaf)
+	testTreeWalkPrefix(t, listDir, isLeaf, isLeafDir)
 	// Simple test when marker is set.
-	testTreeWalkMarker(t, listDir, isLeaf)
+	testTreeWalkMarker(t, listDir, isLeaf, isLeafDir)
 	err = os.RemoveAll(fsDir)
 	if err != nil {
 		t.Fatal(err)
@@ -222,6 +231,15 @@ func TestTreeWalkTimeout(t *testing.T) {
 	isLeaf := func(volume, prefix string) bool {
 		return !hasSuffix(prefix, slashSeparator)
 	}
+
+	isLeafDir := func(volume, prefix string) bool {
+		entries, listErr := disk.ListDir(volume, prefix, 1)
+		if listErr != nil {
+			return false
+		}
+		return len(entries) == 0
+	}
+
 	listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk)
 
 	// TreeWalk pool with 2 seconds timeout for tree-walk go routines.
@@ -231,7 +249,7 @@ func TestTreeWalkTimeout(t *testing.T) {
 	prefix := ""
 	marker := ""
 	recursive := true
-	resultCh := startTreeWalk(context.Background(), volume, prefix, marker, recursive, listDir, isLeaf, endWalkCh)
+	resultCh := startTreeWalk(context.Background(), volume, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
 
 	params := listParams{
 		bucket:    volume,
@@ -373,6 +391,14 @@ func TestRecursiveTreeWalk(t *testing.T) {
 		return !hasSuffix(prefix, slashSeparator)
 	}
 
+	isLeafDir := func(volume, prefix string) bool {
+		entries, listErr := disk1.ListDir(volume, prefix, 1)
+		if listErr != nil {
+			return false
+		}
+		return len(entries) == 0
+	}
+
 	// Create listDir function.
 	listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk1)
 
@@ -450,7 +476,7 @@ func TestRecursiveTreeWalk(t *testing.T) {
 	for i, testCase := range testCases {
 		for entry := range startTreeWalk(context.Background(), volume,
 			testCase.prefix, testCase.marker, testCase.recursive,
-			listDir, isLeaf, endWalkCh) {
+			listDir, isLeaf, isLeafDir, endWalkCh) {
 			if _, found := testCase.expected[entry.entry]; !found {
 				t.Errorf("Test %d: Expected %s, but couldn't find", i+1, entry.entry)
 			}
@@ -479,6 +505,15 @@ func TestSortedness(t *testing.T) {
 	isLeaf := func(volume, prefix string) bool {
 		return !hasSuffix(prefix, slashSeparator)
 	}
+
+	isLeafDir := func(volume, prefix string) bool {
+		entries, listErr := disk1.ListDir(volume, prefix, 1)
+		if listErr != nil {
+			return false
+		}
+		return len(entries) == 0
+	}
+
 	// Create listDir function.
 	listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk1)
 
@@ -522,7 +557,7 @@ func TestSortedness(t *testing.T) {
 		var actualEntries []string
 		for entry := range startTreeWalk(context.Background(), volume,
 			test.prefix, test.marker, test.recursive,
-			listDir, isLeaf, endWalkCh) {
+			listDir, isLeaf, isLeafDir, endWalkCh) {
 			actualEntries = append(actualEntries, entry.entry)
 		}
 		if !sort.IsSorted(sort.StringSlice(actualEntries)) {
@@ -553,6 +588,15 @@ func TestTreeWalkIsEnd(t *testing.T) {
 	isLeaf := func(volume, prefix string) bool {
 		return !hasSuffix(prefix, slashSeparator)
 	}
+
+	isLeafDir := func(volume, prefix string) bool {
+		entries, listErr := disk1.ListDir(volume, prefix, 1)
+		if listErr != nil {
+			return false
+		}
+		return len(entries) == 0
+	}
+
 	// Create listDir function.
 	listDir := listDirFactory(context.Background(), isLeaf, xlTreeWalkIgnoredErrs, disk1)
 
@@ -595,7 +639,7 @@ func TestTreeWalkIsEnd(t *testing.T) {
 	}
 	for i, test := range testCases {
 		var entry treeWalkResult
-		for entry = range startTreeWalk(context.Background(), volume, test.prefix, test.marker, test.recursive, listDir, isLeaf, endWalkCh) {
+		for entry = range startTreeWalk(context.Background(), volume, test.prefix, test.marker, test.recursive, listDir, isLeaf, isLeafDir, endWalkCh) {
 		}
 		if entry.entry != test.expectedEntry {
 			t.Errorf("Test %d: Expected entry %s, but received %s with the EOF marker", i, test.expectedEntry, entry.entry)

cmd/typed-errors.go

@@ -16,7 +16,9 @@
 
 package cmd
 
-import "errors"
+import (
+	"errors"
+)
 
 // errInvalidArgument means that input argument is invalid.
 var errInvalidArgument = errors.New("Invalid arguments specified")

cmd/ui-errors-utils.go

@@ -0,0 +1,135 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"os"
+)
+
+// uiErr is a structure which contains all information
+// to print a fatal error message in json or pretty mode
+// uiErr implements error so we can use it anywhere
+type uiErr struct {
+	msg    string
+	detail string
+	action string
+	help   string
+}
+
+// Return the error message
+func (u uiErr) Error() string {
+	if u.detail == "" {
+		return u.msg
+	}
+	return u.detail
+}
+
+// Replace the current error's message
+func (u uiErr) Msg(m string, args ...interface{}) uiErr {
+	return uiErr{
+		msg:    fmt.Sprintf(m, args...),
+		detail: u.detail,
+		action: u.action,
+		help:   u.help,
+	}
+}
+
+type uiErrFn func(err error) uiErr
+
+// Create a UI error generator, this is needed to simplify
+// the update of the detailed error message in several places
+// in Minio code
+func newUIErrFn(msg, action, help string) uiErrFn {
+	return func(err error) uiErr {
+		u := uiErr{
+			msg:    msg,
+			action: action,
+			help:   help,
+		}
+		if err != nil {
+			u.detail = err.Error()
+		}
+		return u
+	}
+}
+
+// errorToUIError inspects the passed error and transforms it
+// to the appropriate UI error.
+func errorToUIErr(err error) uiErr {
+	// If this is already a uiErr, do nothing
+	if e, ok := err.(uiErr); ok {
+		return e
+	}
+
+	// Show a generic message for known golang errors
+	switch e := err.(type) {
+	case *net.OpError:
+		if e.Op == "listen" {
+			return uiErrPortAlreadyInUse(e).Msg("Port " + e.Addr.String() + " is already in use")
+
+		}
+	case *os.PathError:
+		if os.IsPermission(e) {
+			return uiErrNoPermissionsToAccessDirFiles(e).Msg("Insufficient permissions to access path, `" + e.Path + "`")
+		}
+	}
+
+	switch err {
+	case io.ErrUnexpectedEOF:
+		return uiErrUnexpectedDataContent(err)
+	default:
+		// Failed to identify what type of error this, return a simple UI error
+		return uiErr{msg: err.Error()}
+	}
+
+}
+
+// fmtError() converts a fatal error message to a more clear error
+// using some colors
+func fmtError(introMsg string, err error, jsonFlag bool) string {
+	renderedTxt := ""
+	uiErr := errorToUIErr(err)
+	// JSON print
+	if jsonFlag {
+		// Message text in json should be simple
+		if uiErr.detail != "" {
+			return uiErr.msg + ": " + uiErr.detail
+		}
+		return uiErr.msg
+	}
+	// Pretty print error message
+	introMsg += ": "
+	if uiErr.msg != "" {
+		introMsg += colorBold(uiErr.msg + ".")
+	} else {
+		introMsg += colorBold(err.Error() + ".")
+	}
+	renderedTxt += colorRed(introMsg) + "\n"
+	// Add action message
+	if uiErr.action != "" {
+		renderedTxt += "> " + colorBgYellow(colorBlack(uiErr.action+".")) + "\n"
+	}
+	// Add help
+	if uiErr.help != "" {
+		renderedTxt += colorBold("HELP:") + "\n"
+		renderedTxt += "  " + uiErr.help
+	}
+	return renderedTxt
+}

cmd/ui-errors.go

@@ -0,0 +1,170 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+var (
+	uiErrInvalidConfig = newUIErrFn(
+		"Invalid value found in the configuration file",
+		"Please ensure a valid value in the configuration file, for more details refer https://docs.minio.io/docs/minio-server-configuration-guide",
+		"",
+	)
+
+	uiErrInvalidBrowserValue = newUIErrFn(
+		"Invalid browser value",
+		"Please check the passed value",
+		"Browser can only accept `on` and `off` values. To disable web browser access, set this value to `off`",
+	)
+
+	uiErrInvalidCacheDrivesValue = newUIErrFn(
+		"Invalid cache drive value",
+		"Please check the value in this ENV variable",
+		"MINIO_CACHE_DRIVES: Mounted drives or directories are delimited by `;`",
+	)
+
+	uiErrInvalidCacheExcludesValue = newUIErrFn(
+		"Invalid cache excludes value",
+		"Please check the passed value",
+		"MINIO_CACHE_EXCLUDE: Cache exclusion patterns are delimited by `;`",
+	)
+
+	uiErrInvalidCacheExpiryValue = newUIErrFn(
+		"Invalid cache expiry value",
+		"Please check the passed value",
+		"MINIO_CACHE_EXPIRY: Valid cache expiry duration is in days.",
+	)
+
+	uiErrInvalidCredentials = newUIErrFn(
+		"Invalid credentials",
+		"Please provide correct credentials",
+		`Access key length should be between minimum 3 characters in length.
+Secret key should be in between 8 and 40 characters.`,
+	)
+
+	uiErrInvalidErasureEndpoints = newUIErrFn(
+		"Invalid endpoint(s) in erasure mode",
+		"Please provide correct combination of local/remote paths",
+		"For more information, please refer to https://docs.minio.io/docs/minio-erasure-code-quickstart-guide",
+	)
+
+	uiErrInvalidNumberOfErasureEndpoints = newUIErrFn(
+		"Invalid total number of endpoints for erasure mode",
+		"Please provide an even number of endpoints greater or equal to 4",
+		"For more information, please refer to https://docs.minio.io/docs/minio-erasure-code-quickstart-guide",
+	)
+
+	uiErrStorageClassValue = newUIErrFn(
+		"Invalid storage class value",
+		"Please check the value",
+		`MINIO_STORAGE_CLASS_STANDARD: Format "EC:<Default_Parity_Standard_Class>" (e.g. "EC:3"). This sets the number of parity disks for Minio server in Standard mode. Objects are stored in Standard mode, if storage class is not defined in Put request.
+MINIO_STORAGE_CLASS_RRS: Format "EC:<Default_Parity_Reduced_Redundancy_Class>" (e.g. "EC:3"). This sets the number of parity disks for Minio server in Reduced Redundancy mode. Objects are stored in Reduced Redundancy mode, if Put request specifies RRS storage class.
+Refer to the link https://github.com/minio/minio/tree/master/docs/erasure/storage-class for more information.`,
+	)
+
+	uiErrUnexpectedBackendVersion = newUIErrFn(
+		"Backend version seems to be too recent",
+		"Please update to the latest Minio version",
+		"",
+	)
+
+	uiErrInvalidAddressFlag = newUIErrFn(
+		"--address input is invalid",
+		"Please check --address parameter",
+		`--address binds to a specific ADDRESS:PORT, ADDRESS can be an  IP or hostname (default port is ':9000')
+    Examples: --address ':443'
+              --address '172.16.34.31:9000'`,
+	)
+
+	uiErrInvalidFSEndpoint = newUIErrFn(
+		"Invalid endpoint for standalone FS mode",
+		"Please check the FS endpoint",
+		`FS mode requires only one writable disk path.
+Example 1:
+   $ minio server /data/minio/`,
+	)
+
+	uiErrUnableToWriteInBackend = newUIErrFn(
+		"Unable to write to the backend",
+		"Please ensure Minio binary has write permissions for the backend",
+		"",
+	)
+
+	uiErrUnableToReadFromBackend = newUIErrFn(
+		"Unable to read from the backend",
+		"Please ensure Minio binary has read permissions for the backend",
+		"",
+	)
+
+	uiErrPortAlreadyInUse = newUIErrFn(
+		"Port is already in use",
+		"Please ensure no other program uses the same address/port",
+		"",
+	)
+
+	uiErrNoPermissionsToAccessDirFiles = newUIErrFn(
+		"Missing permissions to access the specified path",
+		"Please ensure the specified path can be accessed",
+		"",
+	)
+
+	uiErrSSLUnexpectedError = newUIErrFn(
+		"Invalid TLS certificate",
+		"Please check the content of your certificate data",
+		`Only PEM (x.509) format is accepted as valid public & private certificates.`,
+	)
+
+	uiErrSSLUnexpectedData = newUIErrFn(
+		"Invalid TLS certificate",
+		"Please check your certificate",
+		"",
+	)
+
+	uiErrSSLNoPassword = newUIErrFn(
+		"Missing TLS password",
+		"Please set the password to environment variable `"+TLSPrivateKeyPassword+"` so that the private key can be decrypted",
+		"",
+	)
+
+	uiErrNoCertsAndHTTPSEndpoints = newUIErrFn(
+		"HTTPS specified in endpoints, but no TLS certificate is found on the local machine",
+		"Please add a certificate or switch to HTTP.",
+		"Refer to https://docs.minio.io/docs/how-to-secure-access-to-minio-server-with-tls for information about how to load a TLS certificate in your server.",
+	)
+
+	uiErrCertsAndHTTPEndpoints = newUIErrFn(
+		"HTTP specified in endpoints, but the server in the local machine is configured with a TLS certificate",
+		"Please remove the certificate in the configuration directory or switch to HTTPS",
+		"",
+	)
+
+	uiErrSSLWrongPassword = newUIErrFn(
+		"Unable to decrypt the private key using the provided password",
+		"Please set the correct password in environment variable "+TLSPrivateKeyPassword,
+		"",
+	)
+
+	uiErrUnexpectedDataContent = newUIErrFn(
+		"Unexpected data content",
+		"Please contact Minio at https://slack.minio.io",
+		"",
+	)
+
+	uiErrUnexpectedError = newUIErrFn(
+		"Unexpected error",
+		"Please contact Minio at https://slack.minio.io",
+		"",
+	)
+)