Fix broken doc links (#6068)

Fixes https://github.com/minio/minio-go/issues/996

Makefile

@@ -89,7 +89,7 @@ pkg-list:
 # Builds minio and installs it to $GOPATH/bin.
 install: build
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
-	@cp $(PWD)/minio $(GOPATH)/bin/minio
+	@mkdir -p $(GOPATH)/bin && cp $(PWD)/minio $(GOPATH)/bin/minio
 	@echo "Installation successful. To learn more, try \"minio --help\"."
 
 clean:

buildscripts/gen-ldflags.go

@@ -33,6 +33,7 @@ func genLDFlags(version string) string {
 	ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
 	ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]
 	ldflagsStr += " -X github.com/minio/minio/cmd.GOPATH=" + os.Getenv("GOPATH")
+	ldflagsStr += " -X github.com/minio/minio/cmd.GOROOT=" + os.Getenv("GOROOT")
 	return ldflagsStr
 }
 

cmd/admin-rpc-client.go

@@ -179,9 +179,9 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
 
 	for _, hostStr := range GetRemotePeers(endpoints) {
 		host, err := xnet.ParseHost(hostStr)
-		logger.CriticalIf(context.Background(), err)
+		logger.FatalIf(err, "Unable to parse Admin RPC Host", context.Background())
 		rpcClient, err := NewAdminRPCClient(host)
-		logger.CriticalIf(context.Background(), err)
+		logger.FatalIf(err, "Unable to initialize Admin RPC Client", context.Background())
 		adminPeerList = append(adminPeerList, adminPeer{
 			addr:      hostStr,
 			cmdRunner: rpcClient,

cmd/admin-rpc-server.go

@@ -121,7 +121,7 @@ func NewAdminRPCServer() (*xrpc.Server, error) {
 // registerAdminRPCRouter - creates and registers Admin RPC server and its router.
 func registerAdminRPCRouter(router *mux.Router) {
 	rpcServer, err := NewAdminRPCServer()
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to initialize Lock RPC Server", context.Background())
 	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
 	subrouter.Path(adminServiceSubPath).Handler(rpcServer)
 }

cmd/bucket-handlers.go

@@ -282,7 +282,13 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	}
 
 	// Allocate incoming content length bytes.
-	deleteXMLBytes := make([]byte, r.ContentLength)
+	var deleteXMLBytes []byte
+	const maxBodySize = 2 * 1000 * 1024 // The max. XML contains 1000 object names (each at most 1024 bytes long) + XML overhead
+	if r.ContentLength > maxBodySize {  // Only allocated memory for at most 1000 objects
+		deleteXMLBytes = make([]byte, maxBodySize)
+	} else {
+		deleteXMLBytes = make([]byte, r.ContentLength)
+	}
 
 	// Read incoming body XML bytes.
 	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {

cmd/bucket-notification-handlers.go

@@ -224,11 +224,17 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
-	host, e := xnet.ParseHost(r.RemoteAddr)
-	logger.CriticalIf(ctx, e)
+	host, err := xnet.ParseHost(r.RemoteAddr)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
 
-	target, e := target.NewHTTPClientTarget(*host, w)
-	logger.CriticalIf(ctx, e)
+	target, err := target.NewHTTPClientTarget(*host, w)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
 
 	rulesMap := event.NewRulesMap(eventNames, pattern, target.ID())
 
@@ -241,10 +247,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
 	defer globalNotificationSys.RemoveRulesMap(bucketName, rulesMap)
 
-	thisAddr, e := xnet.ParseHost(GetLocalPeer(globalEndpoints))
-	logger.CriticalIf(ctx, e)
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
 
-	if err := SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
+	if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
 		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -259,7 +268,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	<-target.DoneCh
 
-	if err := RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
+	if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
 		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)

cmd/build-constants.go

@@ -22,6 +22,9 @@ var (
 	// GOPATH - GOPATH value at the time of build.
 	GOPATH = ""
 
+	// GOROOT - GOROOT value at the time of build.
+	GOROOT = ""
+
 	// Go get development tag.
 	goGetTag = "DEVELOPMENT.GOGET"
 

cmd/certs.go

@@ -86,10 +86,11 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 		return nil, nil
 	}
 
-	rootCAs, err := x509.SystemCertPool()
-	if err != nil {
-		// In some systems like Windows, system cert pool is not supported.
-		// Hence we create a new cert pool.
+	rootCAs, _ := x509.SystemCertPool()
+	if rootCAs == nil {
+		// In some systems (like Windows) system cert pool is
+		// not supported or no certificates are present on the
+		// system - so we create a new cert pool.
 		rootCAs = x509.NewCertPool()
 	}
 

cmd/fs-v1-helpers.go

@@ -178,7 +178,6 @@ func fsStatVolume(ctx context.Context, volume string) (os.FileInfo, error) {
 	}
 
 	if !fi.IsDir() {
-		logger.LogIf(ctx, errVolumeAccessDenied)
 		return nil, errVolumeAccessDenied
 	}
 

cmd/gateway-common.go

@@ -154,6 +154,7 @@ func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
 		UserDefined:     userDefined,
 		ContentType:     oi.ContentType,
 		ContentEncoding: oi.Metadata.Get("Content-Encoding"),
+		StorageClass:    oi.StorageClass,
 	}
 }
 

cmd/gateway-main.go

@@ -34,7 +34,7 @@ import (
 )
 
 func init() {
-	logger.Init(GOPATH)
+	logger.Init(GOPATH, GOROOT)
 	logger.RegisterUIError(fmtError)
 }
 

cmd/gateway/azure/gateway-azure.go

@@ -238,6 +238,8 @@ func s3MetaToAzureProperties(ctx context.Context, s3Metadata map[string]string)
 			props.ContentMD5 = v
 		case k == "Content-Type":
 			props.ContentType = v
+		case k == "Content-Language":
+			props.ContentLanguage = v
 		}
 	}
 	return blobMeta, props, nil
@@ -292,6 +294,9 @@ func azurePropertiesToS3Meta(meta storage.BlobMetadata, props storage.BlobProper
 	if props.ContentType != "" {
 		s3Metadata["Content-Type"] = props.ContentType
 	}
+	if props.ContentLanguage != "" {
+		s3Metadata["Content-Language"] = props.ContentLanguage
+	}
 	return s3Metadata
 }
 

cmd/gateway/s3/gateway-s3.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -97,12 +97,16 @@ EXAMPLES:
 
 // Handler for 'minio gateway s3' command line.
 func s3GatewayMain(ctx *cli.Context) {
-	// Validate gateway arguments.
-	host := ctx.Args().First()
-	// Validate gateway arguments.
-	logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
+	args := ctx.Args()
+	if !ctx.Args().Present() {
+		args = cli.Args{"https://s3.amazonaws.com"}
+	}
 
-	minio.StartGateway(ctx, &S3{host})
+	// Validate gateway arguments.
+	logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), args.First()), "Invalid argument")
+
+	// Start the gateway..
+	minio.StartGateway(ctx, &S3{args.First()})
 }
 
 // S3 implements Gateway.
@@ -115,34 +119,46 @@ func (g *S3) Name() string {
 	return s3Backend
 }
 
-// NewGatewayLayer returns s3 ObjectLayer.
-func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
-	var err error
-	var endpoint string
-	var secure = true
+// newS3 - Initializes a new client by auto probing S3 server signature.
+func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
+	if url == "" {
+		url = "https://s3.amazonaws.com"
+	}
 
-	// Validate host parameters.
-	if g.host != "" {
-		// Override default params if the host is provided
-		endpoint, secure, err = minio.ParseGatewayEndpoint(g.host)
+	// Override default params if the host is provided
+	endpoint, secure, err := minio.ParseGatewayEndpoint(url)
+	if err != nil {
+		return nil, err
+	}
+
+	clnt, err := miniogo.NewV4(endpoint, accessKey, secretKey, secure)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err = clnt.BucketExists("probe-bucket-sign"); err != nil {
+		clnt, err = miniogo.NewV2(endpoint, accessKey, secretKey, secure)
 		if err != nil {
 			return nil, err
 		}
+		if _, err = clnt.BucketExists("probe-bucket-sign"); err != nil {
+			return nil, err
+		}
 	}
 
-	// Default endpoint parameters
-	if endpoint == "" {
-		endpoint = "s3.amazonaws.com"
-	}
+	return &miniogo.Core{Client: clnt}, nil
+}
 
-	// Initialize minio client object.
-	client, err := miniogo.NewCore(endpoint, creds.AccessKey, creds.SecretKey, secure)
+// NewGatewayLayer returns s3 ObjectLayer.
+func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
+	// Probe S3 signature with input credentials.
+	clnt, err := newS3(g.host, creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		return nil, err
 	}
 
 	return &s3Objects{
-		Client: client,
+		Client: clnt,
 	}, nil
 }
 
@@ -255,7 +271,7 @@ func (l *s3Objects) ListObjects(ctx context.Context, bucket string, prefix strin
 
 // ListObjectsV2 lists all blobs in S3 bucket filtered by prefix
 func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, e error) {
-	result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys)
+	result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys, startAfter)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return loi, minio.ErrorRespToObjectError(err, bucket)

cmd/generic-handlers.go

@@ -691,7 +691,7 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
 // cancelled immediately.
 func setRateLimitHandler(h http.Handler) http.Handler {
 	_, maxLimit, err := sys.GetMaxOpenFileLimit()
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to get maximum open file limit", context.Background())
 	// Burst value is set to 1 to allow only maxOpenFileLimit
 	// requests to happen at once.
 	l := rate.NewLimiter(rate.Limit(maxLimit), 1)

cmd/lock-rpc-server.go

@@ -124,7 +124,10 @@ func (l *lockRPCReceiver) lockMaintenance(interval time.Duration) {
 	for _, nlrip := range nlripLongLived {
 		// Initialize client based on the long live locks.
 		host, err := xnet.ParseHost(nlrip.lri.node)
-		logger.CriticalIf(context.Background(), err)
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			continue
+		}
 		c, err := NewLockRPCClient(host)
 		if err != nil {
 			logger.LogIf(context.Background(), err)
@@ -183,7 +186,7 @@ func NewLockRPCServer() (*xrpc.Server, error) {
 // Register distributed NS lock handlers.
 func registerDistNSLockRouter(router *mux.Router) {
 	rpcServer, err := NewLockRPCServer()
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to initialize Lock RPC Server", context.Background())
 
 	// Start lock maintenance from all lock servers.
 	go startLockMaintenance(globalLockServer)

cmd/logger/logger.go

@@ -52,6 +52,25 @@ var matchingFuncNames = [...]string{
 	"http.HandlerFunc.ServeHTTP",
 	"cmd.serverMain",
 	"cmd.StartGateway",
+	"cmd.(*webAPIHandlers).ListBuckets",
+	"cmd.(*webAPIHandlers).MakeBucket",
+	"cmd.(*webAPIHandlers).DeleteBucket",
+	"cmd.(*webAPIHandlers).ListObjects",
+	"cmd.(*webAPIHandlers).RemoveObject",
+	"cmd.(*webAPIHandlers).Login",
+	"cmd.(*webAPIHandlers).GenerateAuth",
+	"cmd.(*webAPIHandlers).SetAuth",
+	"cmd.(*webAPIHandlers).GetAuth",
+	"cmd.(*webAPIHandlers).CreateURLToken",
+	"cmd.(*webAPIHandlers).Upload",
+	"cmd.(*webAPIHandlers).Download",
+	"cmd.(*webAPIHandlers).DownloadZip",
+	"cmd.(*webAPIHandlers).GetBucketPolicy",
+	"cmd.(*webAPIHandlers).ListAllBucketPolicies",
+	"cmd.(*webAPIHandlers).SetBucketPolicy",
+	"cmd.(*webAPIHandlers).PresignedGet",
+	"cmd.(*webAPIHandlers).ServerInfo",
+	"cmd.(*webAPIHandlers).StorageInfo",
 	// add more here ..
 }
 
@@ -142,22 +161,24 @@ func RegisterUIError(f func(string, error, bool) string) {
 // and GOROOT directories. Also append github.com/minio/minio
 // This is done to clean up the filename, when stack trace is
 // displayed when an error happens.
-func Init(goPath string) {
+func Init(goPath string, goRoot string) {
 
 	var goPathList []string
+	var goRootList []string
 	var defaultgoPathList []string
+	var defaultgoRootList []string
+	pathSeperator := ":"
 	// Add all possible GOPATH paths into trimStrings
 	// Split GOPATH depending on the OS type
 	if runtime.GOOS == "windows" {
-		goPathList = strings.Split(goPath, ";")
-		defaultgoPathList = strings.Split(build.Default.GOPATH, ";")
-	} else {
-		// All other types of OSs
-		goPathList = strings.Split(goPath, ":")
-		defaultgoPathList = strings.Split(build.Default.GOPATH, ":")
-
+		pathSeperator = ";"
 	}
 
+	goPathList = strings.Split(goPath, pathSeperator)
+	goRootList = strings.Split(goRoot, pathSeperator)
+	defaultgoPathList = strings.Split(build.Default.GOPATH, pathSeperator)
+	defaultgoRootList = strings.Split(build.Default.GOROOT, pathSeperator)
+
 	// Add trim string "{GOROOT}/src/" into trimStrings
 	trimStrings = []string{filepath.Join(runtime.GOROOT(), "src") + string(filepath.Separator)}
 
@@ -167,10 +188,21 @@ func Init(goPath string) {
 		trimStrings = append(trimStrings, filepath.Join(goPathString, "src")+string(filepath.Separator))
 	}
 
+	for _, goRootString := range goRootList {
+		trimStrings = append(trimStrings, filepath.Join(goRootString, "src")+string(filepath.Separator))
+	}
+
 	for _, defaultgoPathString := range defaultgoPathList {
 		trimStrings = append(trimStrings, filepath.Join(defaultgoPathString, "src")+string(filepath.Separator))
 	}
 
+	for _, defaultgoRootString := range defaultgoRootList {
+		trimStrings = append(trimStrings, filepath.Join(defaultgoRootString, "src")+string(filepath.Separator))
+	}
+
+	// Remove duplicate entries.
+	trimStrings = uniqueEntries(trimStrings)
+
 	// Add "github.com/minio/minio" as the last to cover
 	// paths like "{GOROOT}/src/github.com/minio/minio"
 	// and "{GOPATH}/src/github.com/minio/minio"
@@ -200,7 +232,7 @@ func getTrace(traceLevel int) []string {
 	var trace []string
 	pc, file, lineNumber, ok := runtime.Caller(traceLevel)
 
-	for ok {
+	for ok && file != "" {
 		// Clean up the common prefixes
 		file = trimTrace(file)
 		// Get the function name

cmd/logger/utils.go

@@ -50,3 +50,16 @@ func ansiSaveAttributes() {
 func ansiRestoreAttributes() {
 	ansiEscape("8")
 }
+
+func uniqueEntries(paths []string) []string {
+	found := map[string]bool{}
+	unqiue := []string{}
+
+	for v := range paths {
+		if _, ok := found[paths[v]]; !ok {
+			found[paths[v]] = true
+			unqiue = append(unqiue, paths[v])
+		}
+	}
+	return unqiue
+}

cmd/namespace-lock.go

@@ -86,9 +86,9 @@ func newDsyncNodes(endpoints EndpointList) (clnts []dsync.NetLocker, myNode int)
 			locker = &(receiver.ll)
 		} else {
 			host, err := xnet.ParseHost(endpoint.Host)
-			logger.CriticalIf(context.Background(), err)
+			logger.FatalIf(err, "Unable to parse Lock RPC Host", context.Background())
 			locker, err = NewLockRPCClient(host)
-			logger.CriticalIf(context.Background(), err)
+			logger.FatalIf(err, "Unable to initialize Lock RPC Client", context.Background())
 		}
 
 		clnts = append(clnts, locker)

cmd/peer-rpc-client.go

@@ -166,9 +166,9 @@ func makeRemoteRPCClients(endpoints EndpointList) map[xnet.Host]*PeerRPCClient {
 	peerRPCClientMap := make(map[xnet.Host]*PeerRPCClient)
 	for _, hostStr := range GetRemotePeers(endpoints) {
 		host, err := xnet.ParseHost(hostStr)
-		logger.CriticalIf(context.Background(), err)
+		logger.FatalIf(err, "Unable to parse peer RPC Host", context.Background())
 		rpcClient, err := NewPeerRPCClient(host)
-		logger.CriticalIf(context.Background(), err)
+		logger.FatalIf(err, "Unable to parse peer RPC Client", context.Background())
 		peerRPCClientMap[*host] = rpcClient
 	}
 

cmd/peer-rpc-server.go

@@ -201,7 +201,7 @@ func NewPeerRPCServer() (*xrpc.Server, error) {
 // registerPeerRPCRouter - creates and registers Peer RPC server and its router.
 func registerPeerRPCRouter(router *mux.Router) {
 	rpcServer, err := NewPeerRPCServer()
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to initialize peer RPC Server", context.Background())
 	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
 	subrouter.Path(peerServiceSubPath).Handler(rpcServer)
 }

cmd/posix.go

@@ -135,10 +135,10 @@ func getValidPath(path string) (string, error) {
 	if err != nil {
 		return path, err
 	}
+	defer os.Remove(pathJoin(path, ".writable-check.tmp"))
 	file.Close()
 
-	err = os.Remove(pathJoin(path, ".writable-check.tmp"))
-	return path, err
+	return path, nil
 }
 
 // isDirEmpty - returns whether given directory is empty or not.

cmd/server-main.go

@@ -33,7 +33,7 @@ import (
 )
 
 func init() {
-	logger.Init(GOPATH)
+	logger.Init(GOPATH, GOROOT)
 	logger.RegisterUIError(fmtError)
 }
 

cmd/signature-v4-utils.go

@@ -39,6 +39,9 @@ func skipContentSha256Cksum(r *http.Request) bool {
 
 	if isRequestPresignedSignatureV4(r) {
 		v, ok = r.URL.Query()["X-Amz-Content-Sha256"]
+		if !ok {
+			v, ok = r.Header["X-Amz-Content-Sha256"]
+		}
 	} else {
 		v, ok = r.Header["X-Amz-Content-Sha256"]
 	}
@@ -62,6 +65,9 @@ func getContentSha256Cksum(r *http.Request) string {
 		// will default to 'UNSIGNED-PAYLOAD'.
 		defaultSha256Cksum = unsignedPayload
 		v, ok = r.URL.Query()["X-Amz-Content-Sha256"]
+		if !ok {
+			v, ok = r.Header["X-Amz-Content-Sha256"]
+		}
 	} else {
 		// X-Amz-Content-Sha256, if not set in signed requests, checksum
 		// will default to sha256([]byte("")).

cmd/storage-rpc-client.go

@@ -316,9 +316,9 @@ func NewStorageRPCClient(host *xnet.Host, endpointPath string) (*StorageRPCClien
 // Initialize new storage rpc client.
 func newStorageRPC(endpoint Endpoint) *StorageRPCClient {
 	host, err := xnet.ParseHost(endpoint.Host)
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to parse storage RPC Host", context.Background())
 	rpcClient, err := NewStorageRPCClient(host, endpoint.Path)
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to initialize storage RPC client", context.Background())
 	rpcClient.connected = rpcClient.Call(storageServiceName+".Connect", &AuthArgs{}, &VoidReply{}) == nil
 	return rpcClient
 }

cmd/web-handlers.go

@@ -458,11 +458,6 @@ type LoginRep struct {
 func (web *webAPIHandlers) Login(r *http.Request, args *LoginArgs, reply *LoginRep) error {
 	token, err := authenticateWeb(args.Username, args.Password)
 	if err != nil {
-		// Make sure to log errors related to browser login,
-		// for security and auditing reasons.
-		reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", r.RemoteAddr)
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogIf(ctx, err)
 		return toJSONError(err)
 	}
 
@@ -483,7 +478,9 @@ func (web webAPIHandlers) GenerateAuth(r *http.Request, args *WebGenericArgs, re
 		return toJSONError(errAuthentication)
 	}
 	cred, err := auth.GetNewCredentials()
-	logger.CriticalIf(context.Background(), err)
+	if err != nil {
+		return toJSONError(err)
+	}
 	reply.AccessKey = cred.AccessKey
 	reply.SecretKey = cred.SecretKey
 	reply.UIVersion = browser.UIVersion

cmd/xl-v1-multipart.go

@@ -375,8 +375,17 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	}
 
 	// Fetch buffer for I/O, returns from the pool if not allocates a new one and returns.
-	buffer := xl.bp.Get()
-	defer xl.bp.Put(buffer)
+	var buffer []byte
+	switch size := data.Size(); {
+	case size == 0:
+		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
+	case size < blockSizeV1:
+		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
+		buffer = make([]byte, size, 2*size)
+	default:
+		buffer = xl.bp.Get()
+		defer xl.bp.Put(buffer)
+	}
 
 	file, err := storage.CreateFile(ctx, data, minioMetaTmpBucket, tmpPartPath, buffer, DefaultBitrotAlgorithm, writeQuorum)
 	if err != nil {

cmd/xl-v1-object.go

@@ -611,8 +611,17 @@ func (xl xlObjects) putObject(ctx context.Context, bucket string, object string,
 	}
 
 	// Fetch buffer for I/O, returns from the pool if not allocates a new one and returns.
-	buffer := xl.bp.Get()
-	defer xl.bp.Put(buffer)
+	var buffer []byte
+	switch size := data.Size(); {
+	case size == 0:
+		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
+	case size < blockSizeV1:
+		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
+		buffer = make([]byte, size, 2*size)
+	default:
+		buffer = xl.bp.Get()
+		defer xl.bp.Put(buffer)
+	}
 
 	// Read data and split into parts - similar to multipart mechanism
 	for partIdx := 1; ; partIdx++ {

docs/disk-caching/README.md

@@ -8,7 +8,7 @@ Disk caching feature here refers to the use of caching disks to store content cl
 ## Get started
 
 ### 1. Prerequisites
-Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio).
+Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide).
 
 ### 2. Run Minio with cache
 Disk caching can be enabled by updating the `cache` config settings for Minio server. Config `cache` settings takes the mounted drive(s) or directory paths, cache expiry duration (in days) and any wildcard patterns to exclude from being cached.

docs/federation/lookup/Corefile.example

@@ -5,7 +5,7 @@
   }
   debug
   prometheus
-  cache 160 churchofminio.com
+  cache 160 mydomain.com
   loadbalance
   proxy . /etc/resolv.conf
 }

docs/federation/lookup/README.md

@@ -1,15 +1,20 @@
-# Federation
+# Federation Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
 This document explains how to configure Minio with `Bucket lookup from DNS` style federation.
 
-## Dependencies
+## Get started
+
+### 1. Prerequisites
+Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide).
+
+### 2. Run Minio in federated mode
 Bucket lookup from DNS federation requires two dependencies
 
 - etcd (for config, bucket SRV records)
-- coredns (for DNS management based on populated bucket SRV records)
+- coredns (for DNS management based on populated bucket SRV records, optional)
 
 ## Architecture
 
-![bucket-lookup](./bucket-lookup.png)
+![bucket-lookup](https://github.com/minio/minio/blob/master/docs/federation/lookup/bucket-lookup.png?raw=true)
 
 ### Environment variables
 
@@ -36,7 +41,7 @@ a bucket `bucket1` created on current Minio instance will be accessible as `buck
 
 - This field is mandatory for standalone and erasure code Minio server deployments, to enable federated mode.
 - This field is optional for distributed deployments. If you don't set this field in a federated setup, we use the IP addresses of
-hosts passed to the Minio server startup and use that to make DNS entries.
+hosts passed to the Minio server startup and use them for DNS entries.
 
 ### Run Multiple Clusters
 
@@ -67,3 +72,13 @@ points to the public IP address where each cluster might be accessible, this is
 NOTE: `mybucket` only exists on one cluster either `cluster1` or `cluster2` this is random and
 is decided by how `domain.com` gets resolved, if there is a round-robin DNS on `domain.com` then
 it is randomized which cluster might provision the bucket.
+
+### 3. Test your setup
+To test this setup, access the Minio server via browser or [`mc`](https://docs.minio.io/docs/minio-client-quickstart-guide). You’ll see the uploaded files are accessible from the all the Minio endpoints.
+
+# Explore Further
+- [Use `mc` with Minio Server](https://docs.minio.io/docs/minio-client-quickstart-guide)
+- [Use `aws-cli` with Minio Server](https://docs.minio.io/docs/aws-cli-with-minio)
+- [Use `s3cmd` with Minio Server](https://docs.minio.io/docs/s3cmd-with-minio)
+- [Use `minio-go` SDK with Minio Server](https://docs.minio.io/docs/golang-client-quickstart-guide)
+- [The Minio documentation website](https://docs.minio.io)

docs/large-bucket/README.md

@@ -8,7 +8,7 @@ We call it large-bucket as it allows a single Minio bucket to expand over multip
 If you're aware of distributed Minio setup, the installation and configuration remains the same. Only new addition to the input syntax is `...` convention to abbreviate the drive arguments. Remote drives in a distributed setup are encoded as HTTP(s) URIs which can be similarly abbreviated as well.
 
 ### 1. Prerequisites
-Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio).
+Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide).
 
 ### 2. Run Minio on many drives
 We'll see examples on how to do this in the following sections.

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '2'
 # 9001 through 9004.
 services:
  minio1:
-  image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+  image: minio/minio:RELEASE.2018-06-09T03-43-35Z
   volumes:
    - data1:/data
   ports:
@@ -15,7 +15,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio2:
-  image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+  image: minio/minio:RELEASE.2018-06-09T03-43-35Z
   volumes:
    - data2:/data
   ports:
@@ -25,7 +25,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio3:
-  image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+  image: minio/minio:RELEASE.2018-06-09T03-43-35Z
   volumes:
    - data3:/data
   ports:
@@ -35,7 +35,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio4:
-  image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+  image: minio/minio:RELEASE.2018-06-09T03-43-35Z
   volumes:
    - data4:/data
   ports:

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.1'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+    image: minio/minio:RELEASE.2018-06-09T03-43-35Z
     volumes:
       - minio1-data:/export
     ports:
@@ -20,7 +20,7 @@ services:
       - access_key
 
   minio2:
-    image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+    image: minio/minio:RELEASE.2018-06-09T03-43-35Z
     volumes:
       - minio2-data:/export
     ports:
@@ -38,7 +38,7 @@ services:
       - access_key
 
   minio3:
-    image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+    image: minio/minio:RELEASE.2018-06-09T03-43-35Z
     volumes:
       - minio3-data:/export
     ports:
@@ -56,7 +56,7 @@ services:
       - access_key
 
   minio4:
-    image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+    image: minio/minio:RELEASE.2018-06-09T03-43-35Z
     volumes:
       - minio4-data:/export
     ports:

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+    image: minio/minio:RELEASE.2018-06-09T03-43-35Z
     volumes:
       - minio1-data:/export
     ports:
@@ -20,7 +20,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio2:
-    image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+    image: minio/minio:RELEASE.2018-06-09T03-43-35Z
     volumes:
       - minio2-data:/export
     ports:
@@ -38,7 +38,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio3:
-    image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+    image: minio/minio:RELEASE.2018-06-09T03-43-35Z
     volumes:
       - minio3-data:/export
     ports:
@@ -56,7 +56,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio4:
-    image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+    image: minio/minio:RELEASE.2018-06-09T03-43-35Z
     volumes:
       - minio4-data:/export
     ports:

docs/orchestration/kubernetes-yaml/README.md

@@ -124,7 +124,7 @@ spec:
         - name: data 
           mountPath: "/data"
         # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+        image: minio/minio:RELEASE.2018-06-09T03-43-35Z
         args:
         - server
         - /data
@@ -303,7 +303,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+        image: minio/minio:RELEASE.2018-06-09T03-43-35Z
         args:
         - server
         - http://minio-0.minio.default.svc.cluster.local/data
@@ -514,7 +514,7 @@ spec:
       containers:
       - name: minio
         # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+        image: minio/minio:RELEASE.2018-06-09T03-43-35Z
         args:
         - gateway
         - gcs

docs/orchestration/kubernetes-yaml/minio-distributed-statefulset.yaml

@@ -21,7 +21,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+        image: minio/minio:RELEASE.2018-06-09T03-43-35Z
         args:
         - server
         - http://minio-0.minio.default.svc.cluster.local/data

docs/orchestration/kubernetes-yaml/minio-gcs-gateway-deployment.yaml

@@ -21,7 +21,7 @@ spec:
       containers:
       - name: minio
         # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+        image: minio/minio:RELEASE.2018-06-09T03-43-35Z
         args:
         - gateway
         - gcs

docs/orchestration/kubernetes-yaml/minio-standalone-deployment.yaml

@@ -29,7 +29,7 @@ spec:
         - name: data 
           mountPath: "/data"
         # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+        image: minio/minio:RELEASE.2018-06-09T03-43-35Z
         args:
         - server
         - /data

docs/orchestration/kubernetes/README.md

@@ -30,7 +30,7 @@ Above command deploys Minio on the Kubernetes cluster in the default configurati
 | Parameter                  | Description                         | Default                                                 |
 |----------------------------|-------------------------------------|---------------------------------------------------------|
 | `image`                    | Minio image name                    | `minio/minio`                                           |
-| `imageTag`                 | Minio image tag. Possible values listed [here](https://hub.docker.com/r/minio/minio/tags/).| `RELEASE.2018-06-08T03-49-38Z`|
+| `imageTag`                 | Minio image tag. Possible values listed [here](https://hub.docker.com/r/minio/minio/tags/).| `RELEASE.2018-06-09T03-43-35Z`|
 | `imagePullPolicy`          | Image pull policy                   | `Always`                                                |
 | `mode`                     | Minio server mode (`standalone`, `shared` or `distributed`)| `standalone`                     |
 | `numberOfNodes`            | Number of nodes (applicable only for Minio distributed mode). Should be 4 <= x <= 16 | `4`    |

docs/orchestration/minikube/statefulset.yaml

@@ -54,7 +54,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2018-06-08T03-49-38Z
+        image: minio/minio:RELEASE.2018-06-09T03-43-35Z
         imagePullPolicy: IfNotPresent
         args: ["server", "http://minio-0.minio.default.svc.cluster.local/data", "http://minio-1.minio.default.svc.cluster.local/data", "http://minio-2.minio.default.svc.cluster.local/data", "http://minio-3.minio.default.svc.cluster.local/data"]
         ports:

docs/shared-backend/README.md

@@ -18,7 +18,7 @@ If you're aware of stand-alone Minio set up, the installation and running remain
 
 ## 1. Prerequisites
 
-Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio).
+Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide).
 
 ## 2. Run Minio on Shared Backend
 

docs/zh_CN/disk-caching/README.md

@@ -7,7 +7,7 @@
 ## 开始
 
 ### 1. 前期条件
-安装Minio - [Minio快速入门](https://docs.minio.io/docs/minio)。
+安装Minio - [Minio快速入门](https://docs.minio.io/docs/minio-quickstart-guide)。
 
 ### 2. 运行Minio缓存
 磁盘缓存可以通过修改Minio服务的`cache`配置来进行开启。配置`cache`设置需要指定磁盘路径、缓存过期时间（以天为单位）以及使用统配符方式指定的不需要进行缓存的对象。

docs/zh_CN/large-bucket/README.md

@@ -8,7 +8,7 @@ Minio的纠删码功能限制了最多只能使用16块磁盘。这就限制了
 安装和部署方式和分布式Minio一样。只不过是在输入参数的语法上，用`...`来做为磁盘参数的简写。分布式设置中的远程磁盘被编码为HTTP（s）URI，它也可以被同样的缩写。
 
 ### 1. 前提条件
-安装Minio - [Minio快速入门](https://docs.minio.io/docs/minio)。
+安装Minio - [Minio快速入门](https://docs.minio.io/docs/minio-quickstart-guide)。
 
 ### 2. 在多个磁盘上运行Minio
 我们将在下面的章节中看到如何做到这一点的例子。

pkg/dns/etcd_dns.go

@@ -102,10 +102,8 @@ func (c *coreDNS) list(key string) ([]SrvRecord, error) {
 
 // Adds DNS entries into etcd endpoint in CoreDNS etcd message format.
 func (c *coreDNS) Put(bucket string) error {
-	var err error
 	for ip := range c.domainIPs {
-		var bucketMsg []byte
-		bucketMsg, err = newCoreDNSMsg(bucket, ip, c.domainPort, defaultTTL)
+		bucketMsg, err := newCoreDNSMsg(bucket, ip, c.domainPort, defaultTTL)
 		if err != nil {
 			return err
 		}
@@ -115,8 +113,14 @@ func (c *coreDNS) Put(bucket string) error {
 		ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 		_, err = kapi.Set(ctx, key, string(bucketMsg), nil)
 		cancel()
+		if err != nil {
+			ctx, cancel = context.WithTimeout(context.Background(), defaultContextTimeout)
+			kapi.Delete(ctx, key, &etcd.DeleteOptions{Recursive: true})
+			cancel()
+			return err
+		}
 	}
-	return err
+	return nil
 }
 
 // Removes DNS entries added in Put().

vendor/github.com/minio/minio-go/api-compose-object.go

@@ -1,6 +1,6 @@
 /*
  * Minio Go Library for Amazon S3 Compatible Cloud Storage
- * Copyright 2017 Minio, Inc.
+ * Copyright 2017, 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,6 +20,8 @@ package minio
 import (
 	"context"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -343,11 +345,12 @@ func (c Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID str
 	return p, nil
 }
 
-// ComposeObject - creates an object using server-side copying of
+// ComposeObjectWithProgress - creates an object using server-side copying of
 // existing objects. It takes a list of source objects (with optional
 // offsets) and concatenates them into a new object using only
-// server-side copying operations.
-func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
+// server-side copying operations. Optionally takes progress reader hook
+// for applications to look at current progress.
+func (c Client) ComposeObjectWithProgress(dst DestinationInfo, srcs []SourceInfo, progress io.Reader) error {
 	if len(srcs) < 1 || len(srcs) > maxPartsCount {
 		return ErrInvalidArgument("There must be as least one and up to 10000 source objects.")
 	}
@@ -421,7 +424,7 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
 	// involved, it is being copied wholly and at most 5GiB in
 	// size, emptyfiles are also supported).
 	if (totalParts == 1 && srcs[0].start == -1 && totalSize <= maxPartSize) || (totalSize == 0) {
-		return c.CopyObject(dst, srcs[0])
+		return c.CopyObjectWithProgress(dst, srcs[0], progress)
 	}
 
 	// Now, handle multipart-copy cases.
@@ -476,6 +479,9 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
 			if err != nil {
 				return err
 			}
+			if progress != nil {
+				io.CopyN(ioutil.Discard, progress, start+end-1)
+			}
 			objParts = append(objParts, complPart)
 			partIndex++
 		}
@@ -490,10 +496,20 @@ func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
 	return nil
 }
 
-// partsRequired is ceiling(size / copyPartSize)
+// ComposeObject - creates an object using server-side copying of
+// existing objects. It takes a list of source objects (with optional
+// offsets) and concatenates them into a new object using only
+// server-side copying operations.
+func (c Client) ComposeObject(dst DestinationInfo, srcs []SourceInfo) error {
+	return c.ComposeObjectWithProgress(dst, srcs, nil)
+}
+
+// partsRequired is maximum parts possible with
+// max part size of ceiling(maxMultipartPutObjectSize / (maxPartsCount - 1))
 func partsRequired(size int64) int64 {
-	r := size / copyPartSize
-	if size%copyPartSize > 0 {
+	maxPartSize := maxMultipartPutObjectSize / (maxPartsCount - 1)
+	r := size / int64(maxPartSize)
+	if size%int64(maxPartSize) > 0 {
 		r++
 	}
 	return r

vendor/github.com/minio/minio-go/api-list.go

@@ -118,7 +118,7 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d
 		var continuationToken string
 		for {
 			// Get list of objects a maximum of 1000 per request.
-			result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, 1000)
+			result, err := c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, 1000, "")
 			if err != nil {
 				objectStatCh <- ObjectInfo{
 					Err: err,
@@ -171,11 +171,12 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d
 // You can use the request parameters as selection criteria to return a subset of the objects in a bucket.
 // request parameters :-
 // ---------
-// ?continuation-token - Specifies the key to start with when listing objects in a bucket.
+// ?continuation-token - Used to continue iterating over a set of objects
 // ?delimiter - A delimiter is a character you use to group keys.
 // ?prefix - Limits the response to keys that begin with the specified prefix.
 // ?max-keys - Sets the maximum number of keys returned in the response body.
-func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int) (ListBucketV2Result, error) {
+// ?start-after - Specifies the key to start after when listing objects in a bucket.
+func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) {
 	// Validate bucket name.
 	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
 		return ListBucketV2Result{}, err
@@ -216,6 +217,11 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s
 	// Set max keys.
 	urlValues.Set("max-keys", fmt.Sprintf("%d", maxkeys))
 
+	// Set start-after
+	if startAfter != "" {
+		urlValues.Set("start-after", startAfter)
+	}
+
 	// Execute GET on bucket to list objects.
 	resp, err := c.executeMethod(context.Background(), "GET", requestMetadata{
 		bucketName:       bucketName,

vendor/github.com/minio/minio-go/api-put-object-copy.go

@@ -1,6 +1,6 @@
 /*
  * Minio Go Library for Amazon S3 Compatible Cloud Storage
- * Copyright 2017 Minio, Inc.
+ * Copyright 2017, 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,6 +19,8 @@ package minio
 
 import (
 	"context"
+	"io"
+	"io/ioutil"
 	"net/http"
 
 	"github.com/minio/minio-go/pkg/encrypt"
@@ -26,13 +28,31 @@ import (
 
 // CopyObject - copy a source object into a new object
 func (c Client) CopyObject(dst DestinationInfo, src SourceInfo) error {
+	return c.CopyObjectWithProgress(dst, src, nil)
+}
+
+// CopyObjectWithProgress - copy a source object into a new object, optionally takes
+// progress bar input to notify current progress.
+func (c Client) CopyObjectWithProgress(dst DestinationInfo, src SourceInfo, progress io.Reader) error {
 	header := make(http.Header)
 	for k, v := range src.Headers {
 		header[k] = v
 	}
+
+	var err error
+	var size int64
+	// If progress bar is specified, size should be requested as well initiate a StatObject request.
+	if progress != nil {
+		size, _, _, err = src.getProps(c)
+		if err != nil {
+			return err
+		}
+	}
+
 	if src.encryption != nil {
 		encrypt.SSECopy(src.encryption).Marshal(header)
 	}
+
 	if dst.encryption != nil {
 		dst.encryption.Marshal(header)
 	}
@@ -53,5 +73,11 @@ func (c Client) CopyObject(dst DestinationInfo, src SourceInfo) error {
 	if resp.StatusCode != http.StatusOK {
 		return httpRespToErrorResponse(resp, dst.bucket, dst.object)
 	}
+
+	// Update the progress properly after successful copy.
+	if progress != nil {
+		io.CopyN(ioutil.Discard, progress, size)
+	}
+
 	return nil
 }

vendor/github.com/minio/minio-go/api-put-object.go

@@ -28,21 +28,22 @@ import (
 
 	"github.com/minio/minio-go/pkg/encrypt"
 	"github.com/minio/minio-go/pkg/s3utils"
-	"golang.org/x/net/lex/httplex"
+	"golang.org/x/net/http/httpguts"
 )
 
 // PutObjectOptions represents options specified by user for PutObject call
 type PutObjectOptions struct {
-	UserMetadata         map[string]string
-	Progress             io.Reader
-	ContentType          string
-	ContentEncoding      string
-	ContentDisposition   string
-	ContentLanguage      string
-	CacheControl         string
-	ServerSideEncryption encrypt.ServerSide
-	NumThreads           uint
-	StorageClass         string
+	UserMetadata            map[string]string
+	Progress                io.Reader
+	ContentType             string
+	ContentEncoding         string
+	ContentDisposition      string
+	ContentLanguage         string
+	CacheControl            string
+	ServerSideEncryption    encrypt.ServerSide
+	NumThreads              uint
+	StorageClass            string
+	WebsiteRedirectLocation string
 }
 
 // getNumThreads - gets the number of threads to be used in the multipart
@@ -84,6 +85,9 @@ func (opts PutObjectOptions) Header() (header http.Header) {
 	if opts.StorageClass != "" {
 		header[amzStorageClass] = []string{opts.StorageClass}
 	}
+	if opts.WebsiteRedirectLocation != "" {
+		header[amzWebsiteRedirectLocation] = []string{opts.WebsiteRedirectLocation}
+	}
 	for k, v := range opts.UserMetadata {
 		if !isAmzHeader(k) && !isStandardHeader(k) && !isStorageClassHeader(k) {
 			header["X-Amz-Meta-"+k] = []string{v}
@@ -97,10 +101,10 @@ func (opts PutObjectOptions) Header() (header http.Header) {
 // validate() checks if the UserMetadata map has standard headers or and raises an error if so.
 func (opts PutObjectOptions) validate() (err error) {
 	for k, v := range opts.UserMetadata {
-		if !httplex.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) {
+		if !httpguts.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) {
 			return ErrInvalidArgument(k + " unsupported user defined metadata name")
 		}
-		if !httplex.ValidHeaderFieldValue(v) {
+		if !httpguts.ValidHeaderFieldValue(v) {
 			return ErrInvalidArgument(v + " unsupported user defined metadata value")
 		}
 	}

vendor/github.com/minio/minio-go/api-stat.go

@@ -66,6 +66,9 @@ var defaultFilterKeys = []string{
 	"x-amz-bucket-region",
 	"x-amz-request-id",
 	"x-amz-id-2",
+	"Content-Security-Policy",
+	"X-Xss-Protection",
+
 	// Add new headers to be ignored.
 }
 

vendor/github.com/minio/minio-go/api.go

@@ -99,7 +99,7 @@ type Options struct {
 // Global constants.
 const (
 	libraryName    = "minio-go"
-	libraryVersion = "6.0.1"
+	libraryVersion = "v6.0.3"
 )
 
 // User Agent should always following the below style.

vendor/github.com/minio/minio-go/constants.go

@@ -27,10 +27,6 @@ const absMinPartSize = 1024 * 1024 * 5
 // putObject behaves internally as multipart.
 const minPartSize = 1024 * 1024 * 64
 
-// copyPartSize - default (and maximum) part size to copy in a
-// copy-object request (5GiB)
-const copyPartSize = 1024 * 1024 * 1024 * 5
-
 // maxPartsCount - maximum number of parts for a single multipart session.
 const maxPartsCount = 10000
 
@@ -61,3 +57,6 @@ const (
 
 // Storage class header constant.
 const amzStorageClass = "X-Amz-Storage-Class"
+
+// Website redirect location header constant
+const amzWebsiteRedirectLocation = "X-Amz-Website-Redirect-Location"

vendor/github.com/minio/minio-go/core.go

@@ -48,9 +48,9 @@ func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int)
 }
 
 // ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses
-// continuationToken instead of marker to further filter the results.
-func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int) (ListBucketV2Result, error) {
-	return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, maxkeys)
+// continuationToken instead of marker to support iteration over the results.
+func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int, startAfter string) (ListBucketV2Result, error) {
+	return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, maxkeys, startAfter)
 }
 
 // CopyObject - copies an object from source object to destination object on server side.
@@ -82,6 +82,8 @@ func (c Core) PutObject(bucket, object string, data io.Reader, size int64, md5Ba
 			opts.ContentType = v
 		} else if strings.ToLower(k) == "cache-control" {
 			opts.CacheControl = v
+		} else if strings.ToLower(k) == strings.ToLower(amzWebsiteRedirectLocation) {
+			opts.WebsiteRedirectLocation = v
 		} else {
 			m[k] = metadata[k]
 		}

vendor/github.com/minio/minio-go/retry.go

@@ -131,6 +131,7 @@ var retryableS3Codes = map[string]struct{}{
 	"InternalError":         {},
 	"ExpiredToken":          {},
 	"ExpiredTokenException": {},
+	"SlowDown":              {},
 	// Add more AWS S3 codes here.
 }
 

vendor/github.com/minio/minio-go/utils.go

@@ -222,6 +222,7 @@ var supportedHeaders = []string{
 	"content-encoding",
 	"content-disposition",
 	"content-language",
+	"x-amz-website-redirect-location",
 	// Add more supported headers here.
 }
 

vendor/golang.org/x/net/http/httpguts/guts.go

@@ -0,0 +1,65 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package httpguts provides functions implementing various details
+// of the HTTP specification.
+//
+// This package is shared by the standard library (which vendors it)
+// and x/net/http2. It comes with no API stability promise.
+package httpguts
+
+import (
+	"net/textproto"
+	"strings"
+)
+
+// SniffedContentType reports whether ct is a Content-Type that is known
+// to cause client-side content sniffing.
+//
+// This provides just a partial implementation of mime.ParseMediaType
+// with the assumption that the Content-Type is not attacker controlled.
+func SniffedContentType(ct string) bool {
+	if i := strings.Index(ct, ";"); i != -1 {
+		ct = ct[:i]
+	}
+	ct = strings.ToLower(strings.TrimSpace(ct))
+	return ct == "text/plain" || ct == "application/octet-stream" ||
+		ct == "application/unknown" || ct == "unknown/unknown" || ct == "*/*" ||
+		!strings.Contains(ct, "/")
+}
+
+// ValidTrailerHeader reports whether name is a valid header field name to appear
+// in trailers.
+// See RFC 7230, Section 4.1.2
+func ValidTrailerHeader(name string) bool {
+	name = textproto.CanonicalMIMEHeaderKey(name)
+	if strings.HasPrefix(name, "If-") || badTrailer[name] {
+		return false
+	}
+	return true
+}
+
+var badTrailer = map[string]bool{
+	"Authorization":       true,
+	"Cache-Control":       true,
+	"Connection":          true,
+	"Content-Encoding":    true,
+	"Content-Length":      true,
+	"Content-Range":       true,
+	"Content-Type":        true,
+	"Expect":              true,
+	"Host":                true,
+	"Keep-Alive":          true,
+	"Max-Forwards":        true,
+	"Pragma":              true,
+	"Proxy-Authenticate":  true,
+	"Proxy-Authorization": true,
+	"Proxy-Connection":    true,
+	"Range":               true,
+	"Realm":               true,
+	"Te":                  true,
+	"Trailer":             true,
+	"Transfer-Encoding":   true,
+	"Www-Authenticate":    true,
+}

vendor/golang.org/x/net/http/httpguts/httplex.go

@@ -0,0 +1,346 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package httpguts
+
+import (
+	"net"
+	"strings"
+	"unicode/utf8"
+
+	"golang.org/x/net/idna"
+)
+
+var isTokenTable = [127]bool{
+	'!':  true,
+	'#':  true,
+	'$':  true,
+	'%':  true,
+	'&':  true,
+	'\'': true,
+	'*':  true,
+	'+':  true,
+	'-':  true,
+	'.':  true,
+	'0':  true,
+	'1':  true,
+	'2':  true,
+	'3':  true,
+	'4':  true,
+	'5':  true,
+	'6':  true,
+	'7':  true,
+	'8':  true,
+	'9':  true,
+	'A':  true,
+	'B':  true,
+	'C':  true,
+	'D':  true,
+	'E':  true,
+	'F':  true,
+	'G':  true,
+	'H':  true,
+	'I':  true,
+	'J':  true,
+	'K':  true,
+	'L':  true,
+	'M':  true,
+	'N':  true,
+	'O':  true,
+	'P':  true,
+	'Q':  true,
+	'R':  true,
+	'S':  true,
+	'T':  true,
+	'U':  true,
+	'W':  true,
+	'V':  true,
+	'X':  true,
+	'Y':  true,
+	'Z':  true,
+	'^':  true,
+	'_':  true,
+	'`':  true,
+	'a':  true,
+	'b':  true,
+	'c':  true,
+	'd':  true,
+	'e':  true,
+	'f':  true,
+	'g':  true,
+	'h':  true,
+	'i':  true,
+	'j':  true,
+	'k':  true,
+	'l':  true,
+	'm':  true,
+	'n':  true,
+	'o':  true,
+	'p':  true,
+	'q':  true,
+	'r':  true,
+	's':  true,
+	't':  true,
+	'u':  true,
+	'v':  true,
+	'w':  true,
+	'x':  true,
+	'y':  true,
+	'z':  true,
+	'|':  true,
+	'~':  true,
+}
+
+func IsTokenRune(r rune) bool {
+	i := int(r)
+	return i < len(isTokenTable) && isTokenTable[i]
+}
+
+func isNotToken(r rune) bool {
+	return !IsTokenRune(r)
+}
+
+// HeaderValuesContainsToken reports whether any string in values
+// contains the provided token, ASCII case-insensitively.
+func HeaderValuesContainsToken(values []string, token string) bool {
+	for _, v := range values {
+		if headerValueContainsToken(v, token) {
+			return true
+		}
+	}
+	return false
+}
+
+// isOWS reports whether b is an optional whitespace byte, as defined
+// by RFC 7230 section 3.2.3.
+func isOWS(b byte) bool { return b == ' ' || b == '\t' }
+
+// trimOWS returns x with all optional whitespace removes from the
+// beginning and end.
+func trimOWS(x string) string {
+	// TODO: consider using strings.Trim(x, " \t") instead,
+	// if and when it's fast enough. See issue 10292.
+	// But this ASCII-only code will probably always beat UTF-8
+	// aware code.
+	for len(x) > 0 && isOWS(x[0]) {
+		x = x[1:]
+	}
+	for len(x) > 0 && isOWS(x[len(x)-1]) {
+		x = x[:len(x)-1]
+	}
+	return x
+}
+
+// headerValueContainsToken reports whether v (assumed to be a
+// 0#element, in the ABNF extension described in RFC 7230 section 7)
+// contains token amongst its comma-separated tokens, ASCII
+// case-insensitively.
+func headerValueContainsToken(v string, token string) bool {
+	v = trimOWS(v)
+	if comma := strings.IndexByte(v, ','); comma != -1 {
+		return tokenEqual(trimOWS(v[:comma]), token) || headerValueContainsToken(v[comma+1:], token)
+	}
+	return tokenEqual(v, token)
+}
+
+// lowerASCII returns the ASCII lowercase version of b.
+func lowerASCII(b byte) byte {
+	if 'A' <= b && b <= 'Z' {
+		return b + ('a' - 'A')
+	}
+	return b
+}
+
+// tokenEqual reports whether t1 and t2 are equal, ASCII case-insensitively.
+func tokenEqual(t1, t2 string) bool {
+	if len(t1) != len(t2) {
+		return false
+	}
+	for i, b := range t1 {
+		if b >= utf8.RuneSelf {
+			// No UTF-8 or non-ASCII allowed in tokens.
+			return false
+		}
+		if lowerASCII(byte(b)) != lowerASCII(t2[i]) {
+			return false
+		}
+	}
+	return true
+}
+
+// isLWS reports whether b is linear white space, according
+// to http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2
+//      LWS            = [CRLF] 1*( SP | HT )
+func isLWS(b byte) bool { return b == ' ' || b == '\t' }
+
+// isCTL reports whether b is a control byte, according
+// to http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2
+//      CTL            = <any US-ASCII control character
+//                       (octets 0 - 31) and DEL (127)>
+func isCTL(b byte) bool {
+	const del = 0x7f // a CTL
+	return b < ' ' || b == del
+}
+
+// ValidHeaderFieldName reports whether v is a valid HTTP/1.x header name.
+// HTTP/2 imposes the additional restriction that uppercase ASCII
+// letters are not allowed.
+//
+//  RFC 7230 says:
+//   header-field   = field-name ":" OWS field-value OWS
+//   field-name     = token
+//   token          = 1*tchar
+//   tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." /
+//           "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA
+func ValidHeaderFieldName(v string) bool {
+	if len(v) == 0 {
+		return false
+	}
+	for _, r := range v {
+		if !IsTokenRune(r) {
+			return false
+		}
+	}
+	return true
+}
+
+// ValidHostHeader reports whether h is a valid host header.
+func ValidHostHeader(h string) bool {
+	// The latest spec is actually this:
+	//
+	// http://tools.ietf.org/html/rfc7230#section-5.4
+	//     Host = uri-host [ ":" port ]
+	//
+	// Where uri-host is:
+	//     http://tools.ietf.org/html/rfc3986#section-3.2.2
+	//
+	// But we're going to be much more lenient for now and just
+	// search for any byte that's not a valid byte in any of those
+	// expressions.
+	for i := 0; i < len(h); i++ {
+		if !validHostByte[h[i]] {
+			return false
+		}
+	}
+	return true
+}
+
+// See the validHostHeader comment.
+var validHostByte = [256]bool{
+	'0': true, '1': true, '2': true, '3': true, '4': true, '5': true, '6': true, '7': true,
+	'8': true, '9': true,
+
+	'a': true, 'b': true, 'c': true, 'd': true, 'e': true, 'f': true, 'g': true, 'h': true,
+	'i': true, 'j': true, 'k': true, 'l': true, 'm': true, 'n': true, 'o': true, 'p': true,
+	'q': true, 'r': true, 's': true, 't': true, 'u': true, 'v': true, 'w': true, 'x': true,
+	'y': true, 'z': true,
+
+	'A': true, 'B': true, 'C': true, 'D': true, 'E': true, 'F': true, 'G': true, 'H': true,
+	'I': true, 'J': true, 'K': true, 'L': true, 'M': true, 'N': true, 'O': true, 'P': true,
+	'Q': true, 'R': true, 'S': true, 'T': true, 'U': true, 'V': true, 'W': true, 'X': true,
+	'Y': true, 'Z': true,
+
+	'!':  true, // sub-delims
+	'$':  true, // sub-delims
+	'%':  true, // pct-encoded (and used in IPv6 zones)
+	'&':  true, // sub-delims
+	'(':  true, // sub-delims
+	')':  true, // sub-delims
+	'*':  true, // sub-delims
+	'+':  true, // sub-delims
+	',':  true, // sub-delims
+	'-':  true, // unreserved
+	'.':  true, // unreserved
+	':':  true, // IPv6address + Host expression's optional port
+	';':  true, // sub-delims
+	'=':  true, // sub-delims
+	'[':  true,
+	'\'': true, // sub-delims
+	']':  true,
+	'_':  true, // unreserved
+	'~':  true, // unreserved
+}
+
+// ValidHeaderFieldValue reports whether v is a valid "field-value" according to
+// http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 :
+//
+//        message-header = field-name ":" [ field-value ]
+//        field-value    = *( field-content | LWS )
+//        field-content  = <the OCTETs making up the field-value
+//                         and consisting of either *TEXT or combinations
+//                         of token, separators, and quoted-string>
+//
+// http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2 :
+//
+//        TEXT           = <any OCTET except CTLs,
+//                          but including LWS>
+//        LWS            = [CRLF] 1*( SP | HT )
+//        CTL            = <any US-ASCII control character
+//                         (octets 0 - 31) and DEL (127)>
+//
+// RFC 7230 says:
+//  field-value    = *( field-content / obs-fold )
+//  obj-fold       =  N/A to http2, and deprecated
+//  field-content  = field-vchar [ 1*( SP / HTAB ) field-vchar ]
+//  field-vchar    = VCHAR / obs-text
+//  obs-text       = %x80-FF
+//  VCHAR          = "any visible [USASCII] character"
+//
+// http2 further says: "Similarly, HTTP/2 allows header field values
+// that are not valid. While most of the values that can be encoded
+// will not alter header field parsing, carriage return (CR, ASCII
+// 0xd), line feed (LF, ASCII 0xa), and the zero character (NUL, ASCII
+// 0x0) might be exploited by an attacker if they are translated
+// verbatim. Any request or response that contains a character not
+// permitted in a header field value MUST be treated as malformed
+// (Section 8.1.2.6). Valid characters are defined by the
+// field-content ABNF rule in Section 3.2 of [RFC7230]."
+//
+// This function does not (yet?) properly handle the rejection of
+// strings that begin or end with SP or HTAB.
+func ValidHeaderFieldValue(v string) bool {
+	for i := 0; i < len(v); i++ {
+		b := v[i]
+		if isCTL(b) && !isLWS(b) {
+			return false
+		}
+	}
+	return true
+}
+
+func isASCII(s string) bool {
+	for i := 0; i < len(s); i++ {
+		if s[i] >= utf8.RuneSelf {
+			return false
+		}
+	}
+	return true
+}
+
+// PunycodeHostPort returns the IDNA Punycode version
+// of the provided "host" or "host:port" string.
+func PunycodeHostPort(v string) (string, error) {
+	if isASCII(v) {
+		return v, nil
+	}
+
+	host, port, err := net.SplitHostPort(v)
+	if err != nil {
+		// The input 'v' argument was just a "host" argument,
+		// without a port. This error should not be returned
+		// to the caller.
+		host = v
+		port = ""
+	}
+	host, err = idna.ToASCII(host)
+	if err != nil {
+		// Non-UTF-8? Not representable in Punycode, in any
+		// case.
+		return "", err
+	}
+	if port == "" {
+		return host, nil
+	}
+	return net.JoinHostPort(host, port), nil
+}

vendor/vendor.json

@@ -414,9 +414,9 @@
 			"checksumSHA1": "bKMZjd2wPw13VwoE7mBeSv5djFA=",
 			"path": "github.com/matttproud/golang_protobuf_extensions/pbutil",
 			"revision": "c12348ce28de40eed0136aa2b644d0ee0650e56c",
-		        "revisionTime": "2016-04-24T11:30:07Z"
-                },
-                {
+			"revisionTime": "2016-04-24T11:30:07Z"
+		},
+		{
 			"checksumSHA1": "OUZ1FFXyKs+Cfg9M9rmXqqweQck=",
 			"path": "github.com/miekg/dns",
 			"revision": "db96a2b759cdef4f11a34506a42eb8d1290c598e",
@@ -470,10 +470,10 @@
 			"revisionTime": "2016-02-29T08:42:30-08:00"
 		},
 		{
-			"checksumSHA1": "w7jIUKw0cR7Z34JykcQXGJEPEmI=",
+			"checksumSHA1": "hdWmWbGpljSQMBOcpcwhAnP2aaQ=",
 			"path": "github.com/minio/minio-go",
-			"revision": "a2238f0e60facba1c2ccff86ddc624e23fae4f23",
-			"revisionTime": "2018-04-13T18:58:16Z"
+			"revision": "10531abd0af1579a12dc1977d67c0fec2b348679",
+			"revisionTime": "2018-06-13T23:01:28Z"
 		},
 		{
 			"checksumSHA1": "Qsj+6JPmJ8R5rFNQSHqRb8xAwOw=",
@@ -783,6 +783,12 @@
 			"revision": "f2499483f923065a842d38eb4c7f1927e6fc6e6d",
 			"revisionTime": "2017-01-14T04:22:49Z"
 		},
+		{
+			"checksumSHA1": "c3H2wB/3tGrw6VqCnlye+kSdoXU=",
+			"path": "golang.org/x/net/http/httpguts",
+			"revision": "1e491301e022f8f977054da4c2d852decd59571f",
+			"revisionTime": "2018-05-30T06:29:46Z"
+		},
 		{
 			"checksumSHA1": "Zh++JEDfXo0DKQtoKVfScwVdvww=",
 			"path": "golang.org/x/net/http2",