Detect change in underlying mounted disks (#7229)

- Current implementation was spawning renewer goroutines
without waiting for the lease duration to end. Remove vault renewer
and call vault.RenewToken directly and manage reauthentication if
lease expired.

.gitignore

@@ -22,4 +22,5 @@ parts/
 prime/
 stage/
 .sia_temp/
-config.json
+config.json
+healthcheck

Dockerfile

@@ -11,7 +11,8 @@ RUN  \
      apk add --no-cache git && \
      go get -v -d github.com/minio/minio && \
      cd /go/src/github.com/minio/minio && \
-     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
+     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
+     go build -ldflags "-s -w" -o /usr/bin/healthcheck dockerscripts/healthcheck.go 
 
 FROM alpine:3.7
 
@@ -22,7 +23,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 EXPOSE 9000
 
 COPY --from=0 /go/bin/minio /usr/bin/minio
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \
      apk add --no-cache ca-certificates 'curl>7.61.0' && \
@@ -32,7 +34,6 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
+HEALTHCHECK --interval=1m CMD healthcheck
 
 CMD ["minio"]

Dockerfile.dev

@@ -2,7 +2,7 @@ FROM alpine:3.7
 
 LABEL maintainer="Minio Inc <dev@minio.io>"
 
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck /usr/bin/
 COPY minio /usr/bin/
 
 ENV MINIO_UPDATE off
@@ -14,7 +14,7 @@ RUN \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      chmod +x /usr/bin/minio  && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck.sh
+     chmod +x /usr/bin/healthcheck
 
 EXPOSE 9000
 
@@ -22,7 +22,6 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
+HEALTHCHECK --interval=1m CMD healthcheck
 
 CMD ["minio"]

Dockerfile.release

@@ -1,8 +1,22 @@
+FROM golang:1.11.4-alpine3.7
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+
+WORKDIR /go/src/github.com/minio/
+
+RUN  \
+     apk add --no-cache git && \
+     go get -v -d github.com/minio/minio && \
+     cd /go/src/github.com/minio/minio/dockerscripts && \
+     go build -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go
+
 FROM alpine:3.7
 
 LABEL maintainer="Minio Inc <dev@minio.io>"
 
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
@@ -14,7 +28,7 @@ RUN \
      curl https://dl.minio.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
      chmod +x /usr/bin/minio  && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck.sh
+     chmod +x /usr/bin/healthcheck
 
 EXPOSE 9000
 
@@ -22,7 +36,6 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
+HEALTHCHECK --interval=1m CMD healthcheck
 
 CMD ["minio"]

Dockerfile.simpleci

@@ -0,0 +1,49 @@
+#-------------------------------------------------------------
+# Stage 1: Build and Unit tests
+#-------------------------------------------------------------
+FROM golang:1.11.4
+
+COPY . /go/src/github.com/minio/minio
+WORKDIR /go/src/github.com/minio/minio
+
+RUN apt-get update && apt-get install -y jq
+RUN make
+RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")'
+RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")'
+RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 15m "$d"; done
+RUN make verify
+RUN make coverage || true
+
+#-------------------------------------------------------------
+# Stage 2: Test Frontend
+#-------------------------------------------------------------
+FROM node:10.15-stretch-slim
+
+COPY browser /minio/browser
+WORKDIR /minio/browser
+
+RUN yarn
+RUN yarn test
+
+#-------------------------------------------------------------
+# Stage 3: Run Gateway Tests
+#-------------------------------------------------------------
+
+FROM ubuntu:16.04
+
+COPY --from=0 /go/src/github.com/minio/minio/minio ./minio
+COPY buildscripts/gateway-tests.sh ./gateway-tests.sh
+RUN apt-get update && apt-get install -y git wget jq curl dnsmasq
+
+RUN wget https://dl.google.com/go/go1.11.4.linux-amd64.tar.gz && \
+	tar -C /usr/local -xzf go1.11.4.linux-amd64.tar.gz
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV LANG C.UTF-8
+ENV GOROOT /usr/local/go
+
+RUN mkdir -p /go
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+
+RUN ./gateway-tests.sh

Makefile

@@ -14,43 +14,32 @@ checks:
 
 getdeps:
 	@echo "Installing golint" && go get -u golang.org/x/lint/golint
-	@echo "Installing gocyclo" && go get -u github.com/fzipp/gocyclo
-	@echo "Installing deadcode" && go get -u github.com/remyoudompheng/go-misc/deadcode
+	@echo "Installing staticcheck" && go get -u honnef.co/go/tools/...
 	@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
-	@echo "Installing ineffassign" && go get -u github.com/gordonklaus/ineffassign
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
 
-verifiers: getdeps vet fmt lint cyclo deadcode spelling
+verifiers: getdeps vet fmt lint staticcheck spelling
 
 vet:
 	@echo "Running $@"
-	@go tool vet cmd
-	@go tool vet pkg
+	@go vet github.com/minio/minio/...
 
 fmt:
 	@echo "Running $@"
-	@gofmt -d cmd
-	@gofmt -d pkg
+	@gofmt -d cmd/
+	@gofmt -d pkg/
 
 lint:
 	@echo "Running $@"
-	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd...
-	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg...
+	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd/...
+	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg/...
 
-ineffassign:
+staticcheck:
 	@echo "Running $@"
-	@${GOPATH}/bin/ineffassign .
-
-cyclo:
-	@echo "Running $@"
-	@${GOPATH}/bin/gocyclo -over 200 cmd
-	@${GOPATH}/bin/gocyclo -over 200 pkg
-
-deadcode:
-	@echo "Running $@"
-	@${GOPATH}/bin/deadcode -test $(shell go list ./...) || true
+	@${GOPATH}/bin/staticcheck github.com/minio/minio/cmd/...
+	@${GOPATH}/bin/staticcheck github.com/minio/minio/pkg/...
 
 spelling:
 	@${GOPATH}/bin/misspell -locale US -error `find cmd/`
@@ -77,6 +66,7 @@ coverage: build
 build: checks
 	@echo "Building minio binary to './minio'"
 	@GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
+	@GOFLAGS="" CGO_ENABLED=0 go build --ldflags="-s -w" -o $(PWD)/dockerscripts/healthcheck $(PWD)/dockerscripts/healthcheck.go
 
 docker: build
 	@docker build -t $(TAG) . -f Dockerfile.dev
@@ -105,6 +95,7 @@ install: build
 clean:
 	@echo "Cleaning up all the generated files"
 	@find . -name '*.test' | xargs rm -fv
+	@find . -name '*~' | xargs rm -fv
 	@rm -rvf minio
 	@rm -rvf build
 	@rm -rvf release

buildscripts/gateway-tests.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# Minio Cloud Storage, (C) 2019 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+set -E
+set -o pipefail
+
+export SERVER_ENDPOINT=127.0.0.1:24240
+export ENABLE_HTTPS=0
+export ACCESS_KEY=minio
+export SECRET_KEY=minio123
+export MINIO_ACCESS_KEY=minio
+export MINIO_SECRET_KEY=minio123
+export AWS_ACCESS_KEY_ID=minio
+export AWS_SECRET_ACCESS_KEY=minio123
+
+trap "cat server.log;cat gateway.log" SIGHUP SIGINT SIGTERM
+
+./minio --quiet --json server data --address 127.0.0.1:24242 > server.log &
+sleep 3
+./minio --quiet --json gateway s3 http://127.0.0.1:24242 --address 127.0.0.1:24240 > gateway.log &
+sleep 3
+
+mkdir -p /mint
+git clone https://github.com/minio/mint /mint
+cd /mint
+
+export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint}
+export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core"
+export MINT_RUN_SECURITY_DIR="$MINT_ROOT_DIR/run/security"
+export WGET="wget --quiet --no-check-certificate"
+
+go get github.com/go-ini/ini
+
+./create-data-files.sh
+./preinstall.sh
+
+# install mint app packages
+for pkg in "build"/*/install.sh; do
+    echo "Running $pkg"
+    $pkg
+done
+
+./postinstall.sh
+
+/mint/entrypoint.sh || cat server.log gateway.log fail

cmd/acl-handlers.go

@@ -64,21 +64,21 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Allow getBucketACL if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Before proceeding validate if bucket exists.
 	_, err := objAPI.GetBucketInfo(ctx, bucket)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -92,7 +92,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 		Permission: "FULL_CONTROL",
 	})
 	if err := xml.NewEncoder(w).Encode(acl); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -114,21 +114,21 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Allow getObjectACL if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Before proceeding validate if object exists.
 	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -142,7 +142,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 		Permission: "FULL_CONTROL",
 	})
 	if err := xml.NewEncoder(w).Encode(acl); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/admin-handlers_test.go

@@ -29,7 +29,6 @@ import (
 	"strings"
 	"sync"
 	"testing"
-	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/pkg/auth"
@@ -231,10 +230,9 @@ var (
 // adminXLTestBed - encapsulates subsystems that need to be setup for
 // admin-handler unit tests.
 type adminXLTestBed struct {
-	configPath string
-	xlDirs     []string
-	objLayer   ObjectLayer
-	router     *mux.Router
+	xlDirs   []string
+	objLayer ObjectLayer
+	router   *mux.Router
 }
 
 // prepareAdminXLTestBed - helper function that setups a single-node
@@ -270,10 +268,16 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	// Init global heal state
 	initAllHealState(globalIsXL)
 
-	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
+	globalConfigSys = NewConfigSys()
+
+	globalIAMSys = NewIAMSys()
+	globalIAMSys.Init(objLayer)
 
-	// Create new policy system.
 	globalPolicySys = NewPolicySys()
+	globalPolicySys.Init(objLayer)
+
+	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
+	globalNotificationSys.Init(objLayer)
 
 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()
@@ -309,7 +313,7 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) {
 			objectName := fmt.Sprintf("%s-%d", objName, i)
 			_, err = atb.objLayer.PutObject(context.Background(), bucketName, objectName,
 				mustGetPutObjReader(t, bytes.NewReader([]byte("hello")),
-					int64(len("hello")), "", ""), nil, ObjectOptions{})
+					int64(len("hello")), "", ""), ObjectOptions{})
 			if err != nil {
 				t.Fatalf("Failed to create %s - %v", objectName,
 					err)
@@ -321,7 +325,7 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) {
 	{
 		objName := "mpObject"
 		uploadID, err := atb.objLayer.NewMultipartUpload(context.Background(), bucketName,
-			objName, nil, ObjectOptions{})
+			objName, ObjectOptions{})
 		if err != nil {
 			t.Fatalf("mp new error: %v", err)
 		}
@@ -767,7 +771,7 @@ func TestSetConfigHandler(t *testing.T) {
 
 		rec := httptest.NewRecorder()
 		adminTestBed.router.ServeHTTP(rec, req)
-		respBody := string(rec.Body.Bytes())
+		respBody := rec.Body.String()
 		if rec.Code != http.StatusBadRequest ||
 			!strings.Contains(respBody, "Configuration data provided exceeds the allowed maximum of") {
 			t.Errorf("Got unexpected response code or body %d - %s", rec.Code, respBody)
@@ -786,7 +790,7 @@ func TestSetConfigHandler(t *testing.T) {
 
 		rec := httptest.NewRecorder()
 		adminTestBed.router.ServeHTTP(rec, req)
-		respBody := string(rec.Body.Bytes())
+		respBody := rec.Body.String()
 		if rec.Code != http.StatusBadRequest ||
 			!strings.Contains(respBody, "JSON configuration provided is of incorrect format") {
 			t.Errorf("Got unexpected response code or body %d - %s", rec.Code, respBody)
@@ -842,8 +846,8 @@ func TestAdminServerInfo(t *testing.T) {
 	}
 }
 
-// TestToAdminAPIErr - test for toAdminAPIErr helper function.
-func TestToAdminAPIErr(t *testing.T) {
+// TestToAdminAPIErrCode - test for toAdminAPIErrCode helper function.
+func TestToAdminAPIErrCode(t *testing.T) {
 	testCases := []struct {
 		err            error
 		expectedAPIErr APIErrorCode
@@ -873,168 +877,3 @@ func TestToAdminAPIErr(t *testing.T) {
 		}
 	}
 }
-
-func mkHealStartReq(t *testing.T, bucket, prefix string,
-	opts madmin.HealOpts) *http.Request {
-
-	body, err := json.Marshal(opts)
-	if err != nil {
-		t.Fatalf("Unable marshal heal opts")
-	}
-
-	path := fmt.Sprintf("/minio/admin/v1/heal/%s", bucket)
-	if bucket != "" && prefix != "" {
-		path += "/" + prefix
-	}
-
-	req, err := newTestRequest("POST", path,
-		int64(len(body)), bytes.NewReader(body))
-	if err != nil {
-		t.Fatalf("Failed to construct request - %v", err)
-	}
-	cred := globalServerConfig.GetCredential()
-	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-	if err != nil {
-		t.Fatalf("Failed to sign request - %v", err)
-	}
-
-	return req
-}
-
-func mkHealStatusReq(t *testing.T, bucket, prefix,
-	clientToken string) *http.Request {
-
-	path := fmt.Sprintf("/minio/admin/v1/heal/%s", bucket)
-	if bucket != "" && prefix != "" {
-		path += "/" + prefix
-	}
-	path += fmt.Sprintf("?clientToken=%s", clientToken)
-
-	req, err := newTestRequest("POST", path, 0, nil)
-	if err != nil {
-		t.Fatalf("Failed to construct request - %v", err)
-	}
-	cred := globalServerConfig.GetCredential()
-	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-	if err != nil {
-		t.Fatalf("Failed to sign request - %v", err)
-	}
-
-	return req
-}
-
-func collectHealResults(t *testing.T, adminTestBed *adminXLTestBed, bucket,
-	prefix, clientToken string, timeLimitSecs int) madmin.HealTaskStatus {
-
-	var res, cur madmin.HealTaskStatus
-
-	// loop and fetch heal status. have a time-limit to loop over
-	// all statuses.
-	timeLimit := UTCNow().Add(time.Second * time.Duration(timeLimitSecs))
-	for cur.Summary != healStoppedStatus && cur.Summary != healFinishedStatus {
-		if UTCNow().After(timeLimit) {
-			t.Fatalf("heal-status loop took too long - clientToken: %s", clientToken)
-		}
-		req := mkHealStatusReq(t, bucket, prefix, clientToken)
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		if http.StatusOK != rec.Code {
-			t.Errorf("Unexpected status code - got %d but expected %d",
-				rec.Code, http.StatusOK)
-			break
-		}
-		err := json.NewDecoder(rec.Body).Decode(&cur)
-		if err != nil {
-			t.Errorf("unable to unmarshal resp: %v", err)
-			break
-		}
-
-		// all results are accumulated into a slice
-		// and returned to caller in the end
-		allItems := append(res.Items, cur.Items...)
-		res = cur
-		res.Items = allItems
-
-		time.Sleep(time.Millisecond * 200)
-	}
-
-	return res
-}
-
-func TestHealStartNStatusHandler(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
-	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
-	}
-	defer adminTestBed.TearDown()
-
-	// gen. test data
-	adminTestBed.GenerateHealTestData(t)
-	defer adminTestBed.CleanupHealTestData(t)
-
-	// Prepare heal-start request to send to the server.
-	healOpts := madmin.HealOpts{
-		Recursive: true,
-		DryRun:    false,
-	}
-	bucketName, objName := "mybucket", "myobject-0"
-	var hss madmin.HealStartSuccess
-
-	{
-		req := mkHealStartReq(t, bucketName, objName, healOpts)
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		if http.StatusOK != rec.Code {
-			t.Errorf("Unexpected status code - got %d but expected %d",
-				rec.Code, http.StatusOK)
-		}
-
-		err = json.Unmarshal(rec.Body.Bytes(), &hss)
-		if err != nil {
-			t.Fatal("unable to unmarshal response")
-		}
-
-		if hss.ClientToken == "" {
-			t.Errorf("unexpected result")
-		}
-	}
-
-	{
-		// test with an invalid client token
-		req := mkHealStatusReq(t, bucketName, objName, hss.ClientToken+hss.ClientToken)
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		if rec.Code != http.StatusBadRequest {
-			t.Errorf("Unexpected status code")
-		}
-	}
-
-	{
-		// fetch heal status
-		results := collectHealResults(t, adminTestBed, bucketName,
-			objName, hss.ClientToken, 5)
-
-		// check if we got back an expected record
-		foundIt := false
-		for _, item := range results.Items {
-			if item.Type == madmin.HealItemObject &&
-				item.Bucket == bucketName && item.Object == objName {
-				foundIt = true
-			}
-		}
-		if !foundIt {
-			t.Error("did not find expected heal record in heal results")
-		}
-
-		// check that the heal settings in the results is the
-		// same as what we started the heal seq. with.
-		if results.HealSettings != healOpts {
-			t.Errorf("unexpected heal settings: %v",
-				results.HealSettings)
-		}
-
-		if results.Summary == healStoppedStatus {
-			t.Errorf("heal sequence stopped unexpectedly")
-		}
-	}
-}

cmd/admin-heal-ops.go

@@ -63,8 +63,7 @@ var (
 	errHealStopSignalled    = fmt.Errorf("heal stop signaled")
 
 	errFnHealFromAPIErr = func(ctx context.Context, err error) error {
-		errCode := toAPIErrorCode(ctx, err)
-		apiErr := getAPIError(errCode)
+		apiErr := toAPIError(ctx, err)
 		return fmt.Errorf("Heal internal error: %s: %s",
 			apiErr.Code, apiErr.Description)
 	}
@@ -150,7 +149,7 @@ func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists b
 	return h, exists
 }
 
-func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIErrorCode) {
+func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
 	var hsp madmin.HealStopSuccess
 	he, exists := ahs.getHealSequence(path)
 	if !exists {
@@ -176,7 +175,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIErrorCode) {
 	}
 
 	b, err := json.Marshal(&hsp)
-	return b, toAdminAPIErrCode(context.Background(), err)
+	return b, toAdminAPIErr(context.Background(), err)
 }
 
 // LaunchNewHealSequence - launches a background routine that performs
@@ -190,7 +189,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIErrorCode) {
 // background routine to clean up heal results after the
 // aforementioned duration.
 func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
-	respBytes []byte, errCode APIErrorCode, errMsg string) {
+	respBytes []byte, apiErr APIError, errMsg string) {
 
 	existsAndLive := false
 	he, exists := ahs.getHealSequence(h.path)
@@ -213,8 +212,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 				"(use force-start option to stop and start afresh). " +
 				fmt.Sprintf("The heal was started by IP %s at %s, token is %s",
 					h.clientAddress, h.startTime.Format(http.TimeFormat), h.clientToken)
-
-			return nil, ErrHealAlreadyRunning, errMsg
+			return nil, errorCodes.ToAPIErr(ErrHealAlreadyRunning), errMsg
 		}
 	}
 
@@ -229,7 +227,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 
 			errMsg = "The provided heal sequence path overlaps with an existing " +
 				fmt.Sprintf("heal path: %s", k)
-			return nil, ErrHealOverlappingPaths, errMsg
+			return nil, errorCodes.ToAPIErr(ErrHealOverlappingPaths), errMsg
 		}
 	}
 
@@ -246,9 +244,9 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	})
 	if err != nil {
 		logger.LogIf(h.ctx, err)
-		return nil, ErrInternalError, ""
+		return nil, toAPIError(h.ctx, err), ""
 	}
-	return b, ErrNone, ""
+	return b, noError, ""
 }
 
 // PopHealStatusJSON - Called by heal-status API. It fetches the heal
@@ -546,8 +544,11 @@ func (h *healSequence) traverseAndHeal() {
 	// Start with format healing
 	checkErr(h.healDiskFormat)
 
-	// Start healing the config.
-	checkErr(h.healConfig)
+	// Start healing the config prefix.
+	checkErr(h.healMinioSysMeta(minioConfigPrefix))
+
+	// Start healing the bucket config prefix.
+	checkErr(h.healMinioSysMeta(bucketConfigPrefix))
 
 	// Heal buckets and objects
 	checkErr(h.healBuckets)
@@ -559,63 +560,65 @@ func (h *healSequence) traverseAndHeal() {
 	close(h.traverseAndHealDoneCh)
 }
 
-// healConfig - heals config.json, retrun value indicates if a failure occurred.
-func (h *healSequence) healConfig() error {
-	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
+// healMinioSysMeta - heals all files under a given meta prefix, returns a function
+// which in-turn heals the respective meta directory path and any files in int.
+func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
+	return func() error {
+		// Get current object layer instance.
+		objectAPI := newObjectLayerFn()
+		if objectAPI == nil {
+			return errServerNotInitialized
+		}
+
+		// NOTE: Healing on meta is run regardless
+		// of any bucket being selected, this is to ensure that
+		// meta are always upto date and correct.
+		marker := ""
+		isTruncated := true
+		for isTruncated {
+			if globalHTTPServer != nil {
+				// Wait at max 1 minute for an inprogress request
+				// before proceeding to heal
+				waitCount := 60
+				// Any requests in progress, delay the heal.
+				for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
+					waitCount--
+					time.Sleep(1 * time.Second)
+				}
+			}
+
+			// Lists all objects under `config` prefix.
+			objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, minioMetaBucket, metaPrefix,
+				marker, "", 1000)
+			if err != nil {
+				return errFnHealFromAPIErr(h.ctx, err)
+			}
+
+			for index := range objectInfos.Objects {
+				if h.isQuitting() {
+					return errHealStopSignalled
+				}
+				o := objectInfos.Objects[index]
+				res, herr := objectAPI.HealObject(h.ctx, o.Bucket, o.Name, h.settings.DryRun, h.settings.Remove)
+				// Object might have been deleted, by the time heal
+				// was attempted we ignore this file an move on.
+				if isErrObjectNotFound(herr) {
+					continue
+				}
+				if herr != nil {
+					return herr
+				}
+				res.Type = madmin.HealItemBucketMetadata
+				if err = h.pushHealResultItem(res); err != nil {
+					return err
+				}
+			}
+
+			isTruncated = objectInfos.IsTruncated
+			marker = objectInfos.NextMarker
+		}
+		return nil
 	}
-
-	// NOTE: Healing on configs is run regardless
-	// of any bucket being selected, this is to ensure that
-	// configs are always uptodate and correct.
-	marker := ""
-	isTruncated := true
-	for isTruncated {
-		if globalHTTPServer != nil {
-			// Wait at max 1 minute for an inprogress request
-			// before proceeding to heal
-			waitCount := 60
-			// Any requests in progress, delay the heal.
-			for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
-				waitCount--
-				time.Sleep(1 * time.Second)
-			}
-		}
-
-		// Lists all objects under `config` prefix.
-		objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, minioMetaBucket, minioConfigPrefix,
-			marker, "", 1000)
-		if err != nil {
-			return errFnHealFromAPIErr(h.ctx, err)
-		}
-
-		for index := range objectInfos.Objects {
-			if h.isQuitting() {
-				return errHealStopSignalled
-			}
-			o := objectInfos.Objects[index]
-			res, herr := objectAPI.HealObject(h.ctx, o.Bucket, o.Name, h.settings.DryRun)
-			// Object might have been deleted, by the time heal
-			// was attempted we ignore this file an move on.
-			if isErrObjectNotFound(herr) {
-				continue
-			}
-			if herr != nil {
-				return herr
-			}
-			res.Type = madmin.HealItemBucketMetadata
-			if err = h.pushHealResultItem(res); err != nil {
-				return err
-			}
-		}
-
-		isTruncated = objectInfos.IsTruncated
-		marker = objectInfos.NextMarker
-	}
-
-	return nil
 }
 
 // healDiskFormat - heals format.json, return value indicates if a
@@ -692,18 +695,16 @@ func (h *healSequence) healBucket(bucket string) error {
 		return errServerNotInitialized
 	}
 
-	results, err := objectAPI.HealBucket(h.ctx, bucket, h.settings.DryRun)
-	// push any available results before checking for error
-	for _, result := range results {
-		if perr := h.pushHealResultItem(result); perr != nil {
-			return perr
-		}
-	}
+	result, err := objectAPI.HealBucket(h.ctx, bucket, h.settings.DryRun, h.settings.Remove)
 	// handle heal-bucket error
 	if err != nil {
 		return err
 	}
 
+	if err = h.pushHealResultItem(result); err != nil {
+		return err
+	}
+
 	if !h.settings.Recursive {
 		if h.objPrefix != "" {
 			// Check if an object named as the objPrefix exists,
@@ -775,7 +776,7 @@ func (h *healSequence) healObject(bucket, object string) error {
 		return errServerNotInitialized
 	}
 
-	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun)
+	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun, h.settings.Remove)
 	if isErrObjectNotFound(err) {
 		return nil
 	}

cmd/api-errors.go

@@ -22,6 +22,11 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"google.golang.org/api/googleapi"
+
+	minio "github.com/minio/minio-go"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
@@ -212,7 +217,7 @@ const (
 	ErrHealOverlappingPaths
 	ErrIncorrectContinuationToken
 
-	//S3 Select Errors
+	// S3 Select Errors
 	ErrEmptyRequestBody
 	ErrUnsupportedFunction
 	ErrInvalidExpressionType
@@ -301,11 +306,22 @@ const (
 	ErrAdminConfigNotificationTargetsFailed
 	ErrAdminProfilerNotEnabled
 	ErrInvalidDecompressedSize
+	ErrAddUserInvalidArgument
 )
 
+type errorCodeMap map[APIErrorCode]APIError
+
+func (e errorCodeMap) ToAPIErr(errCode APIErrorCode) APIError {
+	apiErr, ok := e[errCode]
+	if !ok {
+		return e[ErrInternalError]
+	}
+	return apiErr
+}
+
 // error code to APIError structure, these fields carry respective
 // descriptions for all the error responses.
-var errorCodeResponse = map[APIErrorCode]APIError{
+var errorCodes = errorCodeMap{
 	ErrInvalidCopyDest: {
 		Code:           "InvalidRequest",
 		Description:    "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.",
@@ -1447,6 +1463,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "The data provided is unfit for decompression",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAddUserInvalidArgument: {
+		Code:           "XMinioInvalidIAMCredentials",
+		Description:    "User is not allowed to be same as admin access key",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	// Add your error structure here.
 }
 
@@ -1630,22 +1651,79 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	return apiErr
 }
 
+var noError = APIError{}
+
+// toAPIError - Converts embedded errors. Convenience
+// function written to handle all cases where we have known types of
+// errors returned by underlying layers.
+func toAPIError(ctx context.Context, err error) APIError {
+	if err == nil {
+		return noError
+	}
+
+	var apiErr = errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
+	if apiErr.Code == "InternalError" {
+		// If we see an internal error try to interpret
+		// any underlying errors if possible depending on
+		// their internal error types. This code is only
+		// useful with gateway implementations.
+		switch e := err.(type) {
+		case minio.ErrorResponse:
+			apiErr = APIError{
+				Code:           e.Code,
+				Description:    e.Message,
+				HTTPStatusCode: e.StatusCode,
+			}
+		case *googleapi.Error:
+			apiErr = APIError{
+				Code:           "XGCSInternalError",
+				Description:    e.Message,
+				HTTPStatusCode: e.Code,
+			}
+			// GCS may send multiple errors, just pick the first one
+			// since S3 only sends one Error XML response.
+			if len(e.Errors) >= 1 {
+				apiErr.Code = e.Errors[0].Reason
+
+			}
+		case storage.AzureStorageServiceError:
+			apiErr = APIError{
+				Code:           e.Code,
+				Description:    e.Message,
+				HTTPStatusCode: e.StatusCode,
+			}
+		case oss.ServiceError:
+			apiErr = APIError{
+				Code:           e.Code,
+				Description:    e.Message,
+				HTTPStatusCode: e.StatusCode,
+			}
+			// Add more Gateway SDKs here if any in future.
+		}
+	}
+
+	return apiErr
+}
+
 // getAPIError provides API Error for input API error code.
 func getAPIError(code APIErrorCode) APIError {
-	if apiErr, ok := errorCodeResponse[code]; ok {
+	if apiErr, ok := errorCodes[code]; ok {
 		return apiErr
 	}
-	return errorCodeResponse[ErrInternalError]
+	return errorCodes.ToAPIErr(ErrInternalError)
 }
 
 // getErrorResponse gets in standard error and resource value and
 // provides a encodable populated response values
-func getAPIErrorResponse(err APIError, resource, requestid string) APIErrorResponse {
+func getAPIErrorResponse(ctx context.Context, err APIError, resource, requestID, hostID string) APIErrorResponse {
+	reqInfo := logger.GetReqInfo(ctx)
 	return APIErrorResponse{
-		Code:      err.Code,
-		Message:   err.Description,
-		Resource:  resource,
-		RequestID: requestid,
-		HostID:    "3L137",
+		Code:       err.Code,
+		Message:    err.Description,
+		BucketName: reqInfo.BucketName,
+		Key:        reqInfo.ObjectName,
+		Resource:   resource,
+		RequestID:  requestID,
+		HostID:     hostID,
 	}
 }

cmd/api-errors_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/minio/minio/pkg/hash"
 )
 
-var toAPIErrorCodeTests = []struct {
+var toAPIErrorTests = []struct {
 	err     error
 	errCode APIErrorCode
 }{
@@ -66,7 +66,7 @@ var toAPIErrorCodeTests = []struct {
 
 func TestAPIErrCode(t *testing.T) {
 	ctx := context.Background()
-	for i, testCase := range toAPIErrorCodeTests {
+	for i, testCase := range toAPIErrorTests {
 		errCode := toAPIErrorCode(ctx, testCase.err)
 		if errCode != testCase.errCode {
 			t.Errorf("Test %d: Expected error code %d, got %d", i+1, testCase.errCode, errCode)

cmd/api-response-multipart.go

@@ -43,11 +43,12 @@ type completeMultipartAPIError struct {
 // XML doesn't carry the additional fields required to send this
 // error. So we construct a new type which lies well within the scope
 // of this function.
-func writePartSmallErrorResponse(w http.ResponseWriter, r *http.Request, err PartTooSmall) {
-
-	apiError := getAPIError(toAPIErrorCode(context.Background(), err))
+func writePartSmallErrorResponse(ctx context.Context, w http.ResponseWriter, r *http.Request, err PartTooSmall) {
+	apiError := toAPIError(ctx, err)
 	// Generate complete multipart error response.
-	errorResponse := getAPIErrorResponse(apiError, r.URL.Path, w.Header().Get(responseRequestIDKey))
+	errorResponse := getAPIErrorResponse(ctx, apiError, r.URL.Path,
+		w.Header().Get(responseRequestIDKey),
+		w.Header().Get(responseDeploymentIDKey))
 	cmpErrResp := completeMultipartAPIError{err.PartSize, int64(5242880), err.PartNumber, err.PartETag, errorResponse}
 	encodedErrorResponse := encodeResponse(cmpErrResp)
 

cmd/api-response.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"encoding/xml"
 	"net/http"
 	"net/url"
@@ -24,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/handlers"
 )
 
@@ -568,13 +570,13 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
 }
 
 // writeErrorRespone writes error headers
-func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL, browser bool) {
-	switch errorCode {
-	case ErrSlowDown, ErrServerNotInitialized, ErrReadQuorum, ErrWriteQuorum:
+func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL, browser bool) {
+	switch err.Code {
+	case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
 		// Set retry-after header to indicate user-agents to retry request after 120secs.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
 		w.Header().Set("Retry-After", "120")
-	case ErrAccessDenied:
+	case "AccessDenied":
 		// The request is from browser and also if browser
 		// is enabled we need to redirect.
 		if browser && globalIsBrowserEnabled {
@@ -584,42 +586,41 @@ func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *u
 		}
 	}
 
-	apiError := getAPIError(errorCode)
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))
+	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
 	encodedErrorResponse := encodeResponse(errorResponse)
-	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML)
+	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }
 
-func writeErrorResponseHeadersOnly(w http.ResponseWriter, errorCode APIErrorCode) {
-	apiError := getAPIError(errorCode)
-	writeResponse(w, apiError.HTTPStatusCode, nil, mimeNone)
+func writeErrorResponseHeadersOnly(w http.ResponseWriter, err APIError) {
+	writeResponse(w, err.HTTPStatusCode, nil, mimeNone)
 }
 
 // writeErrorResponseJSON - writes error response in JSON format;
 // useful for admin APIs.
-func writeErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
-	apiError := getAPIError(errorCode)
+func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))
+	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
-	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeJSON)
+	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }
 
 // writeCustomErrorResponseJSON - similar to writeErrorResponseJSON,
 // but accepts the error message directly (this allows messages to be
 // dynamically generated.)
-func writeCustomErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode,
+func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError,
 	errBody string, reqURL *url.URL) {
 
-	apiError := getAPIError(errorCode)
+	reqInfo := logger.GetReqInfo(ctx)
 	errorResponse := APIErrorResponse{
-		Code:      apiError.Code,
-		Message:   errBody,
-		Resource:  reqURL.Path,
-		RequestID: w.Header().Get(responseRequestIDKey),
-		HostID:    "3L137",
+		Code:       err.Code,
+		Message:    errBody,
+		Resource:   reqURL.Path,
+		BucketName: reqInfo.BucketName,
+		Key:        reqInfo.ObjectName,
+		RequestID:  w.Header().Get(responseRequestIDKey),
+		HostID:     w.Header().Get(responseDeploymentIDKey),
 	}
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
-	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeJSON)
+	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }

cmd/api-router.go

@@ -67,6 +67,8 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
 		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
 		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
+		// GetObjectTagging - this is a dummy call.
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectTaggingHandler)).Queries("tagging", "")
 		// SelectObjectContent
 		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.SelectObjectContentHandler)).Queries("select", "").Queries("select-type", "2")
 		// GetObject
@@ -84,8 +86,31 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
 		// GetBucketPolicy
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
 
+		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketACLHandler)).Queries("acl", "")
+		// GetBucketCors - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketCorsHandler)).Queries("cors", "")
+		// GetBucketWebsiteHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketWebsiteHandler)).Queries("website", "")
+		// GetBucketVersioningHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketVersioningHandler)).Queries("versioning", "")
+		// GetBucketAccelerateHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketAccelerateHandler)).Queries("accelerate", "")
+		// GetBucketRequestPaymentHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketRequestPaymentHandler)).Queries("requestPayment", "")
+		// GetBucketLoggingHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLoggingHandler)).Queries("logging", "")
+		// GetBucketLifecycleHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
+		// GetBucketReplicationHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketReplicationHandler)).Queries("replication", "")
+		// GetBucketTaggingHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketTaggingHandler)).Queries("tagging", "")
+		//DeleteBucketWebsiteHandler
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketWebsiteHandler)).Queries("website", "")
+		// DeleteBucketTaggingHandler
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketTaggingHandler)).Queries("tagging", "")
 
 		// GetBucketNotification
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")

cmd/auth-handler.go

@@ -416,7 +416,7 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		a.handler.ServeHTTP(w, r)
 		return
 	}
-	writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL, guessIsBrowserReq(r))
+	writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL, guessIsBrowserReq(r))
 }
 
 // isPutAllowed - check if PUT operation is allowed on the resource, this

cmd/auth-handler_test.go

@@ -383,7 +383,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 	for i, testCase := range testCases {
 		if s3Error := isReqAuthenticated(ctx, testCase.req, globalServerConfig.GetRegion()); s3Error != testCase.s3Error {
 			if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
-				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %d)", i, testCase.s3Error, s3Error, toAPIErrorCode(ctx, err))
+				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %s)", i, testCase.s3Error, s3Error, toAPIError(ctx, err).Code)
 			}
 		}
 	}

cmd/benchmark-utils_test.go

@@ -49,7 +49,6 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	textData := generateBytesData(objSize)
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
-	metadata := make(map[string]string)
 
 	md5hex := getMD5Hash(textData)
 	sha256hex := ""
@@ -61,7 +60,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	for i := 0; i < b.N; i++ {
 		// insert the object.
 		objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
+			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -96,14 +95,11 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	textData := generateBytesData(objSize)
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for NewMultipartUpload.
-	metadata := make(map[string]string)
-	opts := ObjectOptions{}
-	uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, metadata, opts)
+	uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, ObjectOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	md5hex := getMD5Hash(textData)
 	sha256hex := ""
 
 	var textPartData []byte
@@ -120,10 +116,10 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 			} else {
 				textPartData = textData[j*partSize:]
 			}
-			md5hex = getMD5Hash([]byte(textPartData))
+			md5hex := getMD5Hash([]byte(textPartData))
 			var partInfo PartInfo
 			partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
-				mustGetPutObjReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), opts)
+				mustGetPutObjReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -194,7 +190,6 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate etag for the generated data.
 	// etag of the data to written is required as input for PutObject.
 	// PutObject is the functions which writes the data onto the FS/XL backend.
-	metadata := make(map[string]string)
 
 	// get text data generated for number of bytes equal to object size.
 	md5hex := getMD5Hash(textData)
@@ -204,7 +199,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
+			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -234,10 +229,8 @@ func getRandomByte() []byte {
 	const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 	// seeding the random number generator.
 	rand.Seed(UTCNow().UnixNano())
-	var b byte
 	// pick a character randomly.
-	b = letterBytes[rand.Intn(len(letterBytes))]
-	return []byte{b}
+	return []byte{letterBytes[rand.Intn(len(letterBytes))]}
 }
 
 // picks a random byte and repeats it to size bytes.
@@ -289,7 +282,6 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	textData := generateBytesData(objSize)
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
-	metadata := make(map[string]string)
 
 	md5hex := getMD5Hash([]byte(textData))
 	sha256hex := ""
@@ -304,7 +296,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		for pb.Next() {
 			// insert the object.
 			objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-				mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
+				mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -335,7 +327,6 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
 	// PutObject is the functions which writes the data onto the FS/XL backend.
-	metadata := make(map[string]string)
 
 	md5hex := getMD5Hash([]byte(textData))
 	sha256hex := ""
@@ -344,7 +335,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
+			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}

cmd/bitrot-streaming.go

@@ -146,7 +146,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	}
 	b.h.Write(buf)
 
-	if bytes.Compare(b.h.Sum(nil), b.hashBytes) != 0 {
+	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
 		err = hashMismatchError{hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))}
 		logger.LogIf(context.Background(), err)
 		return 0, err

cmd/bucket-handlers-listobjects.go

@@ -66,12 +66,12 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -79,18 +79,18 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 
 	// Extract all the listObjectsV2 query params to their native values.
 	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _, errCode := getListObjectsV2Args(urlValues)
-
 	if errCode != ErrNone {
-		writeErrorResponse(w, errCode, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Validate the query params before beginning to serve the request.
 	// fetch-owner is not validated since it is a boolean
 	if s3Error := validateListObjectsArgs(prefix, token, delimiter, maxKeys); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	listObjectsV2 := objectAPI.ListObjectsV2
 	if api.CacheAPI() != nil {
 		listObjectsV2 = api.CacheAPI().ListObjectsV2
@@ -100,7 +100,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	// marshaled into S3 compatible XML header.
 	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -110,7 +110,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 			// Read the decompressed size from the meta.json.
 			actualSize = listObjectsV2Info.Objects[i].GetActualSize()
 			if actualSize < 0 {
-				writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
 				return
 			}
 			// Set the info.Size to the actualSize.
@@ -119,7 +119,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
 			listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 		}
@@ -148,42 +148,39 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Extract all the litsObjectsV1 query params to their native values.
 	prefix, marker, delimiter, maxKeys, _, s3Error := getListObjectsV1Args(r.URL.Query())
 	if s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	// Validate the maxKeys lowerbound. When maxKeys > 1000, S3 returns 1000 but
-	// does not throw an error.
-	if maxKeys < 0 {
-		writeErrorResponse(w, ErrInvalidMaxKeys, r.URL, guessIsBrowserReq(r))
-		return
-	} // Validate all the query params before beginning to serve the request.
+	// Validate all the query params before beginning to serve the request.
 	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	listObjects := objectAPI.ListObjects
 	if api.CacheAPI() != nil {
 		listObjects = api.CacheAPI().ListObjects
 	}
+
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
 	// marshaled into S3 compatible XML header.
 	listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -193,7 +190,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 			// Read the decompressed size from the meta.json.
 			actualSize = listObjectsInfo.Objects[i].GetActualSize()
 			if actualSize < 0 {
-				writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
 				return
 			}
 			// Set the info.Size to the actualSize.
@@ -202,7 +199,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
 			listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 		}

cmd/bucket-handlers.go

@@ -96,12 +96,12 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLocationAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -110,7 +110,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 		getBucketInfo = api.CacheAPI().GetBucketInfo
 	}
 	if _, err := getBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -146,35 +146,37 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketMultipartUploadsAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _, s3Error := getBucketMultipartResources(r.URL.Query())
-	if s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _, errCode := getBucketMultipartResources(r.URL.Query())
+	if errCode != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	if maxUploads < 0 {
-		writeErrorResponse(w, ErrInvalidMaxUploads, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidMaxUploads), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	if keyMarker != "" {
 		// Marker not common with prefix is not implemented.
 		if !hasPrefix(keyMarker, prefix) {
-			writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
 	listMultipartsInfo, err := objectAPI.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	// generate response
@@ -196,25 +198,26 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	listBuckets := objectAPI.ListBuckets
 
+	listBuckets := objectAPI.ListBuckets
 	if api.CacheAPI() != nil {
 		listBuckets = api.CacheAPI().ListBuckets
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListAllMyBucketsAction, "", ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	// If etcd, dns federation configured list buckets from etcd.
 	var bucketsInfo []BucketInfo
 	if globalDNSConfig != nil {
 		dnsBuckets, err := globalDNSConfig.List()
 		if err != nil && err != dns.ErrNoEntriesFound {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 		bucketSet := set.NewStringSet()
@@ -233,7 +236,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 		var err error
 		bucketsInfo, err = listBuckets(ctx)
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -257,7 +260,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -266,7 +269,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		// In the event access is denied, a 200 response should still be returned
 		// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 		if s3Error != ErrAccessDenied {
-			writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -274,14 +277,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Content-Length is required and should be non-zero
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Content-Md5 is requied should be set
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if _, ok := r.Header["Content-Md5"]; !ok {
-		writeErrorResponse(w, ErrMissingContentMD5, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -297,7 +300,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Read incoming body XML bytes.
 	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAdminAPIErr(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -305,7 +308,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	deleteObjects := &DeleteObjectsRequest{}
 	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedXML, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -313,7 +316,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	if globalWORMEnabled {
 		// Not required to check whether given objects exist or not, because
 		// DeleteMultipleObject is always successful irrespective of object existence.
-		writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -352,10 +355,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			deletedObjects = append(deletedObjects, object)
 			continue
 		}
+		apiErr := toAPIError(ctx, err)
 		// Error during delete should be collected separately.
 		deleteErrors = append(deleteErrors, DeleteError{
-			Code:    errorCodeResponse[toAPIErrorCode(ctx, err)].Code,
-			Message: errorCodeResponse[toAPIErrorCode(ctx, err)].Description,
+			Code:    apiErr.Code,
+			Message: apiErr.Description,
 			Key:     object.ObjectName,
 		})
 	}
@@ -401,7 +405,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -409,21 +413,21 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Parse incoming location constraint.
 	location, s3Error := parseLocationConstraint(r)
 	if s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Validate if location sent by the client is valid, reject
 	// requests which do not follow valid region requirements.
 	if !isValidLocation(location) {
-		writeErrorResponse(w, ErrInvalidRegion, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRegion), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -432,12 +436,12 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 			if err == dns.ErrNoEntriesFound {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
-					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 					return
 				}
 				if err = globalDNSConfig.Put(bucket); err != nil {
 					objectAPI.DeleteBucket(ctx, bucket)
-					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 					return
 				}
 
@@ -447,18 +451,18 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				writeSuccessResponseHeadersOnly(w)
 				return
 			}
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 
 		}
-		writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBucketAlreadyOwnedByYou), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -479,33 +483,36 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported
-		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	if !api.EncryptionEnabled() && hasServerSideEncryptionHeader(r.Header) {
-		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	bucket := mux.Vars(r)["bucket"]
 
 	// Require Content-Length to be set in the request
 	size := r.ContentLength
 	if size < 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
 	if err != nil {
-		writeErrorResponse(w, ErrInvalidRequest, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	// Make sure that the URL  does not contain object name.
 	if bucket != filepath.Clean(resource[1:]) {
-		writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -514,7 +521,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	reader, err := r.MultipartReader()
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -522,7 +529,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	form, err := reader.ReadForm(maxFormMemory)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -533,13 +540,13 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if file is provided, error out otherwise.
 	if fileBody == nil {
-		writeErrorResponse(w, ErrPOSTFileRequired, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPOSTFileRequired), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -561,21 +568,21 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if successRedirect != "" {
 		redirectURL, err = url.Parse(successRedirect)
 		if err != nil {
-			writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
 	// Verify policy signature.
-	apiErr := doesPolicySignatureMatch(formValues)
-	if apiErr != ErrNone {
-		writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
+	errCode := doesPolicySignatureMatch(formValues)
+	if errCode != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
 	if err != nil {
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -583,13 +590,13 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if len(policyBytes) > 0 {
 		postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
 		if err != nil {
-			writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
 		// Make sure formValues adhere to policy restrictions.
-		if apiErr = checkPostPolicy(formValues, postPolicyForm); apiErr != ErrNone {
-			writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
+		if errCode = checkPostPolicy(formValues, postPolicyForm); errCode != ErrNone {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -598,12 +605,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		lengthRange := postPolicyForm.Conditions.ContentLengthRange
 		if lengthRange.Valid {
 			if fileSize < lengthRange.Min {
-				writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooSmall), r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, toAPIError(ctx, errDataTooSmall), r.URL, guessIsBrowserReq(r))
 				return
 			}
 
 			if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
-				writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooLarge), r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, toAPIError(ctx, errDataTooLarge), r.URL, guessIsBrowserReq(r))
 				return
 			}
 		}
@@ -613,14 +620,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	metadata := make(map[string]string)
 	err = extractMetadataFromMap(ctx, formValues, metadata)
 	if err != nil {
-		writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	rawReader := hashReader
@@ -633,9 +640,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 	// get gateway encryption options
 	var opts ObjectOptions
-	opts, err = putEncryptionOpts(ctx, r, bucket, object, nil)
+	opts, err = putOpts(ctx, r, bucket, object, metadata)
 	if err != nil {
-		writeErrorResponseHeadersOnly(w, toAPIErrorCode(ctx, err))
+		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 		return
 	}
 	if objectAPI.IsEncryptionSupported() {
@@ -645,28 +652,28 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			if crypto.SSEC.IsRequested(formValues) {
 				key, err = ParseSSECustomerHeader(formValues)
 				if err != nil {
-					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 					return
 				}
 			}
 			reader, objectEncryptionKey, err = newEncryptReader(hashReader, key, bucket, object, metadata, crypto.S3.IsRequested(formValues))
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 			info := ObjectInfo{Size: fileSize}
 			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize) // do not try to verify encrypted content
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 			pReader = NewPutObjReader(rawReader, hashReader, objectEncryptionKey)
 		}
 	}
 
-	objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, metadata, opts)
+	objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, opts)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -732,12 +739,12 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponseHeadersOnly(w, ErrServerNotInitialized)
+		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrServerNotInitialized))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponseHeadersOnly(w, s3Error)
+		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
 		return
 	}
 
@@ -746,7 +753,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 		getBucketInfo = api.CacheAPI().GetBucketInfo
 	}
 	if _, err := getBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponseHeadersOnly(w, toAPIErrorCode(ctx, err))
+		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 		return
 	}
 
@@ -764,12 +771,12 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -779,7 +786,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	}
 	// Attempt to delete bucket.
 	if err := deleteBucket(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -791,7 +798,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		if err := globalDNSConfig.Delete(bucket); err != nil {
 			// Deleting DNS entry failed, attempt to create the bucket again.
 			objectAPI.MakeBucketWithLocation(ctx, bucket, "")
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}

cmd/bucket-handlers_test.go

@@ -95,12 +95,12 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
 		}
 		if !bytes.Equal(testCase.locationResponse, rec.Body.Bytes()) && testCase.shouldPass {
-			t.Errorf("Test %d: %s: Expected the response to be `%s`, but instead found `%s`", i+1, instanceType, string(testCase.locationResponse), string(rec.Body.Bytes()))
+			t.Errorf("Test %d: %s: Expected the response to be `%s`, but instead found `%s`", i+1, instanceType, string(testCase.locationResponse), rec.Body.String())
 		}
 		errorResponse := APIErrorResponse{}
 		err = xml.Unmarshal(rec.Body.Bytes(), &errorResponse)
 		if err != nil && !testCase.shouldPass {
-			t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, string(rec.Body.Bytes()))
+			t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, rec.Body.String())
 		}
 		if errorResponse.Resource != testCase.errorResponse.Resource {
 			t.Errorf("Test %d: %s: Expected the error resource to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Resource, errorResponse.Resource)
@@ -131,7 +131,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 		errorResponse = APIErrorResponse{}
 		err = xml.Unmarshal(recV2.Body.Bytes(), &errorResponse)
 		if err != nil && !testCase.shouldPass {
-			t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, string(recV2.Body.Bytes()))
+			t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, recV2.Body.String())
 		}
 		if errorResponse.Resource != testCase.errorResponse.Resource {
 			t.Errorf("Test %d: %s: Expected the error resource to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Resource, errorResponse.Resource)
@@ -625,7 +625,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := 0; i < 10; i++ {
 		objectName := "test-object-" + strconv.Itoa(i)
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil, ObjectOptions{})
+		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object %d:  Error uploading object: <ERROR> %v", i, err)
@@ -645,8 +645,8 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	getDeleteErrorList := func(objects []ObjectIdentifier) (deleteErrorList []DeleteError) {
 		for _, obj := range objects {
 			deleteErrorList = append(deleteErrorList, DeleteError{
-				Code:    errorCodeResponse[ErrAccessDenied].Code,
-				Message: errorCodeResponse[ErrAccessDenied].Description,
+				Code:    errorCodes[ErrAccessDenied].Code,
+				Message: errorCodes[ErrAccessDenied].Description,
 				Key:     obj.ObjectName,
 			})
 		}

cmd/bucket-notification-handlers.go

@@ -51,23 +51,23 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if !objAPI.IsNotificationSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	_, err := objAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -76,7 +76,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 	if err != nil {
 		// Ignore errNoSuchNotifications to comply with AWS S3.
 		if err != errNoSuchNotifications {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -90,7 +90,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 
 	notificationBytes, err := xml.Marshal(nConfig)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -106,12 +106,12 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if !objectAPI.IsNotificationSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -119,36 +119,36 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 	bucketName := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	_, err := objectAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// PutBucketNotification always needs a Content-Length.
 	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	var config *event.Config
 	config, err = event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalServerConfig.GetRegion(), globalNotificationSys.targetList)
 	if err != nil {
-		apiErr := ErrMalformedXML
+		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		if event.IsEventError(err) {
-			apiErr = toAPIErrorCode(ctx, err)
+			apiErr = toAPIError(ctx, err)
 		}
 
-		writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -169,23 +169,24 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	// Validate if bucket exists.
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	if !objAPI.IsNotificationSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if !objAPI.IsListenBucketSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListenBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -193,11 +194,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	var prefix string
 	if len(values["prefix"]) > 1 {
-		writeErrorResponse(w, ErrFilterNamePrefix, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrFilterNamePrefix), r.URL, guessIsBrowserReq(r))
+		return
 	}
+
 	if len(values["prefix"]) == 1 {
 		if err := event.ValidateFilterRuleValue(values["prefix"][0]); err != nil {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -206,11 +209,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	var suffix string
 	if len(values["suffix"]) > 1 {
-		writeErrorResponse(w, ErrFilterNameSuffix, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrFilterNameSuffix), r.URL, guessIsBrowserReq(r))
+		return
 	}
+
 	if len(values["suffix"]) == 1 {
 		if err := event.ValidateFilterRuleValue(values["suffix"][0]); err != nil {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -223,7 +228,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	for _, s := range values["events"] {
 		eventName, err := event.ParseName(s)
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -231,19 +236,19 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	}
 
 	if _, err := objAPI.GetBucketInfo(ctx, bucketName); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	host, err := xnet.ParseHost(r.RemoteAddr)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	target, err := target.NewHTTPClientTarget(*host, w)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -251,7 +256,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	if err = globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
@@ -259,13 +264,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -275,7 +280,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 }

cmd/bucket-policy-handlers.go

@@ -44,7 +44,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -52,43 +52,43 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Error out if Content-Length is missing.
 	// PutBucketPolicy always needs Content-Length.
 	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Error out if Content-Length is beyond allowed size.
 	if r.ContentLength > maxBucketPolicySize {
-		writeErrorResponse(w, ErrEntityTooLarge, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
 	if err != nil {
-		writeErrorResponse(w, ErrMalformedPolicy, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Version in policy must not be empty
 	if bucketPolicy.Version == "" {
-		writeErrorResponse(w, ErrMalformedPolicy, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err = objAPI.SetBucketPolicy(ctx, bucket, bucketPolicy); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -107,7 +107,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -115,18 +115,18 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err := objAPI.DeleteBucketPolicy(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -145,7 +145,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -153,26 +153,26 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Read bucket access policy.
 	bucketPolicy, err := objAPI.GetBucketPolicy(ctx, bucket)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	policyData, err := json.Marshal(bucketPolicy)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/common-main.go

@@ -122,7 +122,7 @@ func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() s
 func handleCommonCmdArgs(ctx *cli.Context) {
 
 	// Get "json" flag from command line argument and
-	// enable json and quite modes if jason flag is turned on.
+	// enable json and quite modes if json flag is turned on.
 	globalCLIContext.JSON = ctx.IsSet("json") || ctx.GlobalIsSet("json")
 	if globalCLIContext.JSON {
 		logger.EnableJSON()

cmd/config-common.go

@@ -78,7 +78,7 @@ func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data
 		return err
 	}
 
-	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader, nil, nil), nil, ObjectOptions{})
+	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader, nil, nil), ObjectOptions{})
 	return err
 }
 

cmd/config-current.go

@@ -25,6 +25,7 @@ import (
 	"sync"
 
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
@@ -282,7 +283,7 @@ func (s *serverConfig) loadFromEnvs() {
 	if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok {
 		if u, err := xnet.ParseURL(jwksURL); err == nil {
 			s.OpenID.JWKS.URL = u
-			s.OpenID.JWKS.PopulatePublicKey()
+			logger.FatalIf(s.OpenID.JWKS.PopulatePublicKey(), "Unable to populate public key from JWKS URL")
 		}
 	}
 
@@ -552,7 +553,7 @@ func (s *serverConfig) loadToCachedConfigs() {
 			URL:         s.Policy.OPA.URL,
 			AuthToken:   s.Policy.OPA.AuthToken,
 			Transport:   NewCustomHTTPTransport(),
-			CloseRespFn: CloseResponse,
+			CloseRespFn: xhttp.DrainBody,
 		})
 	}
 }

cmd/config-versions.go

@@ -263,9 +263,6 @@ type serverConfigV7 struct {
 
 	// Notification queue configuration.
 	Notify notifierV1 `json:"notify"`
-
-	// Read Write mutex.
-	rwMutex *sync.RWMutex
 }
 
 // serverConfigV8 server configuration version '8'. Adds NATS notifier
@@ -282,9 +279,6 @@ type serverConfigV8 struct {
 
 	// Notification queue configuration.
 	Notify notifierV1 `json:"notify"`
-
-	// Read Write mutex.
-	rwMutex *sync.RWMutex
 }
 
 // serverConfigV9 server configuration version '9'. Adds PostgreSQL
@@ -301,9 +295,6 @@ type serverConfigV9 struct {
 
 	// Notification queue configuration.
 	Notify notifierV1 `json:"notify"`
-
-	// Read Write mutex.
-	rwMutex *sync.RWMutex
 }
 
 type loggerV7 struct {

cmd/config.go

@@ -127,23 +127,18 @@ func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 	//    of the object layer.
 	//  - Write quorum not met when upgrading configuration
 	//    version is needed.
-	retryTimerCh := newRetryTimerSimple(doneCh)
-	for {
-		select {
-		case _ = <-retryTimerCh:
-			err := initConfig(objAPI)
-			if err != nil {
-				if strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
-					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
-					logger.Info("Waiting for configuration to be initialized..")
-					continue
-				}
-				return err
+	for range newRetryTimerSimple(doneCh) {
+		if err := initConfig(objAPI); err != nil {
+			if strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+				strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+				logger.Info("Waiting for configuration to be initialized..")
+				continue
 			}
-
-			return nil
+			return err
 		}
+		break
 	}
+	return nil
 }
 
 // NewConfigSys - creates new config system object.

cmd/copy-part-range.go

@@ -17,21 +17,22 @@
 package cmd
 
 import (
+	"context"
 	"net/http"
 	"net/url"
 )
 
 // Writes S3 compatible copy part range error.
-func writeCopyPartErr(w http.ResponseWriter, err error, url *url.URL, browser bool) {
+func writeCopyPartErr(ctx context.Context, w http.ResponseWriter, err error, url *url.URL, browser bool) {
 	switch err {
 	case errInvalidRange:
-		writeErrorResponse(w, ErrInvalidCopyPartRange, url, browser)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopyPartRange), url, browser)
 		return
 	case errInvalidRangeSource:
-		writeErrorResponse(w, ErrInvalidCopyPartRangeSource, url, browser)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopyPartRangeSource), url, browser)
 		return
 	default:
-		writeErrorResponse(w, ErrInternalError, url, browser)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), url, browser)
 		return
 	}
 }

cmd/crypto/vault.go

@@ -23,7 +23,6 @@ import (
 	"time"
 
 	vault "github.com/hashicorp/vault/api"
-	"github.com/minio/minio/cmd/logger"
 )
 
 var (
@@ -63,8 +62,8 @@ type VaultConfig struct {
 type vaultService struct {
 	config        *VaultConfig
 	client        *vault.Client
+	secret        *vault.Secret
 	leaseDuration time.Duration
-	tokenRenewer  *vault.Renewer
 }
 
 var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements KMS
@@ -123,57 +122,48 @@ func NewVault(config VaultConfig) (KMS, error) {
 		client.SetNamespace(config.Namespace)
 	}
 	v := &vaultService{client: client, config: &config}
-
 	if err := v.authenticate(); err != nil {
 		return nil, err
 	}
+	v.renewToken()
 	return v, nil
 }
 
-// reauthenticate() tries to login in 1 minute
-// intervals until successful.
-func (v *vaultService) reauthenticate() {
-	retryDelay := 1 * time.Minute
+// renewToken starts a new go-routine which renews
+// the vault authentication token periodically and re-authenticates
+// if the token renewal fails
+func (v *vaultService) renewToken() {
+	retryDelay := v.leaseDuration / 2
 	go func() {
 		for {
-			if err := v.authenticate(); err != nil {
+			if v.secret == nil {
+				if err := v.authenticate(); err != nil {
+					time.Sleep(retryDelay)
+					continue
+				}
+			}
+			s, err := v.client.Auth().Token().RenewSelf(int(v.leaseDuration))
+			if err != nil || s == nil {
+				v.secret = nil
 				time.Sleep(retryDelay)
 				continue
 			}
-			return
+			if ok, err := s.TokenIsRenewable(); !ok || err != nil {
+				v.secret = nil
+				continue
+			}
+			ttl, err := s.TokenTTL()
+			if err != nil {
+				v.secret = nil
+				continue
+			}
+			v.secret = s
+			retryDelay = ttl / 2
+			time.Sleep(retryDelay)
 		}
 	}()
 }
 
-// renewer calls vault client's renewer that automatically
-// renews secret periodically
-func (v *vaultService) renewer(secret *vault.Secret) {
-	renewer, err := v.client.NewRenewer(&vault.RenewerInput{
-		Secret: secret,
-	})
-	if err != nil {
-		logger.FatalIf(err, "crypto: hashicorp vault token renewer could not be started")
-	}
-	v.tokenRenewer = renewer
-	go renewer.Renew()
-	defer renewer.Stop()
-
-	for {
-		select {
-		case err := <-renewer.DoneCh():
-			if err != nil {
-				v.reauthenticate()
-				renewer.Stop()
-				return
-			}
-
-			// Renewal is now over
-		case renewal := <-renewer.RenewCh():
-			v.leaseDuration = time.Duration(renewal.Secret.Auth.LeaseDuration)
-		}
-	}
-}
-
 // authenticate logs the app to vault, and starts the auto renewer
 // before secret expires
 func (v *vaultService) authenticate() (err error) {
@@ -181,18 +171,31 @@ func (v *vaultService) authenticate() (err error) {
 		"role_id":   v.config.Auth.AppRole.ID,
 		"secret_id": v.config.Auth.AppRole.Secret,
 	}
+	var tokenID string
+	var ttl time.Duration
 	var secret *vault.Secret
 	secret, err = v.client.Logical().Write("auth/approle/login", payload)
 	if err != nil {
 		return
 	}
-	if secret.Auth == nil {
+	if secret == nil {
 		err = ErrKMSAuthLogin
 		return
 	}
-	v.client.SetToken(secret.Auth.ClientToken)
-	v.leaseDuration = time.Duration(secret.Auth.LeaseDuration)
-	go v.renewer(secret)
+
+	tokenID, err = secret.TokenID()
+	if err != nil {
+		err = ErrKMSAuthLogin
+		return
+	}
+	ttl, err = secret.TokenTTL()
+	if err != nil {
+		err = ErrKMSAuthLogin
+		return
+	}
+	v.client.SetToken(tokenID)
+	v.secret = secret
+	v.leaseDuration = ttl
 	return
 }
 

cmd/disk-cache-fs.go

@@ -257,7 +257,7 @@ func (cfs *cacheFSObjects) IsOnline() bool {
 }
 
 // Caches the object to disk
-func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) error {
+func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) error {
 	if cfs.diskUsageHigh() {
 		select {
 		case cfs.purgeChan <- struct{}{}:
@@ -274,7 +274,7 @@ func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data
 			return pErr
 		}
 	}
-	_, err := cfs.PutObject(ctx, bucket, object, data, metadata, opts)
+	_, err := cfs.PutObject(ctx, bucket, object, data, opts)
 	// if err is due to disk being offline , mark cache drive as offline
 	if IsErr(err, baseErrs...) {
 		cfs.setOnline(false)
@@ -300,7 +300,7 @@ func (cfs *cacheFSObjects) Exists(ctx context.Context, bucket, object string) bo
 
 // Identical to fs PutObject operation except that it uses ETag in metadata
 // headers.
-func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
+func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
 	data := r.Reader
 	fs := cfs.FSObjects
 	// Lock the object.
@@ -312,7 +312,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 
 	// No metadata is set, allocate a new one.
 	meta := make(map[string]string)
-	for k, v := range metadata {
+	for k, v := range opts.UserDefined {
 		meta[k] = v
 	}
 
@@ -438,7 +438,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 // Implements S3 compatible initiate multipart API. Operation here is identical
 // to fs backend implementation - with the exception that cache FS uses the uploadID
 // generated on the backend
-func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, uploadID string, opts ObjectOptions) (string, error) {
+func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, uploadID string, opts ObjectOptions) (string, error) {
 	if cfs.diskUsageHigh() {
 		select {
 		case cfs.purgeChan <- struct{}{}:
@@ -472,7 +472,7 @@ func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, objec
 
 	// Initialize fs.json values.
 	fsMeta := newFSMetaV1()
-	fsMeta.Meta = meta
+	fsMeta.Meta = opts.UserDefined
 
 	fsMetaBytes, err := json.Marshal(fsMeta)
 	if err != nil {

cmd/disk-cache.go

@@ -60,13 +60,13 @@ type cacheObjects struct {
 	GetObjectNInfoFn          func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
 	GetObjectFn               func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
 	GetObjectInfoFn           func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	PutObjectFn               func(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	PutObjectFn               func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObjectFn            func(ctx context.Context, bucket, object string) error
 	ListObjectsFn             func(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
 	ListObjectsV2Fn           func(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error)
 	ListBucketsFn             func(ctx context.Context) (buckets []BucketInfo, err error)
 	GetBucketInfoFn           func(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error)
-	NewMultipartUploadFn      func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
+	NewMultipartUploadFn      func(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error)
 	PutObjectPartFn           func(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
 	AbortMultipartUploadFn    func(ctx context.Context, bucket, object, uploadID string) error
 	CompleteMultipartUploadFn func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
@@ -92,11 +92,11 @@ type CacheObjectLayer interface {
 	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
 	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
 	GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObject(ctx context.Context, bucket, object string) error
 
 	// Multipart operations.
-	NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
+	NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error)
 	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
 	AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error
 	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
@@ -246,8 +246,7 @@ func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 	}
 
 	go func() {
-		opts := ObjectOptions{}
-		putErr := dcache.Put(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), c.getMetadata(bkReader.ObjInfo), opts)
+		putErr := dcache.Put(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), ObjectOptions{UserDefined: c.getMetadata(bkReader.ObjInfo)})
 		// close the write end of the pipe, so the error gets
 		// propagated to getObjReader
 		pipeWriter.CloseWithError(putErr)
@@ -320,7 +319,9 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 		gerr := GetObjectFn(ctx, bucket, object, 0, objInfo.Size, io.MultiWriter(writer, pipeWriter), etag, opts)
 		pipeWriter.CloseWithError(gerr) // Close writer explicitly signaling we wrote all data.
 	}()
-	err = dcache.Put(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), c.getMetadata(objInfo), opts)
+
+	opts.UserDefined = c.getMetadata(objInfo)
+	err = dcache.Put(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), opts)
 	if err != nil {
 		return err
 	}
@@ -644,25 +645,25 @@ func (c cacheObjects) isCacheExclude(bucket, object string) bool {
 }
 
 // PutObject - caches the uploaded object for single Put operations
-func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	putObjectFn := c.PutObjectFn
 	data := r.Reader
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return putObjectFn(ctx, bucket, object, r, metadata, opts)
+		return putObjectFn(ctx, bucket, object, r, opts)
 	}
 	size := r.Size()
 
 	// fetch from backend if there is no space on cache drive
 	if !dcache.diskAvailable(size) {
-		return putObjectFn(ctx, bucket, object, r, metadata, opts)
+		return putObjectFn(ctx, bucket, object, r, opts)
 	}
 	// fetch from backend if cache exclude pattern or cache-control
 	// directive set to exclude
-	if c.isCacheExclude(bucket, object) || filterFromCache(metadata) {
+	if c.isCacheExclude(bucket, object) || filterFromCache(opts.UserDefined) {
 		dcache.Delete(ctx, bucket, object)
-		return putObjectFn(ctx, bucket, object, r, metadata, opts)
+		return putObjectFn(ctx, bucket, object, r, opts)
 	}
 	objInfo = ObjectInfo{}
 	// Initialize pipe to stream data to backend
@@ -680,7 +681,7 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *P
 	oinfoCh := make(chan ObjectInfo)
 	errCh := make(chan error)
 	go func() {
-		oinfo, perr := putObjectFn(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), metadata, opts)
+		oinfo, perr := putObjectFn(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), opts)
 		if perr != nil {
 			pipeWriter.CloseWithError(perr)
 			wPipe.CloseWithError(perr)
@@ -693,7 +694,7 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *P
 	}()
 
 	go func() {
-		if err = dcache.Put(ctx, bucket, object, NewPutObjReader(cHashReader, nil, nil), metadata, opts); err != nil {
+		if err = dcache.Put(ctx, bucket, object, NewPutObjReader(cHashReader, nil, nil), opts); err != nil {
 			wPipe.CloseWithError(err)
 			return
 		}
@@ -712,25 +713,25 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *P
 }
 
 // NewMultipartUpload - Starts a new multipart upload operation to backend and cache.
-func (c cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
+func (c cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error) {
 	newMultipartUploadFn := c.NewMultipartUploadFn
 
-	if c.isCacheExclude(bucket, object) || filterFromCache(metadata) {
-		return newMultipartUploadFn(ctx, bucket, object, metadata, opts)
+	if c.isCacheExclude(bucket, object) || filterFromCache(opts.UserDefined) {
+		return newMultipartUploadFn(ctx, bucket, object, opts)
 	}
 
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return newMultipartUploadFn(ctx, bucket, object, metadata, opts)
+		return newMultipartUploadFn(ctx, bucket, object, opts)
 	}
 
-	uploadID, err = newMultipartUploadFn(ctx, bucket, object, metadata, opts)
+	uploadID, err = newMultipartUploadFn(ctx, bucket, object, opts)
 	if err != nil {
 		return
 	}
 	// create new multipart upload in cache with same uploadID
-	dcache.NewMultipartUpload(ctx, bucket, object, metadata, uploadID, opts)
+	dcache.NewMultipartUpload(ctx, bucket, object, uploadID, opts)
 	return uploadID, err
 }
 
@@ -971,8 +972,8 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
 		GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
 			return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 		},
-		PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-			return newObjectLayerFn().PutObject(ctx, bucket, object, data, metadata, opts)
+		PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+			return newObjectLayerFn().PutObject(ctx, bucket, object, data, opts)
 		},
 		DeleteObjectFn: func(ctx context.Context, bucket, object string) error {
 			return newObjectLayerFn().DeleteObject(ctx, bucket, object)
@@ -989,8 +990,8 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
 		GetBucketInfoFn: func(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
 			return newObjectLayerFn().GetBucketInfo(ctx, bucket)
 		},
-		NewMultipartUploadFn: func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
-			return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, metadata, opts)
+		NewMultipartUploadFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error) {
+			return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, opts)
 		},
 		PutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
 			return newObjectLayerFn().PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)

cmd/disk-cache_test.go

@@ -192,14 +192,13 @@ func TestDiskCache(t *testing.T) {
 	objInfo.ContentType = contentType
 	objInfo.ETag = etag
 	objInfo.UserDefined = httpMeta
-	opts := ObjectOptions{}
-
+	var opts ObjectOptions
 	byteReader := bytes.NewReader([]byte(content))
 	hashReader, err := hash.NewReader(byteReader, int64(size), "", "", int64(size))
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
+	err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), ObjectOptions{UserDefined: httpMeta})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -275,12 +274,12 @@ func TestDiskCacheMaxUse(t *testing.T) {
 		t.Fatal(err)
 	}
 	if !cache.diskAvailable(int64(size)) {
-		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
+		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), ObjectOptions{UserDefined: httpMeta})
 		if err != errDiskFull {
 			t.Fatal("Cache max-use limit violated.")
 		}
 	} else {
-		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
+		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), ObjectOptions{UserDefined: httpMeta})
 		if err != nil {
 			t.Fatal(err)
 		}

cmd/dummy-handlers.go

@@ -0,0 +1,234 @@
+/*
+ * Minio Cloud Storage, (C) 2018, 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/policy"
+)
+
+// Data types used for returning dummy tagging XML.
+// These variables shouldn't be used elsewhere.
+// They are only defined to be used in this file alone.
+
+type tagging struct {
+	XMLName xml.Name `xml:"Tagging"`
+	TagSet  tagSet   `xml:"TagSet"`
+}
+
+type tagSet struct {
+	Tag []tagElem `xml:"Tag"`
+}
+
+type tagElem struct {
+	Key   string `xml:"Key"`
+	Value string `xml:"Value"`
+}
+
+// GetBucketWebsite  - GET bucket website, a dummy api
+func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// GetBucketVersioning - GET bucket versioning, a dummy api
+func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// GetBucketAccelerate  - GET bucket accelerate, a dummy api
+func (api objectAPIHandlers) GetBucketAccelerateHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// GetBucketRequestPaymentHandler - GET bucket requestPayment, a dummy api
+func (api objectAPIHandlers) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// GetBucketLoggingHandler - GET bucket logging, a dummy api
+func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// GetBucketLifecycleHandler - GET bucket lifecycle, a dummy api
+func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// GetBucketReplicationHandler - GET bucket replication, a dummy api
+func (api objectAPIHandlers) GetBucketReplicationHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// DeleteBucketTaggingHandler - DELETE bucket tagging, a dummy api
+func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// DeleteBucketWebsiteHandler - DELETE bucket website, a dummy api
+func (api objectAPIHandlers) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+type allowedMethod string
+
+// Define strings
+const (
+	GET    allowedMethod = http.MethodGet
+	PUT    allowedMethod = http.MethodPut
+	HEAD   allowedMethod = http.MethodHead
+	POST   allowedMethod = http.MethodPost
+	DELETE allowedMethod = http.MethodDelete
+)
+
+// GetBucketCorsHandler - GET bucket cors, a dummy api
+func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketCorsHandler")
+
+	type corsRule struct {
+		AllowedHeaders []string        `xml:"AllowedHeaders"`
+		AllowedMethods []allowedMethod `xml:"AllowedMethod"`
+		AllowedOrigins []string        `xml:"AllowedOrigin"`
+		ExposeHeaders  []string        `xml:"ExposeHeader"`
+		MaxAgeSeconds  int64           `xml:"MaxAgeSeconds"`
+	}
+
+	type corsConfiguration struct {
+		XMLName  xml.Name   `xml:"CORSConfiguration"`
+		CorsRule []corsRule `xml:"CORSRule"`
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Allow getBucketCors if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Validate if bucket exists, before proceeding further...
+	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	cors := &corsConfiguration{}
+	if err := xml.NewEncoder(w).Encode(cors); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// GetBucketTaggingHandler - GET bucket tagging, a dummy api
+func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketTagging")
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Allow getBucketTagging if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Validate if bucket exists, before proceeding further...
+	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	tags := &tagging{}
+	tags.TagSet.Tag = append(tags.TagSet.Tag, tagElem{})
+
+	if err := xml.NewEncoder(w).Encode(tags); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// GetObjectTaggingHandler - GET object tagging, a dummy api
+func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetObjectTagging")
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Allow getObjectTagging if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Validate if object exists, before proceeding further...
+	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	tags := &tagging{}
+	tags.TagSet.Tag = append(tags.TagSet.Tag, tagElem{})
+
+	if err := xml.NewEncoder(w).Encode(tags); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}

cmd/dummy-object-layer_test.go

@@ -1,167 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"io"
-	"net/http"
-
-	"github.com/minio/minio/pkg/madmin"
-	"github.com/minio/minio/pkg/policy"
-)
-
-type DummyObjectLayer struct{}
-
-func (api *DummyObjectLayer) Shutdown(context.Context) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) StorageInfo(context.Context) (si StorageInfo) {
-	return
-}
-
-func (api *DummyObjectLayer) MakeBucketWithLocation(ctx context.Context, bucket string, location string) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ListBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) DeleteBucket(ctx context.Context, bucket string) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lock LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) DeleteObject(ctx context.Context, bucket, object string) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (info PartInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) HealFormat(ctx context.Context, dryRun bool) (item madmin.HealResultItem, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) HealBucket(ctx context.Context, bucket string, dryRun bool) (items []madmin.HealResultItem, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) HealObject(ctx context.Context, bucket, object string, dryRun bool) (item madmin.HealResultItem, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (info ListObjectsInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) SetBucketPolicy(context.Context, string, *policy.Policy) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) GetBucketPolicy(context.Context, string) (bucketPolicy *policy.Policy, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) RefreshBucketPolicy(context.Context, string) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) DeleteBucketPolicy(context.Context, string) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) IsNotificationSupported() (b bool) {
-	return
-}
-func (api *DummyObjectLayer) IsListenBucketSupported() (b bool) {
-	return
-}
-
-func (api *DummyObjectLayer) IsEncryptionSupported() (b bool) {
-	return
-}
-
-func (api *DummyObjectLayer) IsCompressionSupported() (b bool) {
-	return
-}

cmd/encryption-v1.go

@@ -742,7 +742,7 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
 		return writer, encStartOffset, encLength, nil
 	}
 
-	seqNumber, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
+	_, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
 
 	var partStartIndex int
 	var partStartOffset = startOffset
@@ -1103,22 +1103,22 @@ func (o *ObjectInfo) EncryptedSize() int64 {
 // decryption succeeded.
 //
 // DecryptCopyObjectInfo also returns whether the object is encrypted or not.
-func DecryptCopyObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErrorCode, encrypted bool) {
+func DecryptCopyObjectInfo(info *ObjectInfo, headers http.Header) (errCode APIErrorCode, encrypted bool) {
 	// Directories are never encrypted.
 	if info.IsDir {
 		return ErrNone, false
 	}
-	if apiErr, encrypted = ErrNone, crypto.IsEncrypted(info.UserDefined); !encrypted && crypto.SSECopy.IsRequested(headers) {
-		apiErr = ErrInvalidEncryptionParameters
+	if errCode, encrypted = ErrNone, crypto.IsEncrypted(info.UserDefined); !encrypted && crypto.SSECopy.IsRequested(headers) {
+		errCode = ErrInvalidEncryptionParameters
 	} else if encrypted {
 		if (!crypto.SSECopy.IsRequested(headers) && crypto.SSEC.IsEncrypted(info.UserDefined)) ||
 			(crypto.SSECopy.IsRequested(headers) && crypto.S3.IsEncrypted(info.UserDefined)) {
-			apiErr = ErrSSEEncryptedObject
+			errCode = ErrSSEEncryptedObject
 			return
 		}
 		var err error
 		if info.Size, err = info.DecryptedSize(); err != nil {
-			apiErr = toAPIErrorCode(context.Background(), err)
+			errCode = toAPIErrorCode(context.Background(), err)
 		}
 	}
 	return
@@ -1171,8 +1171,8 @@ func deriveClientKey(clientKey [32]byte, bucket, object string) [32]byte {
 	return key
 }
 
-// extract encryption options for pass through to backend in the case of gateway
-func extractEncryptionOption(header http.Header, copySource bool, metadata map[string]string) (opts ObjectOptions, err error) {
+// set encryption options for pass through to backend in the case of gateway and UserDefined metadata
+func getDefaultOpts(header http.Header, copySource bool, metadata map[string]string) (opts ObjectOptions, err error) {
 	var clientKey [32]byte
 	var sse encrypt.ServerSide
 
@@ -1185,7 +1185,7 @@ func extractEncryptionOption(header http.Header, copySource bool, metadata map[s
 			if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
 				return
 			}
-			return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(sse)}, nil
+			return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(sse), UserDefined: metadata}, nil
 		}
 		return
 	}
@@ -1198,16 +1198,16 @@ func extractEncryptionOption(header http.Header, copySource bool, metadata map[s
 		if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
 			return
 		}
-		return ObjectOptions{ServerSideEncryption: sse}, nil
+		return ObjectOptions{ServerSideEncryption: sse, UserDefined: metadata}, nil
 	}
 	if crypto.S3.IsRequested(header) || (metadata != nil && crypto.S3.IsEncrypted(metadata)) {
-		return ObjectOptions{ServerSideEncryption: encrypt.NewSSE()}, nil
+		return ObjectOptions{ServerSideEncryption: encrypt.NewSSE(), UserDefined: metadata}, nil
 	}
-	return opts, nil
+	return ObjectOptions{UserDefined: metadata}, nil
 }
 
 // get ObjectOptions for GET calls from encryption headers
-func getEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
+func getOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
 	var (
 		encryption encrypt.ServerSide
 		opts       ObjectOptions
@@ -1223,30 +1223,32 @@ func getEncryptionOpts(ctx context.Context, r *http.Request, bucket, object stri
 		return ObjectOptions{ServerSideEncryption: encryption}, nil
 	}
 	// default case of passing encryption headers to backend
-	return extractEncryptionOption(r.Header, false, nil)
+	return getDefaultOpts(r.Header, false, nil)
 }
 
-// get ObjectOptions for PUT calls from encryption headers
-func putEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
+// get ObjectOptions for PUT calls from encryption headers and metadata
+func putOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
 	// In the case of multipart custom format, the metadata needs to be checked in addition to header to see if it
 	// is SSE-S3 encrypted, primarily because S3 protocol does not require SSE-S3 headers in PutObjectPart calls
 	if GlobalGatewaySSE.SSES3() && (crypto.S3.IsRequested(r.Header) || crypto.S3.IsEncrypted(metadata)) {
-		return ObjectOptions{ServerSideEncryption: encrypt.NewSSE()}, nil
+		return ObjectOptions{ServerSideEncryption: encrypt.NewSSE(), UserDefined: metadata}, nil
 	}
 	if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) {
-		return getEncryptionOpts(ctx, r, bucket, object)
+		opts, err = getOpts(ctx, r, bucket, object)
+		opts.UserDefined = metadata
+		return
 	}
-	// default case of passing encryption headers to backend
-	return extractEncryptionOption(r.Header, false, metadata)
+	// default case of passing encryption headers and UserDefined metadata to backend
+	return getDefaultOpts(r.Header, false, metadata)
 }
 
-// get ObjectOptions for Copy calls for encryption headers provided on the target side
-func copyDstEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
-	return putEncryptionOpts(ctx, r, bucket, object, metadata)
+// get ObjectOptions for Copy calls with encryption headers provided on the target side and source side metadata
+func copyDstOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
+	return putOpts(ctx, r, bucket, object, metadata)
 }
 
-// get ObjectOptions for Copy calls for encryption headers provided on the source side
-func copySrcEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
+// get ObjectOptions for Copy calls with encryption headers provided on the source side
+func copySrcOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
 	var (
 		ssec encrypt.ServerSide
 		opts ObjectOptions
@@ -1266,5 +1268,5 @@ func copySrcEncryptionOpts(ctx context.Context, r *http.Request, bucket, object
 	}
 
 	// default case of passing encryption headers to backend
-	return extractEncryptionOption(r.Header, true, nil)
+	return getDefaultOpts(r.Header, true, nil)
 }

cmd/encryption-v1_test.go

@@ -678,7 +678,7 @@ func TestGetDecryptedRange(t *testing.T) {
 	}
 }
 
-var extractEncryptionOptionTests = []struct {
+var getDefaultOptsTests = []struct {
 	headers        http.Header
 	copySource     bool
 	metadata       map[string]string
@@ -741,9 +741,9 @@ var extractEncryptionOptionTests = []struct {
 		err:            nil}, // 7
 }
 
-func TestExtractEncryptionOptions(t *testing.T) {
-	for i, test := range extractEncryptionOptionTests {
-		opts, err := extractEncryptionOption(test.headers, test.copySource, test.metadata)
+func TestGetDefaultOpts(t *testing.T) {
+	for i, test := range getDefaultOptsTests {
+		opts, err := getDefaultOpts(test.headers, test.copySource, test.metadata)
 		if test.err != err {
 			t.Errorf("Case %d: expected err: %v , actual err: %v", i, test.err, err)
 		}

cmd/endpoint.go

@@ -94,7 +94,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 		// - Scheme field must contain "http" or "https"
 		// - All field should be empty except Host and Path.
 		if !((u.Scheme == "http" || u.Scheme == "https") &&
-			u.User == nil && u.Opaque == "" && u.ForceQuery == false && u.RawQuery == "" && u.Fragment == "") {
+			u.User == nil && u.Opaque == "" && !u.ForceQuery && u.RawQuery == "" && u.Fragment == "") {
 			return ep, fmt.Errorf("invalid URL endpoint format")
 		}
 

cmd/erasure-decode_test.go

@@ -28,7 +28,7 @@ import (
 	humanize "github.com/dustin/go-humanize"
 )
 
-func (d badDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
+func (a badDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
 	return 0, errFaultyDisk
 }
 

cmd/format-fs.go

@@ -341,35 +341,35 @@ func formatFSFixDeploymentID(fsFormatPath string) error {
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 
-	retryTimerCh := newRetryTimerSimple(doneCh)
-	for {
-		select {
-		case <-retryTimerCh:
+	var wlk *lock.LockedFile
+	for range newRetryTimerSimple(doneCh) {
+		wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
+		if err == lock.ErrAlreadyLocked {
+			// Lock already present, sleep and attempt again
+			logger.Info("Another minio process(es) might be holding a lock to the file %s. Please kill that minio process(es) (elapsed %s)\n", fsFormatPath, getElapsedTime())
+			continue
+		}
+		if err != nil {
+			break
+		}
 
-			wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
-			if err == lock.ErrAlreadyLocked {
-				// Lock already present, sleep and attempt again
+		if err = jsonLoad(wlk, format); err != nil {
+			break
+		}
 
-				logger.Info("Another minio process(es) might be holding a lock to the file %s. Please kill that minio process(es) (elapsed %s)\n", fsFormatPath, getElapsedTime())
-				continue
-			}
-			if err != nil {
-				return err
-			}
-			defer wlk.Close()
+		// Check if format needs to be updated
+		if format.ID != "" {
+			err = nil
+			break
+		}
 
-			err = jsonLoad(wlk, format)
-			if err != nil {
-				return err
-			}
-
-			// Check if it needs to be updated
-			if format.ID != "" {
-				return nil
-			}
-			format.ID = mustGetUUID()
-			return jsonSave(wlk, format)
+		format.ID = mustGetUUID()
+		if err = jsonSave(wlk, format); err != nil {
+			break
 		}
 	}
-
+	if wlk != nil {
+		wlk.Close()
+	}
+	return err
 }

cmd/format-xl_test.go

@@ -522,7 +522,7 @@ func TestGetXLID(t *testing.T) {
 	}
 
 	formats[2].ID = "bad-id"
-	if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != errCorruptedFormat {
+	if _, err = formatXLGetDeploymentID(quorumFormat, formats); err != errCorruptedFormat {
 		t.Fatal("Unexpected Success")
 	}
 }

cmd/fs-v1-helpers.go

@@ -294,7 +294,7 @@ func fsOpenFile(ctx context.Context, readPath string, offset int64) (io.ReadClos
 
 	// Seek to the requested offset.
 	if offset > 0 {
-		_, err = fr.Seek(offset, os.SEEK_SET)
+		_, err = fr.Seek(offset, io.SeekStart)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			return nil, 0, err

cmd/fs-v1-metadata_test.go

@@ -54,7 +54,7 @@ func TestReadFSMetadata(t *testing.T) {
 	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
-	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
+	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 
@@ -89,7 +89,7 @@ func TestWriteFSMetadata(t *testing.T) {
 	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
-	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
+	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 

cmd/fs-v1-multipart.go

@@ -210,7 +210,7 @@ func (fs *FSObjects) ListMultipartUploads(ctx context.Context, bucket, object, k
 // subsequent request each UUID is unique.
 //
 // Implements S3 compatible initiate multipart API.
-func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, opts ObjectOptions) (string, error) {
+func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (string, error) {
 	if err := checkNewMultipartArgs(ctx, bucket, object, fs); err != nil {
 		return "", toObjectErr(err, bucket)
 	}
@@ -230,7 +230,7 @@ func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object stri
 
 	// Initialize fs.json values.
 	fsMeta := newFSMetaV1()
-	fsMeta.Meta = meta
+	fsMeta.Meta = opts.UserDefined
 
 	fsMetaBytes, err := json.Marshal(fsMeta)
 	if err != nil {

cmd/fs-v1-multipart_test.go

@@ -42,7 +42,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 	objectName := "object"
 
 	obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil, ObjectOptions{})
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
@@ -81,7 +81,7 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) {
 
 	// Test with disk removed.
 	os.RemoveAll(disk)
-	if _, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{}); err != nil {
+	if _, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}}); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}
@@ -105,7 +105,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -136,7 +136,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -168,7 +168,7 @@ func TestCompleteMultipartUpload(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -202,7 +202,7 @@ func TestAbortMultipartUpload(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -234,7 +234,7 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	_, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
+	_, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}

cmd/fs-v1.go

@@ -449,7 +449,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 		// Return the new object info.
 		return fsMeta.ToObjectInfo(srcBucket, srcObject, fi), nil
 	}
-	objInfo, err := fs.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, srcInfo.UserDefined, dstOpts)
+	objInfo, err := fs.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, ObjectOptions{ServerSideEncryption: dstOpts.ServerSideEncryption, UserDefined: srcInfo.UserDefined})
 	if err != nil {
 		return oi, toObjectErr(err, dstBucket, dstObject)
 	}
@@ -532,8 +532,7 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		nsUnlocker()
 		return nil, toObjectErr(err, bucket, object)
 	}
-	var reader io.Reader
-	reader = io.LimitReader(readCloser, length)
+	reader := io.LimitReader(readCloser, length)
 	closeFn := func() {
 		readCloser.Close()
 	}
@@ -804,7 +803,7 @@ func (fs *FSObjects) parentDirIsObject(ctx context.Context, bucket, parent strin
 // until EOF, writes data directly to configured filesystem path.
 // Additionally writes `fs.json` which carries the necessary metadata
 // for future object operations.
-func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
+func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
 	if err := checkPutObjectArgs(ctx, bucket, object, fs, r.Size()); err != nil {
 		return ObjectInfo{}, err
 	}
@@ -815,16 +814,16 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
 		return objInfo, err
 	}
 	defer objectLock.Unlock()
-	return fs.putObject(ctx, bucket, object, r, metadata, opts)
+	return fs.putObject(ctx, bucket, object, r, opts)
 }
 
 // putObject - wrapper for PutObject
-func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
+func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
 	data := r.Reader
 
 	// No metadata is set, allocate a new one.
 	meta := make(map[string]string)
-	for k, v := range metadata {
+	for k, v := range opts.UserDefined {
 		meta[k] = v
 	}
 	var err error
@@ -1245,17 +1244,17 @@ func (fs *FSObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealRe
 }
 
 // HealObject - no-op for fs. Valid only for XL.
-func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, dryRun bool) (
+func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool) (
 	res madmin.HealResultItem, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return res, NotImplemented{}
 }
 
 // HealBucket - no-op for fs, Valid only for XL.
-func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun bool) ([]madmin.HealResultItem,
+func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (madmin.HealResultItem,
 	error) {
 	logger.LogIf(ctx, NotImplemented{})
-	return nil, NotImplemented{}
+	return madmin.HealResultItem{}, NotImplemented{}
 }
 
 // ListObjectsHeal - list all objects to be healed. Valid only for XL

cmd/fs-v1_test.go

@@ -40,7 +40,7 @@ func TestFSParentDirIsObject(t *testing.T) {
 	}
 	objectContent := "12345"
 	objInfo, err := obj.PutObject(context.Background(), bucketName, objectName,
-		mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
+		mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -124,7 +124,7 @@ func TestFSShutdown(t *testing.T) {
 
 		objectContent := "12345"
 		obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-		obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
+		obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), ObjectOptions{})
 		return fs, disk
 	}
 
@@ -203,7 +203,7 @@ func TestFSPutObject(t *testing.T) {
 	}
 
 	// With a regular object.
-	_, err := obj.PutObject(context.Background(), bucketName+"non-existent", objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
+	_, err := obj.PutObject(context.Background(), bucketName+"non-existent", objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
@@ -212,7 +212,7 @@ func TestFSPutObject(t *testing.T) {
 	}
 
 	// With a directory object.
-	_, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+"/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+"/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
@@ -220,11 +220,11 @@ func TestFSPutObject(t *testing.T) {
 		t.Fatalf("Expected error type BucketNotFound, got %#v", err)
 	}
 
-	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backend corruption occurred")
 	}
@@ -239,7 +239,7 @@ func TestFSPutObject(t *testing.T) {
 		}
 	}
 
-	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backned corruption occurred")
 	}
@@ -267,7 +267,7 @@ func TestFSDeleteObject(t *testing.T) {
 	objectName := "object"
 
 	obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-	obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
+	obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 
 	// Test with invalid bucket name
 	if err := fs.DeleteObject(context.Background(), "fo", objectName); !isSameType(err, BucketNameInvalid{}) {
@@ -390,7 +390,7 @@ func TestFSHealObject(t *testing.T) {
 	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
-	_, err := obj.HealObject(context.Background(), "bucket", "object", false)
+	_, err := obj.HealObject(context.Background(), "bucket", "object", false, false)
 	if err == nil || !isSameType(err, NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}

cmd/gateway-main.go

@@ -203,6 +203,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		// Override any values from ENVs.
 		srvCfg.loadFromEnvs()
 
+		// Load values to cached global values.
+		srvCfg.loadToCachedConfigs()
+
 		// hold the mutex lock before a new config is assigned.
 		globalServerConfigMu.Lock()
 		globalServerConfig = srvCfg

cmd/gateway-unsupported.go

@@ -33,7 +33,7 @@ func (a GatewayUnsupported) ListMultipartUploads(ctx context.Context, bucket str
 }
 
 // NewMultipartUpload upload object in multiple parts
-func (a GatewayUnsupported) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
+func (a GatewayUnsupported) NewMultipartUpload(ctx context.Context, bucket string, object string, opts ObjectOptions) (uploadID string, err error) {
 	return "", NotImplemented{}
 }
 
@@ -92,8 +92,8 @@ func (a GatewayUnsupported) HealFormat(ctx context.Context, dryRun bool) (madmin
 }
 
 // HealBucket - Not implemented stub
-func (a GatewayUnsupported) HealBucket(ctx context.Context, bucket string, dryRun bool) ([]madmin.HealResultItem, error) {
-	return nil, NotImplemented{}
+func (a GatewayUnsupported) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (madmin.HealResultItem, error) {
+	return madmin.HealResultItem{}, NotImplemented{}
 }
 
 // ListBucketsHeal - Not implemented stub
@@ -102,7 +102,7 @@ func (a GatewayUnsupported) ListBucketsHeal(ctx context.Context) (buckets []Buck
 }
 
 // HealObject - Not implemented stub
-func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun bool) (h madmin.HealResultItem, e error) {
+func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool) (h madmin.HealResultItem, e error) {
 	return h, NotImplemented{}
 }
 

cmd/gateway/azure/gateway-azure.go

@@ -26,6 +26,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"path"
 	"sort"
 	"strconv"
 	"strings"
@@ -52,6 +53,8 @@ const (
 	metadataObjectNameTemplate = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x/azure.json"
 	azureBackend               = "azure"
 	azureMarkerPrefix          = "{minio}"
+	metadataPartNamePrefix     = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x"
+	maxPartsCount              = 10000
 )
 
 func init() {
@@ -261,6 +264,26 @@ func s3MetaToAzureProperties(ctx context.Context, s3Metadata map[string]string)
 	return blobMeta, props, nil
 }
 
+const (
+	partMetaVersionV1 = "1"
+)
+
+// partMetadataV1 struct holds the part specific metadata for
+// multipart operations.
+type partMetadataV1 struct {
+	Version  string   `json:"version"`
+	Size     int64    `json:"Size"`
+	BlockIDs []string `json:"blockIDs"`
+	ETag     string   `json:"etag"`
+}
+
+// Returns the initialized part metadata struct
+func newPartMetaV1(uploadID string, partID int) (partMeta *partMetadataV1) {
+	p := &partMetadataV1{}
+	p.Version = partMetaVersionV1
+	return p
+}
+
 // azurePropertiesToS3Meta converts Azure metadata/properties to S3
 // metadata. It is the reverse of s3MetaToAzureProperties. Azure's
 // `.GetMetadata()` lower-cases all header keys, so this is taken into
@@ -405,37 +428,13 @@ func checkAzureUploadID(ctx context.Context, uploadID string) (err error) {
 	return nil
 }
 
-// Encode partID, subPartNumber, uploadID and md5Hex to blockID.
-func azureGetBlockID(partID, subPartNumber int, uploadID, md5Hex string) string {
-	return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%05d.%02d.%s.%s", partID, subPartNumber, uploadID, md5Hex)))
-}
-
-// Parse blockID into partID, subPartNumber and md5Hex.
-func azureParseBlockID(blockID string) (partID, subPartNumber int, uploadID, md5Hex string, err error) {
-	var blockIDBytes []byte
-	if blockIDBytes, err = base64.StdEncoding.DecodeString(blockID); err != nil {
+// parses partID from part metadata file name
+func parseAzurePart(metaPartFileName, prefix string) (partID int, err error) {
+	partStr := strings.TrimPrefix(metaPartFileName, prefix+"/")
+	if partID, err = strconv.Atoi(partStr); err != nil || partID <= 0 {
+		err = fmt.Errorf("invalid part number in block id '%s'", string(partID))
 		return
 	}
-
-	tokens := strings.Split(string(blockIDBytes), ".")
-	if len(tokens) != 4 {
-		err = fmt.Errorf("invalid block id '%s'", string(blockIDBytes))
-		return
-	}
-
-	if partID, err = strconv.Atoi(tokens[0]); err != nil || partID <= 0 {
-		err = fmt.Errorf("invalid part number in block id '%s'", string(blockIDBytes))
-		return
-	}
-
-	if subPartNumber, err = strconv.Atoi(tokens[1]); err != nil || subPartNumber <= 0 {
-		err = fmt.Errorf("invalid sub-part number in block id '%s'", string(blockIDBytes))
-		return
-	}
-
-	uploadID = tokens[2]
-	md5Hex = tokens[3]
-
 	return
 }
 
@@ -736,25 +735,21 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 
 // PutObject - Create a new blob with the incoming data,
 // uses Azure equivalent CreateBlockBlobFromReader.
-func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	data := r.Reader
 	if data.Size() < azureBlockSize/10 {
 		blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
-		blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, metadata)
+		blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, opts.UserDefined)
+		if err != nil {
+			return objInfo, azureToObjectError(err, bucket, object)
+		}
 		if err = blob.CreateBlockBlobFromReader(data, nil); err != nil {
 			return objInfo, azureToObjectError(err, bucket, object)
 		}
 		return a.GetObjectInfo(ctx, bucket, object, opts)
 	}
 
-	uuid, err := getAzureUploadID()
-	if err != nil {
-		return objInfo, err
-	}
-	etag := data.MD5HexString()
-	if etag == "" {
-		etag = minio.GenETag()
-	}
+	blockIDs := make(map[string]string)
 
 	blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
 	subPartSize, subPartNumber := int64(azureBlockSize), 1
@@ -768,7 +763,8 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 			subPartSize = remainingSize
 		}
 
-		id := azureGetBlockID(1, subPartNumber, uuid, etag)
+		id := base64.StdEncoding.EncodeToString([]byte(minio.MustGetUUID()))
+		blockIDs[id] = ""
 		if err = blob.PutBlockWithLength(id, uint64(subPartSize), io.LimitReader(data, subPartSize), nil); err != nil {
 			return objInfo, azureToObjectError(err, bucket, object)
 		}
@@ -780,17 +776,9 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
-
-	getBlocks := func(partNumber int, etag string) (blocks []storage.Block, size int64, aerr error) {
+	getBlocks := func(blocksMap map[string]string) (blocks []storage.Block, size int64, aerr error) {
 		for _, part := range resp.UncommittedBlocks {
-			var partID int
-			var readUploadID string
-			var md5Hex string
-			if partID, _, readUploadID, md5Hex, aerr = azureParseBlockID(part.Name); aerr != nil {
-				return nil, 0, aerr
-			}
-
-			if partNumber == partID && uuid == readUploadID && etag == md5Hex {
+			if _, ok := blocksMap[part.Name]; ok {
 				blocks = append(blocks, storage.Block{
 					ID:     part.Name,
 					Status: storage.BlockStatusUncommitted,
@@ -808,7 +796,7 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 	}
 
 	var blocks []storage.Block
-	blocks, _, err = getBlocks(1, etag)
+	blocks, _, err = getBlocks(blockIDs)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return objInfo, err
@@ -818,13 +806,13 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
 
-	if len(metadata) == 0 {
-		metadata = map[string]string{}
+	if len(opts.UserDefined) == 0 {
+		opts.UserDefined = map[string]string{}
 	}
 
 	// Save md5sum for future processing on the object.
-	metadata["x-amz-meta-md5sum"] = hex.EncodeToString(data.MD5Current())
-	objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(ctx, metadata)
+	opts.UserDefined["x-amz-meta-md5sum"] = hex.EncodeToString(data.MD5Current())
+	objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(ctx, opts.UserDefined)
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
@@ -896,6 +884,17 @@ func getAzureMetadataObjectName(objectName, uploadID string) string {
 	return fmt.Sprintf(metadataObjectNameTemplate, uploadID, sha256.Sum256([]byte(objectName)))
 }
 
+// gets the name of part metadata file for multipart upload operations
+func getAzureMetadataPartName(objectName, uploadID string, partID int) string {
+	partMetaPrefix := getAzureMetadataPartPrefix(uploadID, objectName)
+	return path.Join(partMetaPrefix, fmt.Sprintf("%d", partID))
+}
+
+// gets the prefix of part metadata file
+func getAzureMetadataPartPrefix(uploadID, objectName string) string {
+	return fmt.Sprintf(metadataPartNamePrefix, uploadID, sha256.Sum256([]byte(objectName)))
+}
+
 func (a *azureObjects) checkUploadIDExists(ctx context.Context, bucketName, objectName, uploadID string) (err error) {
 	blob := a.client.GetContainerReference(bucketName).GetBlobReference(
 		getAzureMetadataObjectName(objectName, uploadID))
@@ -914,7 +913,7 @@ func (a *azureObjects) checkUploadIDExists(ctx context.Context, bucketName, obje
 }
 
 // NewMultipartUpload - Use Azure equivalent CreateBlockBlob.
-func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts minio.ObjectOptions) (uploadID string, err error) {
+func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (uploadID string, err error) {
 	uploadID, err = getAzureUploadID()
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -923,7 +922,7 @@ func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object st
 	metadataObject := getAzureMetadataObjectName(object, uploadID)
 
 	var jsonData []byte
-	if jsonData, err = json.Marshal(azureMultipartMetadata{Name: object, Metadata: metadata}); err != nil {
+	if jsonData, err = json.Marshal(azureMultipartMetadata{Name: object, Metadata: opts.UserDefined}); err != nil {
 		logger.LogIf(ctx, err)
 		return "", err
 	}
@@ -948,11 +947,7 @@ func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, upload
 		return info, err
 	}
 
-	etag := data.MD5HexString()
-	if etag == "" {
-		etag = minio.GenETag()
-	}
-
+	partMetaV1 := newPartMetaV1(uploadID, partID)
 	subPartSize, subPartNumber := int64(azureBlockSize), 1
 	for remainingSize := data.Size(); remainingSize >= 0; remainingSize -= subPartSize {
 		// Allow to create zero sized part.
@@ -964,7 +959,9 @@ func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, upload
 			subPartSize = remainingSize
 		}
 
-		id := azureGetBlockID(partID, subPartNumber, uploadID, etag)
+		id := base64.StdEncoding.EncodeToString([]byte(minio.MustGetUUID()))
+		partMetaV1.BlockIDs = append(partMetaV1.BlockIDs, id)
+
 		blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
 		err = blob.PutBlockWithLength(id, uint64(subPartSize), io.LimitReader(data, subPartSize), nil)
 		if err != nil {
@@ -973,8 +970,26 @@ func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, upload
 		subPartNumber++
 	}
 
+	partMetaV1.ETag = r.MD5CurrentHexString()
+	partMetaV1.Size = data.Size()
+
+	// maintain per part md5sum in a temporary part metadata file until upload
+	// is finalized.
+	metadataObject := getAzureMetadataPartName(object, uploadID, partID)
+	var jsonData []byte
+	if jsonData, err = json.Marshal(partMetaV1); err != nil {
+		logger.LogIf(ctx, err)
+		return info, err
+	}
+
+	blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
+	err = blob.CreateBlockBlobFromReader(bytes.NewBuffer(jsonData), nil)
+	if err != nil {
+		return info, azureToObjectError(err, bucket, metadataObject)
+	}
+
 	info.PartNumber = partID
-	info.ETag = etag
+	info.ETag = partMetaV1.ETag
 	info.LastModified = minio.UTCNow()
 	info.Size = data.Size()
 	return info, nil
@@ -991,48 +1006,57 @@ func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uplo
 	result.UploadID = uploadID
 	result.MaxParts = maxParts
 
-	objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object)
-	resp, err := objBlob.GetBlockList(storage.BlockListTypeUncommitted, nil)
-	azureErr, ok := err.(storage.AzureStorageServiceError)
-	if ok && azureErr.StatusCode == http.StatusNotFound {
-		// If no parts are uploaded yet then we return empty list.
-		return result, nil
-	}
-	if err != nil {
-		return result, azureToObjectError(err, bucket, object)
-	}
-	// Build a sorted list of parts and return the requested entries.
-	partsMap := make(map[int]minio.PartInfo)
-	for _, block := range resp.UncommittedBlocks {
-		var partNumber int
-		var parsedUploadID string
-		var md5Hex string
-		if partNumber, _, parsedUploadID, md5Hex, err = azureParseBlockID(block.Name); err != nil {
-			return result, azureToObjectError(fmt.Errorf("Unexpected error"), bucket, object)
-		}
-		if parsedUploadID != uploadID {
-			continue
-		}
-		part, ok := partsMap[partNumber]
-		if !ok {
-			partsMap[partNumber] = minio.PartInfo{
-				PartNumber: partNumber,
-				Size:       block.Size,
-				ETag:       md5Hex,
-			}
-			continue
-		}
-		if part.ETag != md5Hex {
-			// If two parts of same partNumber were uploaded with different contents
-			// return error as we won't be able to decide which the latest part is.
-			return result, azureToObjectError(fmt.Errorf("Unexpected error"), bucket, object)
-		}
-		part.Size += block.Size
-		partsMap[partNumber] = part
-	}
 	var parts []minio.PartInfo
-	for _, part := range partsMap {
-		parts = append(parts, part)
+	var marker, delimiter string
+	maxKeys := maxPartsCount
+	if partNumberMarker == 0 {
+		maxKeys = maxParts
+	}
+	prefix := getAzureMetadataPartPrefix(uploadID, object)
+	container := a.client.GetContainerReference(bucket)
+	resp, err := container.ListBlobs(storage.ListBlobsParameters{
+		Prefix:     prefix,
+		Marker:     marker,
+		Delimiter:  delimiter,
+		MaxResults: uint(maxKeys),
+	})
+	if err != nil {
+		return result, azureToObjectError(err, bucket, prefix)
+	}
+
+	for _, blob := range resp.Blobs {
+		if delimiter == "" && !strings.HasPrefix(blob.Name, minio.GatewayMinioSysTmp) {
+			// We filter out non minio.GatewayMinioSysTmp entries in the recursive listing.
+			continue
+		}
+		// filter temporary metadata file for blob
+		if strings.HasSuffix(blob.Name, "azure.json") {
+			continue
+		}
+		if !isAzureMarker(marker) && blob.Name <= marker {
+			// If the application used ListObjectsV1 style marker then we
+			// skip all the entries till we reach the marker.
+			continue
+		}
+		partNumber, err := parseAzurePart(blob.Name, prefix)
+		if err != nil {
+			return result, azureToObjectError(fmt.Errorf("Unexpected error"), bucket, object)
+		}
+		var metadata partMetadataV1
+		var metadataReader io.Reader
+		blob := a.client.GetContainerReference(bucket).GetBlobReference(blob.Name)
+		if metadataReader, err = blob.Get(nil); err != nil {
+			return result, azureToObjectError(fmt.Errorf("Unexpected error"), bucket, object)
+		}
+		if err = json.NewDecoder(metadataReader).Decode(&metadata); err != nil {
+			logger.LogIf(ctx, err)
+			return result, azureToObjectError(err, bucket, object)
+		}
+		parts = append(parts, minio.PartInfo{
+			PartNumber: partNumber,
+			Size:       metadata.Size,
+			ETag:       metadata.ETag,
+		})
 	}
 	sort.Slice(parts, func(i int, j int) bool {
 		return parts[i].PartNumber < parts[j].PartNumber
@@ -1071,6 +1095,22 @@ func (a *azureObjects) AbortMultipartUpload(ctx context.Context, bucket, object,
 	if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return err
 	}
+	var partNumberMarker int
+	for {
+		lpi, err := a.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxPartsCount, minio.ObjectOptions{})
+		if err != nil {
+			break
+		}
+		for _, part := range lpi.Parts {
+			pblob := a.client.GetContainerReference(bucket).GetBlobReference(
+				getAzureMetadataPartName(object, uploadID, part.PartNumber))
+			pblob.Delete(nil)
+		}
+		partNumberMarker = lpi.NextPartNumberMarker
+		if !lpi.IsTruncated {
+			break
+		}
+	}
 
 	blob := a.client.GetContainerReference(bucket).GetBlobReference(
 		getAzureMetadataObjectName(object, uploadID))
@@ -1100,66 +1140,33 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 		return objInfo, azureToObjectError(err, bucket, metadataObject)
 	}
 
-	defer func() {
-		blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
-		derr := blob.Delete(nil)
-		logger.GetReqInfo(ctx).AppendTags("uploadID", uploadID)
-		logger.LogIf(ctx, derr)
-	}()
-
 	objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object)
-	resp, err := objBlob.GetBlockList(storage.BlockListTypeUncommitted, nil)
-	if err != nil {
-		return objInfo, azureToObjectError(err, bucket, object)
-	}
-
-	getBlocks := func(partNumber int, etag string) (blocks []storage.Block, size int64, err error) {
-		for _, part := range resp.UncommittedBlocks {
-			var partID int
-			var readUploadID string
-			var md5Hex string
-			if partID, _, readUploadID, md5Hex, err = azureParseBlockID(part.Name); err != nil {
-				return nil, 0, err
-			}
-
-			if partNumber == partID && uploadID == readUploadID && etag == md5Hex {
-				blocks = append(blocks, storage.Block{
-					ID:     part.Name,
-					Status: storage.BlockStatusUncommitted,
-				})
-
-				size += part.Size
-			}
-		}
-
-		if len(blocks) == 0 {
-			return nil, 0, minio.InvalidPart{}
-		}
-
-		return blocks, size, nil
-	}
 
 	var allBlocks []storage.Block
-	partSizes := make([]int64, len(uploadedParts))
 	for i, part := range uploadedParts {
-		var blocks []storage.Block
-		var size int64
-		blocks, size, err = getBlocks(part.PartNumber, part.ETag)
-		if err != nil {
-			logger.LogIf(ctx, err)
-			return objInfo, err
+		var partMetadataReader io.Reader
+		var partMetadata partMetadataV1
+		partMetadataObject := getAzureMetadataPartName(object, uploadID, part.PartNumber)
+		pblob := a.client.GetContainerReference(bucket).GetBlobReference(partMetadataObject)
+		if partMetadataReader, err = pblob.Get(nil); err != nil {
+			return objInfo, azureToObjectError(err, bucket, partMetadataObject)
 		}
 
-		allBlocks = append(allBlocks, blocks...)
-		partSizes[i] = size
-	}
+		if err = json.NewDecoder(partMetadataReader).Decode(&partMetadata); err != nil {
+			logger.LogIf(ctx, err)
+			return objInfo, azureToObjectError(err, bucket, partMetadataObject)
+		}
 
-	// Error out if parts except last part sizing < 5MiB.
-	for i, size := range partSizes[:len(partSizes)-1] {
-		if size < azureS3MinPartSize {
+		if partMetadata.ETag != part.ETag {
+			return objInfo, minio.InvalidPart{}
+		}
+		for _, blockID := range partMetadata.BlockIDs {
+			allBlocks = append(allBlocks, storage.Block{ID: blockID, Status: storage.BlockStatusUncommitted})
+		}
+		if i < (len(uploadedParts)-1) && partMetadata.Size < azureS3MinPartSize {
 			return objInfo, minio.PartTooSmall{
 				PartNumber: uploadedParts[i].PartNumber,
-				PartSize:   size,
+				PartSize:   partMetadata.Size,
 				PartETag:   uploadedParts[i].ETag,
 			}
 		}
@@ -1183,6 +1190,28 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 			return objInfo, azureToObjectError(err, bucket, object)
 		}
 	}
+	var partNumberMarker int
+	for {
+		lpi, err := a.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxPartsCount, minio.ObjectOptions{})
+		if err != nil {
+			break
+		}
+		for _, part := range lpi.Parts {
+			pblob := a.client.GetContainerReference(bucket).GetBlobReference(
+				getAzureMetadataPartName(object, uploadID, part.PartNumber))
+			pblob.Delete(nil)
+		}
+		partNumberMarker = lpi.NextPartNumberMarker
+		if !lpi.IsTruncated {
+			break
+		}
+	}
+
+	blob = a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
+	derr := blob.Delete(nil)
+	logger.GetReqInfo(ctx).AppendTags("uploadID", uploadID)
+	logger.LogIf(ctx, derr)
+
 	return a.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
 }
 

cmd/gateway/azure/gateway-azure_test.go

@@ -188,72 +188,6 @@ func TestAzureToObjectError(t *testing.T) {
 	}
 }
 
-// Test azureGetBlockID().
-func TestAzureGetBlockID(t *testing.T) {
-	testCases := []struct {
-		partID        int
-		subPartNumber int
-		uploadID      string
-		md5           string
-		blockID       string
-	}{
-		{1, 7, "f328c35cad938137", "d41d8cd98f00b204e9800998ecf8427e", "MDAwMDEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQ=="},
-		{2, 19, "abcdc35cad938137", "a7fb6b7b36ee4ed66b5546fac4690273", "MDAwMDIuMTkuYWJjZGMzNWNhZDkzODEzNy5hN2ZiNmI3YjM2ZWU0ZWQ2NmI1NTQ2ZmFjNDY5MDI3Mw=="},
-	}
-	for _, test := range testCases {
-		blockID := azureGetBlockID(test.partID, test.subPartNumber, test.uploadID, test.md5)
-		if blockID != test.blockID {
-			t.Fatalf("%s is not equal to %s", blockID, test.blockID)
-		}
-	}
-}
-
-// Test azureParseBlockID().
-func TestAzureParseBlockID(t *testing.T) {
-	testCases := []struct {
-		blockID       string
-		partID        int
-		subPartNumber int
-		uploadID      string
-		md5           string
-		success       bool
-	}{
-		// Invalid base64.
-		{"MDAwMDEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQ=", 0, 0, "", "", false},
-		// Invalid number of tokens.
-		{"MDAwMDEuQUEuZjMyOGMzNWNhZDkzODEzNwo=", 0, 0, "", "", false},
-		// Invalid encoded part ID.
-		{"MDAwMGEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQo=", 0, 0, "", "", false},
-		// Invalid sub part ID.
-		{"MDAwMDEuQUEuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQo=", 0, 0, "", "", false},
-		{"MDAwMDEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQ==", 1, 7, "f328c35cad938137", "d41d8cd98f00b204e9800998ecf8427e", true},
-		{"MDAwMDIuMTkuYWJjZGMzNWNhZDkzODEzNy5hN2ZiNmI3YjM2ZWU0ZWQ2NmI1NTQ2ZmFjNDY5MDI3Mw==", 2, 19, "abcdc35cad938137", "a7fb6b7b36ee4ed66b5546fac4690273", true},
-	}
-	for i, test := range testCases {
-		partID, subPartNumber, uploadID, md5, err := azureParseBlockID(test.blockID)
-		if err != nil && test.success {
-			t.Errorf("Test %d: Expected success but failed %s", i+1, err)
-		}
-		if err == nil && !test.success {
-			t.Errorf("Test %d: Expected to fail but succeeeded insteadl", i+1)
-		}
-		if err == nil {
-			if partID != test.partID {
-				t.Errorf("Test %d: %d not equal to %d", i+1, partID, test.partID)
-			}
-			if subPartNumber != test.subPartNumber {
-				t.Errorf("Test %d: %d not equal to %d", i+1, subPartNumber, test.subPartNumber)
-			}
-			if uploadID != test.uploadID {
-				t.Errorf("Test %d: %s not equal to %s", i+1, uploadID, test.uploadID)
-			}
-			if md5 != test.md5 {
-				t.Errorf("Test %d: %s not equal to %s", i+1, md5, test.md5)
-			}
-		}
-	}
-}
-
 func TestAnonErrToObjectErr(t *testing.T) {
 	testCases := []struct {
 		name       string

cmd/gateway/b2/gateway-b2.go

@@ -0,0 +1,850 @@
+/*
+ * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package b2
+
+import (
+	"context"
+	"crypto/sha1"
+	"fmt"
+	"hash"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"strings"
+	"sync"
+	"time"
+
+	b2 "github.com/minio/blazer/base"
+	"github.com/minio/cli"
+	miniogopolicy "github.com/minio/minio-go/pkg/policy"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/auth"
+	h2 "github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/policy/condition"
+
+	minio "github.com/minio/minio/cmd"
+)
+
+// Supported bucket types by B2 backend.
+const (
+	bucketTypePrivate  = "allPrivate"
+	bucketTypeReadOnly = "allPublic"
+	b2Backend          = "b2"
+)
+
+func init() {
+	const b2GatewayTemplate = `NAME:
+  {{.HelpName}} - {{.Usage}}
+
+USAGE:
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}}
+{{if .VisibleFlags}}
+FLAGS:
+  {{range .VisibleFlags}}{{.}}
+  {{end}}{{end}}
+ENVIRONMENT VARIABLES:
+  ACCESS:
+     MINIO_ACCESS_KEY: B2 account id.
+     MINIO_SECRET_KEY: B2 application key.
+
+  BROWSER:
+     MINIO_BROWSER: To disable web browser access, set this value to "off".
+
+  DOMAIN:
+     MINIO_DOMAIN: To enable virtual-host-style requests, set this value to Minio host domain name.
+
+  CACHE:
+     MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
+     MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
+     MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
+
+EXAMPLES:
+  1. Start minio gateway server for B2 backend.
+     $ export MINIO_ACCESS_KEY=accountID
+     $ export MINIO_SECRET_KEY=applicationKey
+     $ {{.HelpName}}
+
+  2. Start minio gateway server for B2 backend with edge caching enabled.
+     $ export MINIO_ACCESS_KEY=accountID
+     $ export MINIO_SECRET_KEY=applicationKey
+     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
+     $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
+     $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
+     $ {{.HelpName}}
+`
+	minio.RegisterGatewayCommand(cli.Command{
+		Name:               b2Backend,
+		Usage:              "Backblaze B2",
+		Action:             b2GatewayMain,
+		CustomHelpTemplate: b2GatewayTemplate,
+		HideHelpCommand:    true,
+	})
+}
+
+// Handler for 'minio gateway b2' command line.
+func b2GatewayMain(ctx *cli.Context) {
+	minio.StartGateway(ctx, &B2{})
+}
+
+// B2 implements Minio Gateway
+type B2 struct{}
+
+// Name implements Gateway interface.
+func (g *B2) Name() string {
+	return b2Backend
+}
+
+// NewGatewayLayer returns b2 gateway layer, implements ObjectLayer interface to
+// talk to B2 remote backend.
+func (g *B2) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
+	ctx := context.Background()
+	client, err := b2.AuthorizeAccount(ctx, creds.AccessKey, creds.SecretKey, b2.Transport(minio.NewCustomHTTPTransport()))
+	if err != nil {
+		return nil, err
+	}
+
+	return &b2Objects{
+		creds:    creds,
+		b2Client: client,
+		ctx:      ctx,
+	}, nil
+}
+
+// Production - Ready for production use.
+func (g *B2) Production() bool {
+	return true
+}
+
+// b2Object implements gateway for Minio and BackBlaze B2 compatible object storage servers.
+type b2Objects struct {
+	minio.GatewayUnsupported
+	mu       sync.Mutex
+	creds    auth.Credentials
+	b2Client *b2.B2
+	ctx      context.Context
+}
+
+// Convert B2 errors to minio object layer errors.
+func b2ToObjectError(err error, params ...string) error {
+	if err == nil {
+		return nil
+	}
+	bucket := ""
+	object := ""
+	uploadID := ""
+	if len(params) >= 1 {
+		bucket = params[0]
+	}
+	if len(params) == 2 {
+		object = params[1]
+	}
+	if len(params) == 3 {
+		uploadID = params[2]
+	}
+
+	// Following code is a non-exhaustive check to convert
+	// B2 errors into S3 compatible errors.
+	//
+	// For a more complete information - https://www.backblaze.com/b2/docs/
+	statusCode, code, msg := b2.Code(err)
+	if statusCode == 0 {
+		// We don't interpret non B2 errors. B2 errors have statusCode
+		// to help us convert them to S3 object errors.
+		return err
+	}
+
+	switch code {
+	case "duplicate_bucket_name":
+		err = minio.BucketAlreadyOwnedByYou{Bucket: bucket}
+	case "bad_request":
+		if object != "" {
+			err = minio.ObjectNameInvalid{
+				Bucket: bucket,
+				Object: object,
+			}
+		} else if bucket != "" {
+			err = minio.BucketNotFound{Bucket: bucket}
+		}
+	case "bad_json":
+		if object != "" {
+			err = minio.ObjectNameInvalid{
+				Bucket: bucket,
+				Object: object,
+			}
+		} else if bucket != "" {
+			err = minio.BucketNameInvalid{Bucket: bucket}
+		}
+	case "bad_bucket_id":
+		err = minio.BucketNotFound{Bucket: bucket}
+	case "file_not_present", "not_found":
+		err = minio.ObjectNotFound{
+			Bucket: bucket,
+			Object: object,
+		}
+	case "cannot_delete_non_empty_bucket":
+		err = minio.BucketNotEmpty{Bucket: bucket}
+	}
+
+	// Special interpretation like this is required for Multipart sessions.
+	if strings.Contains(msg, "No active upload for") && uploadID != "" {
+		err = minio.InvalidUploadID{UploadID: uploadID}
+	}
+
+	return err
+}
+
+// Shutdown saves any gateway metadata to disk
+// if necessary and reload upon next restart.
+func (l *b2Objects) Shutdown(ctx context.Context) error {
+	// TODO
+	return nil
+}
+
+// StorageInfo is not relevant to B2 backend.
+func (l *b2Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
+	return si
+}
+
+// MakeBucket creates a new container on B2 backend.
+func (l *b2Objects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
+	// location is ignored for B2 backend.
+
+	// All buckets are set to private by default.
+	_, err := l.b2Client.CreateBucket(l.ctx, bucket, bucketTypePrivate, nil, nil)
+	logger.LogIf(ctx, err)
+	return b2ToObjectError(err, bucket)
+}
+
+func (l *b2Objects) reAuthorizeAccount(ctx context.Context) error {
+	client, err := b2.AuthorizeAccount(l.ctx, l.creds.AccessKey, l.creds.SecretKey, b2.Transport(minio.NewCustomHTTPTransport()))
+	if err != nil {
+		return err
+	}
+	l.mu.Lock()
+	l.b2Client.Update(client)
+	l.mu.Unlock()
+	return nil
+}
+
+// listBuckets is a wrapper similar to ListBuckets, which re-authorizes
+// the account and updates the B2 client safely. Once successfully
+// authorized performs the call again and returns list of buckets.
+// For any errors which are not actionable we return an error.
+func (l *b2Objects) listBuckets(ctx context.Context, err error) ([]*b2.Bucket, error) {
+	if err != nil {
+		if b2.Action(err) != b2.ReAuthenticate {
+			return nil, err
+		}
+		if rerr := l.reAuthorizeAccount(ctx); rerr != nil {
+			return nil, rerr
+		}
+	}
+	bktList, lerr := l.b2Client.ListBuckets(l.ctx)
+	if lerr != nil {
+		return l.listBuckets(ctx, lerr)
+	}
+	return bktList, nil
+}
+
+// Bucket - is a helper which provides a *Bucket instance
+// for performing an API operation. B2 API doesn't
+// provide a direct way to access the bucket so we need
+// to employ following technique.
+func (l *b2Objects) Bucket(ctx context.Context, bucket string) (*b2.Bucket, error) {
+	bktList, err := l.listBuckets(ctx, nil)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return nil, b2ToObjectError(err, bucket)
+	}
+	for _, bkt := range bktList {
+		if bkt.Name == bucket {
+			return bkt, nil
+		}
+	}
+	return nil, minio.BucketNotFound{Bucket: bucket}
+}
+
+// GetBucketInfo gets bucket metadata..
+func (l *b2Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, err error) {
+	if _, err = l.Bucket(ctx, bucket); err != nil {
+		return bi, err
+	}
+	return minio.BucketInfo{
+		Name:    bucket,
+		Created: time.Unix(0, 0),
+	}, nil
+}
+
+// ListBuckets lists all B2 buckets
+func (l *b2Objects) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error) {
+	bktList, err := l.listBuckets(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	var bktInfo []minio.BucketInfo
+	for _, bkt := range bktList {
+		bktInfo = append(bktInfo, minio.BucketInfo{
+			Name:    bkt.Name,
+			Created: time.Unix(0, 0),
+		})
+	}
+	return bktInfo, nil
+}
+
+// DeleteBucket deletes a bucket on B2
+func (l *b2Objects) DeleteBucket(ctx context.Context, bucket string) error {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	err = bkt.DeleteBucket(l.ctx)
+	logger.LogIf(ctx, err)
+	return b2ToObjectError(err, bucket)
+}
+
+// ListObjects lists all objects in B2 bucket filtered by prefix, returns upto at max 1000 entries at a time.
+func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return loi, err
+	}
+	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
+	if lerr != nil {
+		logger.LogIf(ctx, lerr)
+		return loi, b2ToObjectError(lerr, bucket)
+	}
+	loi.IsTruncated = next != ""
+	loi.NextMarker = next
+	for _, file := range files {
+		switch file.Status {
+		case "folder":
+			loi.Prefixes = append(loi.Prefixes, file.Name)
+		case "upload":
+			loi.Objects = append(loi.Objects, minio.ObjectInfo{
+				Bucket:      bucket,
+				Name:        file.Name,
+				ModTime:     file.Timestamp,
+				Size:        file.Size,
+				ETag:        minio.ToS3ETag(file.Info.ID),
+				ContentType: file.Info.ContentType,
+				UserDefined: file.Info.Info,
+			})
+		}
+	}
+	return loi, nil
+}
+
+// ListObjectsV2 lists all objects in B2 bucket filtered by prefix, returns upto max 1000 entries at a time.
+func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int,
+	fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
+	// fetchOwner is not supported and unused.
+	marker := continuationToken
+	if marker == "" {
+		// B2's continuation token is an object name to "start at" rather than "start after"
+		// startAfter plus the lowest character B2 supports is used so that the startAfter
+		// object isn't included in the results
+		marker = startAfter + " "
+	}
+
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return loi, err
+	}
+	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
+	if lerr != nil {
+		logger.LogIf(ctx, lerr)
+		return loi, b2ToObjectError(lerr, bucket)
+	}
+	loi.IsTruncated = next != ""
+	loi.ContinuationToken = continuationToken
+	loi.NextContinuationToken = next
+	for _, file := range files {
+		switch file.Status {
+		case "folder":
+			loi.Prefixes = append(loi.Prefixes, file.Name)
+		case "upload":
+			loi.Objects = append(loi.Objects, minio.ObjectInfo{
+				Bucket:      bucket,
+				Name:        file.Name,
+				ModTime:     file.Timestamp,
+				Size:        file.Size,
+				ETag:        minio.ToS3ETag(file.Info.ID),
+				ContentType: file.Info.ContentType,
+				UserDefined: file.Info.Info,
+			})
+		}
+	}
+	return loi, nil
+}
+
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (l *b2Objects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var objInfo minio.ObjectInfo
+	objInfo, err = l.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var startOffset, length int64
+	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+}
+
+// GetObject reads an object from B2. Supports additional
+// parameters like offset and length which are synonymous with
+// HTTP Range requests.
+//
+// startOffset indicates the starting read location of the object.
+// length indicates the total length of the object.
+func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	reader, err := bkt.DownloadFileByName(l.ctx, object, startOffset, length)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return b2ToObjectError(err, bucket, object)
+	}
+	defer reader.Close()
+	_, err = io.Copy(writer, reader)
+	logger.LogIf(ctx, err)
+	return b2ToObjectError(err, bucket, object)
+}
+
+// GetObjectInfo reads object info and replies back ObjectInfo
+func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return objInfo, err
+	}
+	f, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return objInfo, b2ToObjectError(err, bucket, object)
+	}
+	f.Close()
+	fi, err := bkt.File(f.ID, object).GetFileInfo(l.ctx)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return objInfo, b2ToObjectError(err, bucket, object)
+	}
+	return minio.ObjectInfo{
+		Bucket:      bucket,
+		Name:        object,
+		ETag:        minio.ToS3ETag(fi.ID),
+		Size:        fi.Size,
+		ModTime:     fi.Timestamp,
+		ContentType: fi.ContentType,
+		UserDefined: fi.Info,
+	}, nil
+}
+
+// In B2 - You must always include the X-Bz-Content-Sha1 header with
+// your upload request. The value you provide can be:
+// (1) the 40-character hex checksum of the file,
+// (2) the string hex_digits_at_end, or
+// (3) the string do_not_verify.
+// For more reference - https://www.backblaze.com/b2/docs/uploading.html
+//
+// In our case we are going to use (2) option
+const sha1AtEOF = "hex_digits_at_end"
+
+// With the second option mentioned above, you append the 40-character hex sha1
+// to the end of the request body, immediately after the contents of the file
+// being uploaded. Note that the content length is the size of the file plus 40
+// of the original size of the reader.
+//
+// newB2Reader implements a B2 compatible reader by wrapping the hash.Reader into
+// a new io.Reader which will emit out the sha1 hex digits at io.EOF.
+// It also means that your overall content size is now original size + 40 bytes.
+// Additionally this reader also verifies Hash encapsulated inside hash.Reader
+// at io.EOF if the verification failed we return an error and do not send
+// the content to server.
+func newB2Reader(r *h2.Reader, size int64) *Reader {
+	return &Reader{
+		r:        r,
+		size:     size,
+		sha1Hash: sha1.New(),
+	}
+}
+
+// Reader - is a Reader wraps the hash.Reader which will emit out the sha1
+// hex digits at io.EOF. It also means that your overall content size is
+// now original size + 40 bytes. Additionally this reader also verifies
+// Hash encapsulated inside hash.Reader at io.EOF if the verification
+// failed we return an error and do not send the content to server.
+type Reader struct {
+	r        *h2.Reader
+	size     int64
+	sha1Hash hash.Hash
+
+	isEOF bool
+	buf   *strings.Reader
+}
+
+// Size - Returns the total size of Reader.
+func (nb *Reader) Size() int64 { return nb.size + 40 }
+func (nb *Reader) Read(p []byte) (int, error) {
+	if nb.isEOF {
+		return nb.buf.Read(p)
+	}
+	// Read into hash to update the on going checksum.
+	n, err := io.TeeReader(nb.r, nb.sha1Hash).Read(p)
+	if err == io.EOF {
+		// Stream is not corrupted on this end
+		// now fill in the last 40 bytes of sha1 hex
+		// so that the server can verify the stream on
+		// their end.
+		err = nil
+		nb.isEOF = true
+		nb.buf = strings.NewReader(fmt.Sprintf("%x", nb.sha1Hash.Sum(nil)))
+	}
+	return n, err
+}
+
+// PutObject uploads the single upload to B2 backend by using *b2_upload_file* API, uploads upto 5GiB.
+func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	data := r.Reader
+
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return objInfo, err
+	}
+	contentType := opts.UserDefined["content-type"]
+	delete(opts.UserDefined, "content-type")
+
+	var u *b2.URL
+	u, err = bkt.GetUploadURL(l.ctx)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return objInfo, b2ToObjectError(err, bucket, object)
+	}
+
+	hr := newB2Reader(data, data.Size())
+	var f *b2.File
+	f, err = u.UploadFile(l.ctx, hr, int(hr.Size()), object, contentType, sha1AtEOF, opts.UserDefined)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return objInfo, b2ToObjectError(err, bucket, object)
+	}
+
+	var fi *b2.FileInfo
+	fi, err = f.GetFileInfo(l.ctx)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return objInfo, b2ToObjectError(err, bucket, object)
+	}
+
+	return minio.ObjectInfo{
+		Bucket:      bucket,
+		Name:        object,
+		ETag:        minio.ToS3ETag(fi.ID),
+		Size:        fi.Size,
+		ModTime:     fi.Timestamp,
+		ContentType: fi.ContentType,
+		UserDefined: fi.Info,
+	}, nil
+}
+
+// DeleteObject deletes a blob in bucket
+func (l *b2Objects) DeleteObject(ctx context.Context, bucket string, object string) error {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	reader, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return b2ToObjectError(err, bucket, object)
+	}
+	io.Copy(ioutil.Discard, reader)
+	reader.Close()
+	err = bkt.File(reader.ID, object).DeleteFileVersion(l.ctx)
+	logger.LogIf(ctx, err)
+	return b2ToObjectError(err, bucket, object)
+}
+
+// ListMultipartUploads lists all multipart uploads.
+func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string,
+	delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) {
+	// keyMarker, prefix, delimiter are all ignored, Backblaze B2 doesn't support any
+	// of these parameters only equivalent parameter is uploadIDMarker.
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return lmi, err
+	}
+	// The maximum number of files to return from this call.
+	// The default value is 100, and the maximum allowed is 100.
+	if maxUploads > 100 {
+		maxUploads = 100
+	}
+	largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, uploadIDMarker, maxUploads)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return lmi, b2ToObjectError(err, bucket)
+	}
+	lmi = minio.ListMultipartsInfo{
+		MaxUploads: maxUploads,
+	}
+	if nextMarker != "" {
+		lmi.IsTruncated = true
+		lmi.NextUploadIDMarker = nextMarker
+	}
+	for _, largeFile := range largeFiles {
+		lmi.Uploads = append(lmi.Uploads, minio.MultipartInfo{
+			Object:    largeFile.Name,
+			UploadID:  largeFile.ID,
+			Initiated: largeFile.Timestamp,
+		})
+	}
+	return lmi, nil
+}
+
+// NewMultipartUpload upload object in multiple parts, uses B2's LargeFile upload API.
+// Large files can range in size from 5MB to 10TB.
+// Each large file must consist of at least 2 parts, and all of the parts except the
+// last one must be at least 5MB in size. The last part must contain at least one byte.
+// For more information - https://www.backblaze.com/b2/docs/large_files.html
+func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (string, error) {
+	var uploadID string
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return uploadID, err
+	}
+
+	contentType := opts.UserDefined["content-type"]
+	delete(opts.UserDefined, "content-type")
+	lf, err := bkt.StartLargeFile(l.ctx, object, contentType, opts.UserDefined)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return uploadID, b2ToObjectError(err, bucket, object)
+	}
+
+	return lf.ID, nil
+}
+
+// PutObjectPart puts a part of object in bucket, uses B2's LargeFile upload API.
+func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, r *minio.PutObjReader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) {
+	data := r.Reader
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return pi, err
+	}
+
+	fc, err := bkt.File(uploadID, object).CompileParts(0, nil).GetUploadPartURL(l.ctx)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return pi, b2ToObjectError(err, bucket, object, uploadID)
+	}
+
+	hr := newB2Reader(data, data.Size())
+	sha1, err := fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return pi, b2ToObjectError(err, bucket, object, uploadID)
+	}
+
+	return minio.PartInfo{
+		PartNumber:   partID,
+		LastModified: minio.UTCNow(),
+		ETag:         minio.ToS3ETag(sha1),
+		Size:         data.Size(),
+	}, nil
+}
+
+// ListObjectParts returns all object parts for specified object in specified bucket, uses B2's LargeFile upload API.
+func (l *b2Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int, opts minio.ObjectOptions) (lpi minio.ListPartsInfo, err error) {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return lpi, err
+	}
+	lpi = minio.ListPartsInfo{
+		Bucket:           bucket,
+		Object:           object,
+		UploadID:         uploadID,
+		MaxParts:         maxParts,
+		PartNumberMarker: partNumberMarker,
+	}
+	// startPartNumber must be in the range 1 - 10000 for B2.
+	partNumberMarker++
+	partsList, next, err := bkt.File(uploadID, object).ListParts(l.ctx, partNumberMarker, maxParts)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return lpi, b2ToObjectError(err, bucket, object, uploadID)
+	}
+	if next != 0 {
+		lpi.IsTruncated = true
+		lpi.NextPartNumberMarker = next
+	}
+	for _, part := range partsList {
+		lpi.Parts = append(lpi.Parts, minio.PartInfo{
+			PartNumber: part.Number,
+			ETag:       minio.ToS3ETag(part.SHA1),
+			Size:       part.Size,
+		})
+	}
+	return lpi, nil
+}
+
+// AbortMultipartUpload aborts a on going multipart upload, uses B2's LargeFile upload API.
+func (l *b2Objects) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	err = bkt.File(uploadID, object).CompileParts(0, nil).CancelLargeFile(l.ctx)
+	logger.LogIf(ctx, err)
+	return b2ToObjectError(err, bucket, object, uploadID)
+}
+
+// CompleteMultipartUpload completes ongoing multipart upload and finalizes object, uses B2's LargeFile upload API.
+func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, err error) {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return oi, err
+	}
+	hashes := make(map[int]string)
+	for i, uploadedPart := range uploadedParts {
+		// B2 requires contigous part numbers starting with 1, they do not support
+		// hand picking part numbers, we return an S3 compatible error instead.
+		if i+1 != uploadedPart.PartNumber {
+			logger.LogIf(ctx, minio.InvalidPart{})
+			return oi, b2ToObjectError(minio.InvalidPart{}, bucket, object, uploadID)
+		}
+
+		// Trim "-1" suffix in ETag as PutObjectPart() treats B2 returned SHA1 as ETag.
+		hashes[uploadedPart.PartNumber] = strings.TrimSuffix(uploadedPart.ETag, "-1")
+	}
+
+	if _, err = bkt.File(uploadID, object).CompileParts(0, hashes).FinishLargeFile(l.ctx); err != nil {
+		logger.LogIf(ctx, err)
+		return oi, b2ToObjectError(err, bucket, object, uploadID)
+	}
+
+	return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
+}
+
+// SetBucketPolicy - B2 supports 2 types of bucket policies:
+// bucketType.AllPublic - bucketTypeReadOnly means that anybody can download the files is the bucket;
+// bucketType.AllPrivate - bucketTypePrivate means that you need an authorization token to download them.
+// Default is AllPrivate for all buckets.
+func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
+	policyInfo, err := minio.PolicyToBucketAccessPolicy(bucketPolicy)
+	if err != nil {
+		// This should not happen.
+		return b2ToObjectError(err, bucket)
+	}
+
+	var policies []minio.BucketAccessPolicy
+	for prefix, policy := range miniogopolicy.GetPolicies(policyInfo.Statements, bucket, "") {
+		policies = append(policies, minio.BucketAccessPolicy{
+			Prefix: prefix,
+			Policy: policy,
+		})
+	}
+	prefix := bucket + "/*" // For all objects inside the bucket.
+	if len(policies) != 1 {
+		logger.LogIf(ctx, minio.NotImplemented{})
+		return minio.NotImplemented{}
+	}
+	if policies[0].Prefix != prefix {
+		logger.LogIf(ctx, minio.NotImplemented{})
+		return minio.NotImplemented{}
+	}
+	if policies[0].Policy != miniogopolicy.BucketPolicyReadOnly {
+		logger.LogIf(ctx, minio.NotImplemented{})
+		return minio.NotImplemented{}
+	}
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	bkt.Type = bucketTypeReadOnly
+	_, err = bkt.Update(l.ctx)
+	logger.LogIf(ctx, err)
+	return b2ToObjectError(err)
+}
+
+// GetBucketPolicy, returns the current bucketType from B2 backend and convert
+// it into S3 compatible bucket policy info.
+func (l *b2Objects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return nil, err
+	}
+
+	// bkt.Type can also be snapshot, but it is only allowed through B2 browser console,
+	// just return back as policy not found for all cases.
+	// CreateBucket always sets the value to allPrivate by default.
+	if bkt.Type != bucketTypeReadOnly {
+		return nil, minio.BucketPolicyNotFound{Bucket: bucket}
+	}
+
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(
+					policy.GetBucketLocationAction,
+					policy.ListBucketAction,
+					policy.GetObjectAction,
+				),
+				policy.NewResourceSet(
+					policy.NewResource(bucket, ""),
+					policy.NewResource(bucket, "*"),
+				),
+				condition.NewFunctions(),
+			),
+		},
+	}, nil
+}
+
+// DeleteBucketPolicy - resets the bucketType of bucket on B2 to 'allPrivate'.
+func (l *b2Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	bkt.Type = bucketTypePrivate
+	_, err = bkt.Update(l.ctx)
+	logger.LogIf(ctx, err)
+	return b2ToObjectError(err)
+}
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (l *b2Objects) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/b2/gateway-b2_test.go

@@ -0,0 +1,119 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package b2
+
+import (
+	"fmt"
+	"testing"
+
+	b2 "github.com/minio/blazer/base"
+
+	minio "github.com/minio/minio/cmd"
+)
+
+// Test b2 object error.
+func TestB2ObjectError(t *testing.T) {
+	testCases := []struct {
+		params      []string
+		b2Err       error
+		expectedErr error
+	}{
+		{
+			[]string{}, nil, nil,
+		},
+		{
+			[]string{}, fmt.Errorf("Not *Error"), fmt.Errorf("Not *Error"),
+		},
+		{
+			[]string{}, fmt.Errorf("Non B2 Error"), fmt.Errorf("Non B2 Error"),
+		},
+		{
+			[]string{"bucket"}, b2.Error{
+				StatusCode: 1,
+				Code:       "duplicate_bucket_name",
+			}, minio.BucketAlreadyOwnedByYou{
+				Bucket: "bucket",
+			},
+		},
+		{
+			[]string{"bucket"}, b2.Error{
+				StatusCode: 1,
+				Code:       "bad_request",
+			}, minio.BucketNotFound{
+				Bucket: "bucket",
+			},
+		},
+		{
+			[]string{"bucket", "object"}, b2.Error{
+				StatusCode: 1,
+				Code:       "bad_request",
+			}, minio.ObjectNameInvalid{
+				Bucket: "bucket",
+				Object: "object",
+			},
+		},
+		{
+			[]string{"bucket"}, b2.Error{
+				StatusCode: 1,
+				Code:       "bad_bucket_id",
+			}, minio.BucketNotFound{Bucket: "bucket"},
+		},
+		{
+			[]string{"bucket", "object"}, b2.Error{
+				StatusCode: 1,
+				Code:       "file_not_present",
+			}, minio.ObjectNotFound{
+				Bucket: "bucket",
+				Object: "object",
+			},
+		},
+		{
+			[]string{"bucket", "object"}, b2.Error{
+				StatusCode: 1,
+				Code:       "not_found",
+			}, minio.ObjectNotFound{
+				Bucket: "bucket",
+				Object: "object",
+			},
+		},
+		{
+			[]string{"bucket"}, b2.Error{
+				StatusCode: 1,
+				Code:       "cannot_delete_non_empty_bucket",
+			}, minio.BucketNotEmpty{
+				Bucket: "bucket",
+			},
+		},
+		{
+			[]string{"bucket", "object", "uploadID"}, b2.Error{
+				StatusCode: 1,
+				Message:    "No active upload for",
+			}, minio.InvalidUploadID{
+				UploadID: "uploadID",
+			},
+		},
+	}
+
+	for i, testCase := range testCases {
+		actualErr := b2ToObjectError(testCase.b2Err, testCase.params...)
+		if actualErr != nil {
+			if actualErr.Error() != testCase.expectedErr.Error() {
+				t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, actualErr)
+			}
+		}
+	}
+}

cmd/gateway/gateway.go

@@ -23,5 +23,9 @@ import (
 	_ "github.com/minio/minio/cmd/gateway/nas"
 	_ "github.com/minio/minio/cmd/gateway/oss"
 	_ "github.com/minio/minio/cmd/gateway/s3"
+
+	// B2 is specifically kept here to avoid re-ordering by goimports,
+	// please ask on github.com/minio/minio/issues before changing this.
+	_ "github.com/minio/minio/cmd/gateway/b2"
 	// Add your gateway here.
 )

cmd/gateway/gcs/gateway-gcs.go

@@ -324,8 +324,6 @@ func gcsToObjectError(err error, params ...string) error {
 			break
 		}
 		err = minio.BucketNotEmpty{Bucket: bucket}
-	default:
-		err = fmt.Errorf("Unsupported error reason: %s", reason)
 	}
 
 	return err
@@ -877,7 +875,7 @@ func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object st
 }
 
 // PutObject - Create a new object with the incoming data,
-func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
+func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r *minio.PutObjReader, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
 	data := r.Reader
 
 	// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
@@ -890,12 +888,13 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r
 	object := l.client.Bucket(bucket).Object(key)
 
 	w := object.NewWriter(ctx)
+
 	// Disable "chunked" uploading in GCS client if the size of the data to be uploaded is below
 	// the current chunk-size of the writer. This avoids an unnecessary memory allocation.
 	if data.Size() < int64(w.ChunkSize) {
 		w.ChunkSize = 0
 	}
-	applyMetadataToGCSAttrs(metadata, &w.ObjectAttrs)
+	applyMetadataToGCSAttrs(opts.UserDefined, &w.ObjectAttrs)
 
 	if _, err := io.Copy(w, data); err != nil {
 		// Close the object writer upon error.
@@ -947,7 +946,7 @@ func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object str
 }
 
 // NewMultipartUpload - upload object in multiple parts
-func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
+func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key string, o minio.ObjectOptions) (uploadID string, err error) {
 	// generate new uploadid
 	uploadID = minio.MustGetUUID()
 
@@ -957,7 +956,7 @@ func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key
 	w := l.client.Bucket(bucket).Object(meta).NewWriter(ctx)
 	defer w.Close()
 
-	applyMetadataToGCSAttrs(metadata, &w.ObjectAttrs)
+	applyMetadataToGCSAttrs(o.UserDefined, &w.ObjectAttrs)
 
 	if err = json.NewEncoder(w).Encode(gcsMultipartMetaV1{
 		gcsMinioMultipartMetaCurrentVersion,

cmd/gateway/gcs/gateway-gcs_test.go

@@ -375,15 +375,6 @@ func TestGCSToObjectError(t *testing.T) {
 				Object: "object",
 			},
 		},
-		{
-			[]string{"bucket", "object"},
-			&googleapi.Error{
-				Errors: []googleapi.ErrorItem{{
-					Reason: "unknown",
-				}},
-			},
-			fmt.Errorf("Unsupported error reason: unknown"),
-		},
 	}
 
 	for i, testCase := range testCases {

cmd/gateway/oss/gateway-oss.go

@@ -31,6 +31,7 @@ import (
 	"github.com/minio/cli"
 	miniogopolicy "github.com/minio/minio-go/pkg/policy"
 	minio "github.com/minio/minio/cmd"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/hash"
@@ -655,10 +656,10 @@ func ossPutObject(ctx context.Context, client *oss.Client, bucket, object string
 }
 
 // PutObject creates a new object with the incoming data.
-func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	data := r.Reader
 
-	return ossPutObject(ctx, l.Client, bucket, object, data, metadata)
+	return ossPutObject(ctx, l.Client, bucket, object, data, opts.UserDefined)
 }
 
 // CopyObject copies an object from source bucket to a destination bucket.
@@ -752,7 +753,7 @@ func (l *ossObjects) ListMultipartUploads(ctx context.Context, bucket, prefix, k
 }
 
 // NewMultipartUpload upload object in multiple parts.
-func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
+func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object string, o minio.ObjectOptions) (uploadID string, err error) {
 	bkt, err := l.Client.Bucket(bucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -760,7 +761,7 @@ func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object stri
 	}
 
 	// Build OSS metadata
-	opts, err := appendS3MetaToOSSOptions(ctx, nil, metadata)
+	opts, err := appendS3MetaToOSSOptions(ctx, nil, o.UserDefined)
 	if err != nil {
 		return uploadID, ossToObjectError(err, bucket, object)
 	}
@@ -844,7 +845,7 @@ func ossListObjectParts(client *oss.Client, bucket, object, uploadID string, par
 	}
 
 	// always drain output (response body)
-	defer minio.CloseResponse(resp.Body)
+	defer xhttp.DrainBody(resp.Body)
 
 	err = xml.NewDecoder(resp.Body).Decode(&lupr)
 	if err != nil {

cmd/gateway/oss/gateway-oss_test.go

@@ -111,9 +111,8 @@ func TestOSSToObjectError(t *testing.T) {
 
 func TestS3MetaToOSSOptions(t *testing.T) {
 	var err error
-	var headers map[string]string
 
-	headers = map[string]string{
+	headers := map[string]string{
 		"x-amz-meta-invalid_meta": "value",
 	}
 	_, err = appendS3MetaToOSSOptions(context.Background(), nil, headers)

cmd/gateway/s3/gateway-s3-sse.go

@@ -236,7 +236,7 @@ func (l *s3EncObjects) writeGWMetadata(ctx context.Context, bucket, metaFileName
 		logger.LogIf(ctx, err)
 		return err
 	}
-	_, err = l.s3Objects.PutObject(ctx, bucket, metaFileName, reader, map[string]string{}, o)
+	_, err = l.s3Objects.PutObject(ctx, bucket, metaFileName, reader, o)
 	return err
 }
 
@@ -373,7 +373,8 @@ func (l *s3EncObjects) CopyObject(ctx context.Context, srcBucket string, srcObje
 			return gwMeta.ToObjectInfo(dstBucket, dstObject), nil
 		}
 	}
-	return l.PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, srcInfo.UserDefined, d)
+	dstOpts := minio.ObjectOptions{ServerSideEncryption: d.ServerSideEncryption, UserDefined: srcInfo.UserDefined}
+	return l.PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, dstOpts)
 }
 
 // DeleteObject deletes a blob in bucket
@@ -406,23 +407,24 @@ func (l *s3EncObjects) ListMultipartUploads(ctx context.Context, bucket string,
 }
 
 // NewMultipartUpload uploads object in multiple parts
-func (l *s3EncObjects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
-	var opts minio.ObjectOptions
-	if o.ServerSideEncryption != nil &&
-		((minio.GlobalGatewaySSE.SSEC() && o.ServerSideEncryption.Type() == encrypt.SSEC) ||
-			(minio.GlobalGatewaySSE.SSES3() && o.ServerSideEncryption.Type() == encrypt.S3)) {
-		opts = o
-	}
+func (l *s3EncObjects) NewMultipartUpload(ctx context.Context, bucket string, object string, o minio.ObjectOptions) (uploadID string, err error) {
+	var sseOpts encrypt.ServerSide
 	if o.ServerSideEncryption == nil {
-		return l.s3Objects.NewMultipartUpload(ctx, bucket, object, metadata, opts)
+		return l.s3Objects.NewMultipartUpload(ctx, bucket, object, minio.ObjectOptions{UserDefined: o.UserDefined})
 	}
-	uploadID, err = l.s3Objects.NewMultipartUpload(ctx, bucket, getGWContentPath(object), map[string]string{}, opts)
+	// Decide if sse options needed to be passed to backend
+	if (minio.GlobalGatewaySSE.SSEC() && o.ServerSideEncryption.Type() == encrypt.SSEC) ||
+		(minio.GlobalGatewaySSE.SSES3() && o.ServerSideEncryption.Type() == encrypt.S3) {
+		sseOpts = o.ServerSideEncryption
+	}
+
+	uploadID, err = l.s3Objects.NewMultipartUpload(ctx, bucket, getGWContentPath(object), minio.ObjectOptions{ServerSideEncryption: sseOpts})
 	if err != nil {
 		return
 	}
 	// Create uploadID and write a temporary dare.meta object under object/uploadID prefix
 	gwmeta := newGWMetaV1()
-	gwmeta.Meta = metadata
+	gwmeta.Meta = o.UserDefined
 	gwmeta.Stat.ModTime = time.Now().UTC()
 	err = l.writeGWMetadata(ctx, bucket, getTmpDareMetaPath(object, uploadID), gwmeta, minio.ObjectOptions{})
 	if err != nil {
@@ -432,30 +434,28 @@ func (l *s3EncObjects) NewMultipartUpload(ctx context.Context, bucket string, ob
 }
 
 // PutObject creates a new object with the incoming data,
-func (l *s3EncObjects) PutObject(ctx context.Context, bucket string, object string, data *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
-	var s3Opts minio.ObjectOptions
+func (l *s3EncObjects) PutObject(ctx context.Context, bucket string, object string, data *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	var sseOpts encrypt.ServerSide
+	// Decide if sse options needed to be passed to backend
 	if opts.ServerSideEncryption != nil &&
 		((minio.GlobalGatewaySSE.SSEC() && opts.ServerSideEncryption.Type() == encrypt.SSEC) ||
 			(minio.GlobalGatewaySSE.SSES3() && opts.ServerSideEncryption.Type() == encrypt.S3)) {
-		s3Opts = opts
+		sseOpts = opts.ServerSideEncryption
 	}
 	if opts.ServerSideEncryption == nil {
 		defer l.deleteGWMetadata(ctx, bucket, getDareMetaPath(object))
 		defer l.DeleteObject(ctx, bucket, getGWContentPath(object))
-		return l.s3Objects.PutObject(ctx, bucket, object, data, metadata, s3Opts)
+		return l.s3Objects.PutObject(ctx, bucket, object, data, minio.ObjectOptions{UserDefined: opts.UserDefined})
 	}
 
-	oi, err := l.s3Objects.PutObject(ctx, bucket, getGWContentPath(object), data, map[string]string{}, s3Opts)
+	oi, err := l.s3Objects.PutObject(ctx, bucket, getGWContentPath(object), data, minio.ObjectOptions{ServerSideEncryption: sseOpts})
 	if err != nil {
 		return objInfo, minio.ErrorRespToObjectError(err)
 	}
 
 	gwMeta := newGWMetaV1()
 	gwMeta.Meta = make(map[string]string)
-	for k, v := range oi.UserDefined {
-		gwMeta.Meta[k] = v
-	}
-	for k, v := range metadata {
+	for k, v := range opts.UserDefined {
 		gwMeta.Meta[k] = v
 	}
 	encMD5 := data.MD5CurrentHexString()

cmd/gateway/s3/gateway-s3.go

@@ -448,15 +448,15 @@ func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object str
 }
 
 // PutObject creates a new object with the incoming data,
-func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	data := r.Reader
-	oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), minio.ToMinioClientMetadata(metadata), opts.ServerSideEncryption)
+	oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), minio.ToMinioClientMetadata(opts.UserDefined), opts.ServerSideEncryption)
 	if err != nil {
 		return objInfo, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 	// On success, populate the key & metadata so they are present in the notification
 	oi.Key = object
-	oi.Metadata = minio.ToMinioClientObjectInfoMetadata(metadata)
+	oi.Metadata = minio.ToMinioClientObjectInfoMetadata(opts.UserDefined)
 
 	return minio.FromMinioClientObjectInfo(bucket, oi), nil
 }
@@ -508,9 +508,9 @@ func (l *s3Objects) ListMultipartUploads(ctx context.Context, bucket string, pre
 }
 
 // NewMultipartUpload upload object in multiple parts
-func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
+func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, o minio.ObjectOptions) (uploadID string, err error) {
 	// Create PutObject options
-	opts := miniogo.PutObjectOptions{UserMetadata: metadata, ServerSideEncryption: o.ServerSideEncryption}
+	opts := miniogo.PutObjectOptions{UserMetadata: o.UserDefined, ServerSideEncryption: o.ServerSideEncryption}
 	uploadID, err = l.Client.NewMultipartUpload(bucket, object, opts)
 	if err != nil {
 		return uploadID, minio.ErrorRespToObjectError(err, bucket, object)

cmd/generic-handlers.go

@@ -90,7 +90,7 @@ func setRequestHeaderSizeLimitHandler(h http.Handler) http.Handler {
 // of the user-defined metadata to 2 KB.
 func (h requestHeaderSizeLimitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if isHTTPHeaderSizeTooLarge(r.Header) {
-		writeErrorResponse(w, ErrMetadataTooLarge, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrMetadataTooLarge), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	h.Handler.ServeHTTP(w, r)
@@ -133,7 +133,7 @@ func filterReservedMetadata(h http.Handler) http.Handler {
 // would be treated as metadata.
 func (h reservedMetadataHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if containsReservedMetadata(r.Header) {
-		writeErrorResponse(w, ErrUnsupportedMetadata, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrUnsupportedMetadata), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	h.Handler.ServeHTTP(w, r)
@@ -300,7 +300,7 @@ func (h minioReservedBucketHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
 		// buckets
 		bucketName, _ := urlPath2BucketObjectName(r.URL.Path)
 		if isMinioReservedBucket(bucketName) || isMinioMetaBucket(bucketName) {
-			writeErrorResponse(w, ErrAllAccessDisabled, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrAllAccessDisabled), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -358,19 +358,19 @@ func (h timeValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	aType := getRequestAuthType(r)
 	if aType == authTypeSigned || aType == authTypeSignedV2 || aType == authTypeStreamingSigned {
 		// Verify if date headers are set, if not reject the request
-		amzDate, apiErr := parseAmzDateHeader(r)
-		if apiErr != ErrNone {
+		amzDate, errCode := parseAmzDateHeader(r)
+		if errCode != ErrNone {
 			// All our internal APIs are sensitive towards Date
 			// header, for all requests where Date header is not
 			// present we will reject such clients.
-			writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
 			return
 		}
 		// Verify if the request date header is shifted by less than globalMaxSkewTime parameter in the past
 		// or in the future, reject request otherwise.
 		curTime := UTCNow()
 		if curTime.Sub(amzDate) > globalMaxSkewTime || amzDate.Sub(curTime) > globalMaxSkewTime {
-			writeErrorResponse(w, ErrRequestTimeTooSkewed, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -381,37 +381,9 @@ type resourceHandler struct {
 	handler http.Handler
 }
 
-// List of default allowable HTTP methods.
-var defaultAllowableHTTPMethods = []string{
-	http.MethodGet,
-	http.MethodPut,
-	http.MethodHead,
-	http.MethodPost,
-	http.MethodDelete,
-	http.MethodOptions,
-}
-
 // setCorsHandler handler for CORS (Cross Origin Resource Sharing)
 func setCorsHandler(h http.Handler) http.Handler {
-	commonS3Headers := []string{
-		"Date",
-		"ETag",
-		"Server",
-		"Connection",
-		"Accept-Ranges",
-		"Content-Range",
-		"Content-Encoding",
-		"Content-Length",
-		"Content-Type",
-		"x-amz-request-id",
-	}
-	c := cors.New(cors.Options{
-		AllowedOrigins:   []string{"*"},
-		AllowedMethods:   defaultAllowableHTTPMethods,
-		AllowedHeaders:   []string{"*"},
-		ExposedHeaders:   commonS3Headers,
-		AllowCredentials: true,
-	})
+	c := cors.AllowAll()
 	return c.Handler(h)
 }
 
@@ -426,8 +398,24 @@ func setIgnoreResourcesHandler(h http.Handler) http.Handler {
 // Checks requests for not implemented Bucket resources
 func ignoreNotImplementedBucketResources(req *http.Request) bool {
 	for name := range req.URL.Query() {
-		// Enable GetBucketACL dummy call specifically.
-		if name == "acl" && req.Method == http.MethodGet {
+		// Enable GetBucketACL, GetBucketCors, GetBucketWebsite,
+		// GetBucketAcccelerate, GetBucketRequestPayment,
+		// GetBucketLogging, GetBucketLifecycle,
+		// GetBucketReplication, GetBucketTagging,
+		// GetBucketVersioning, DeleteBucketTagging,
+		// and DeleteBucketWebsite dummy calls specifically.
+		if ((name == "acl" ||
+			name == "cors" ||
+			name == "website" ||
+			name == "accelerate" ||
+			name == "requestPayment" ||
+			name == "logging" ||
+			name == "lifecycle" ||
+			name == "replication" ||
+			name == "tagging" ||
+			name == "versioning") && req.Method == http.MethodGet) ||
+			((name == "tagging" ||
+				name == "website") && req.Method == http.MethodDelete) {
 			return false
 		}
 
@@ -441,8 +429,8 @@ func ignoreNotImplementedBucketResources(req *http.Request) bool {
 // Checks requests for not implemented Object resources
 func ignoreNotImplementedObjectResources(req *http.Request) bool {
 	for name := range req.URL.Query() {
-		// Enable GetObjectACL dummy call specifically.
-		if name == "acl" && req.Method == http.MethodGet {
+		// Enable GetObjectACL and GetObjectTagging dummy calls specifically.
+		if (name == "acl" || name == "tagging") && req.Method == http.MethodGet {
 			return false
 		}
 		if notimplementedObjectResourceNames[name] {
@@ -454,28 +442,28 @@ func ignoreNotImplementedObjectResources(req *http.Request) bool {
 
 // List of not implemented bucket queries
 var notimplementedBucketResourceNames = map[string]bool{
+	"accelerate":     true,
 	"acl":            true,
 	"cors":           true,
+	"inventory":      true,
 	"lifecycle":      true,
 	"logging":        true,
-	"replication":    true,
-	"tagging":        true,
-	"versions":       true,
-	"requestPayment": true,
-	"versioning":     true,
-	"website":        true,
-	"inventory":      true,
 	"metrics":        true,
-	"accelerate":     true,
+	"replication":    true,
+	"requestPayment": true,
+	"tagging":        true,
+	"versioning":     true,
+	"versions":       true,
+	"website":        true,
 }
 
 // List of not implemented object queries
 var notimplementedObjectResourceNames = map[string]bool{
-	"torrent": true,
 	"acl":     true,
 	"policy":  true,
-	"tagging": true,
 	"restore": true,
+	"tagging": true,
+	"torrent": true,
 }
 
 // Resource handler ServeHTTP() wrapper
@@ -485,14 +473,14 @@ func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// If bucketName is present and not objectName check for bucket level resource queries.
 	if bucketName != "" && objectName == "" {
 		if ignoreNotImplementedBucketResources(r) {
-			writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 	// If bucketName and objectName are present check for its resource queries.
 	if bucketName != "" && objectName != "" {
 		if ignoreNotImplementedObjectResources(r) {
-			writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -562,14 +550,14 @@ func (h httpStatsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	globalHTTPStats.updateStats(r, ww, durationSecs)
 }
 
-// pathValidityHandler validates all the incoming paths for
-// any bad components and rejects them.
-type pathValidityHandler struct {
+// requestValidityHandler validates all the incoming paths for
+// any malicious requests.
+type requestValidityHandler struct {
 	handler http.Handler
 }
 
-func setPathValidityHandler(h http.Handler) http.Handler {
-	return pathValidityHandler{handler: h}
+func setRequestValidityHandler(h http.Handler) http.Handler {
+	return requestValidityHandler{handler: h}
 }
 
 // Bad path components to be rejected by the path validity handler.
@@ -593,21 +581,36 @@ func hasBadPathComponent(path string) bool {
 	return false
 }
 
-func (h pathValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+// Check if client is sending a malicious request.
+func hasMultipleAuth(r *http.Request) bool {
+	authTypeCount := 0
+	for _, hasValidAuth := range []func(*http.Request) bool{isRequestSignatureV2, isRequestPresignedSignatureV2, isRequestSignatureV4, isRequestPresignedSignatureV4, isRequestJWT, isRequestPostPolicySignatureV4} {
+		if hasValidAuth(r) {
+			authTypeCount++
+		}
+	}
+	return authTypeCount > 1
+}
+
+func (h requestValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Check for bad components in URL path.
 	if hasBadPathComponent(r.URL.Path) {
-		writeErrorResponse(w, ErrInvalidResourceName, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	// Check for bad components in URL query values.
 	for _, vv := range r.URL.Query() {
 		for _, v := range vv {
 			if hasBadPathComponent(v) {
-				writeErrorResponse(w, ErrInvalidResourceName, r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL, guessIsBrowserReq(r))
 				return
 			}
 		}
 	}
+	if hasMultipleAuth(r) {
+		writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL, guessIsBrowserReq(r))
+		return
+	}
 	h.handler.ServeHTTP(w, r)
 }
 
@@ -647,9 +650,9 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 			sr, err := globalDNSConfig.Get(bucket)
 			if err != nil {
 				if err == dns.ErrNoEntriesFound {
-					writeErrorResponse(w, ErrNoSuchBucket, r.URL, guessIsBrowserReq(r))
+					writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrNoSuchBucket), r.URL, guessIsBrowserReq(r))
 				} else {
-					writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL, guessIsBrowserReq(r))
+					writeErrorResponse(context.Background(), w, toAPIError(context.Background(), err), r.URL, guessIsBrowserReq(r))
 				}
 				return
 			}
@@ -691,9 +694,9 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 	sr, err := globalDNSConfig.Get(bucket)
 	if err != nil {
 		if err == dns.ErrNoEntriesFound {
-			writeErrorResponse(w, ErrNoSuchBucket, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrNoSuchBucket), r.URL, guessIsBrowserReq(r))
 		} else {
-			writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, toAPIError(context.Background(), err), r.URL, guessIsBrowserReq(r))
 		}
 		return
 	}
@@ -753,7 +756,7 @@ func (l rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// potential pileup on the server.
 	if err := l.Wait(ctx); err != nil {
 		// Send an S3 compatible error, SlowDown.
-		writeErrorResponse(w, ErrSlowDown, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrSlowDown), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -807,7 +810,7 @@ type criticalErrorHandler struct{ handler http.Handler }
 func (h criticalErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	defer func() {
 		if err := recover(); err == logger.ErrCritical { // handle
-			writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrInternalError), r.URL, guessIsBrowserReq(r))
 		} else if err != nil {
 			panic(err) // forward other panic calls
 		}
@@ -824,9 +827,9 @@ func (h sseTLSHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Deny SSE-C requests if not made over TLS
 	if !globalIsSSL && (crypto.SSEC.IsRequested(r.Header) || crypto.SSECopy.IsRequested(r.Header)) {
 		if r.Method == http.MethodHead {
-			writeErrorResponseHeadersOnly(w, ErrInsecureSSECustomerRequest)
+			writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest))
 		} else {
-			writeErrorResponse(w, ErrInsecureSSECustomerRequest, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest), r.URL, guessIsBrowserReq(r))
 		}
 		return
 	}

cmd/handler-utils.go

@@ -40,7 +40,7 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError
 	// be created at default region.
 	locationConstraint := createBucketLocationConfiguration{}
 	err := xmlDecoder(r.Body, &locationConstraint, r.ContentLength)
-	if err != nil && err != io.EOF {
+	if err != nil && r.ContentLength != 0 {
 		logger.LogIf(context.Background(), err)
 		// Treat all other failures as XML parsing errors.
 		return "", ErrMalformedXML
@@ -366,12 +366,10 @@ func getResource(path string, host string, domain string) (string, error) {
 
 // If none of the http routes match respond with MethodNotAllowed, in JSON
 func notFoundHandlerJSON(w http.ResponseWriter, r *http.Request) {
-	writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
-	return
+	writeErrorResponseJSON(context.Background(), w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 }
 
 // If none of the http routes match respond with MethodNotAllowed
 func notFoundHandler(w http.ResponseWriter, r *http.Request) {
-	writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
-	return
+	writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
 }

cmd/handler-utils_test.go

@@ -39,48 +39,50 @@ func TestIsValidLocationContraint(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// Test with corrupted XML
+	// Corrupted XML
 	malformedReq := &http.Request{
 		Body:          ioutil.NopCloser(bytes.NewBuffer([]byte("<>"))),
 		ContentLength: int64(len("<>")),
 	}
-	if _, err := parseLocationConstraint(malformedReq); err != ErrMalformedXML {
-		t.Fatal("Unexpected error: ", err)
+
+	// Not an XML
+	badRequest := &http.Request{
+		Body:          ioutil.NopCloser(bytes.NewReader([]byte("garbage"))),
+		ContentLength: int64(len("garbage")),
 	}
 
 	// generates the input request with XML bucket configuration set to the request body.
-	createExpectedRequest := func(req *http.Request, location string) (*http.Request, error) {
+	createExpectedRequest := func(req *http.Request, location string) *http.Request {
 		createBucketConfig := createBucketLocationConfiguration{}
 		createBucketConfig.Location = location
-		var createBucketConfigBytes []byte
-		createBucketConfigBytes, e := xml.Marshal(createBucketConfig)
-		if e != nil {
-			return nil, e
-		}
+		createBucketConfigBytes, _ := xml.Marshal(createBucketConfig)
 		createBucketConfigBuffer := bytes.NewBuffer(createBucketConfigBytes)
 		req.Body = ioutil.NopCloser(createBucketConfigBuffer)
 		req.ContentLength = int64(createBucketConfigBuffer.Len())
-		return req, nil
+		return req
 	}
 
 	testCases := []struct {
-		locationForInputRequest string
-		serverConfigRegion      string
-		expectedCode            APIErrorCode
+		request            *http.Request
+		serverConfigRegion string
+		expectedCode       APIErrorCode
 	}{
 		// Test case - 1.
-		{globalMinioDefaultRegion, globalMinioDefaultRegion, ErrNone},
+		{createExpectedRequest(&http.Request{}, "eu-central-1"), globalMinioDefaultRegion, ErrNone},
 		// Test case - 2.
 		// In case of empty request body ErrNone is returned.
-		{"", globalMinioDefaultRegion, ErrNone},
+		{createExpectedRequest(&http.Request{}, ""), globalMinioDefaultRegion, ErrNone},
+		// Test case - 3
+		// In case of garbage request body ErrMalformedXML is returned.
+		{badRequest, globalMinioDefaultRegion, ErrMalformedXML},
+		// Test case - 4
+		// In case of invalid XML request body ErrMalformedXML is returned.
+		{malformedReq, globalMinioDefaultRegion, ErrMalformedXML},
 	}
+
 	for i, testCase := range testCases {
-		inputRequest, e := createExpectedRequest(&http.Request{}, testCase.locationForInputRequest)
-		if e != nil {
-			t.Fatalf("Test %d: Failed to Marshal bucket configuration", i+1)
-		}
 		globalServerConfig.SetRegion(testCase.serverConfigRegion)
-		_, actualCode := parseLocationConstraint(inputRequest)
+		_, actualCode := parseLocationConstraint(testCase.request)
 		if testCase.expectedCode != actualCode {
 			t.Errorf("Test %d: Expected the APIErrCode to be %d, but instead found %d", i+1, testCase.expectedCode, actualCode)
 		}

cmd/healthcheck-handler_test.go

@@ -34,7 +34,7 @@ func TestGoroutineCountCheck(t *testing.T) {
 		// Make goroutines -- to make sure number of go-routines is higher than threshold
 		if tt.threshold == 5 || tt.threshold == 6 {
 			for i := 0; i < 6; i++ {
-				go time.Sleep(5)
+				go time.Sleep(5 * time.Nanosecond)
 			}
 		}
 		if err := goroutineCountCheck(tt.threshold); (err != nil) != tt.wantErr {

cmd/http-stats.go

@@ -19,58 +19,28 @@ package cmd
 import (
 	"fmt"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
-
 	"go.uber.org/atomic"
 )
 
-func getRequestResource(r *http.Request) string {
-	if r == nil {
-		// http.Request is nil when non-HTTP data (like TLS record) is read/written.
-		return ""
-	}
-
-	if globalDomainName != "" {
-		host := r.Header.Get("Host")
-		if strings.HasSuffix(host, "."+globalDomainName) {
-			return "/" + strings.TrimSuffix(host, "."+globalDomainName) + r.URL.Path
-		}
-	}
-
-	return r.URL.Path
-}
-
 // ConnStats - Network statistics
 // Count total input/output transferred bytes during
 // the server's life.
 type ConnStats struct {
-	totalInputBytes     atomic.Uint64
-	totalOutputBytes    atomic.Uint64
-	totalRPCInputBytes  atomic.Uint64
-	totalRPCOutputBytes atomic.Uint64
+	totalInputBytes  atomic.Uint64
+	totalOutputBytes atomic.Uint64
 }
 
 // Increase total input bytes
-func (s *ConnStats) incInputBytes(r *http.Request, n int) {
-	resource := getRequestResource(r)
-	if resource == minioReservedBucketPath || strings.HasPrefix(resource, minioReservedBucketPath+"/") {
-		s.totalRPCInputBytes.Add(uint64(n))
-	} else {
-		s.totalInputBytes.Add(uint64(n))
-	}
+func (s *ConnStats) incInputBytes(n int) {
+	s.totalInputBytes.Add(uint64(n))
 }
 
 // Increase total output bytes
-func (s *ConnStats) incOutputBytes(r *http.Request, n int) {
-	resource := getRequestResource(r)
-	if resource == minioReservedBucketPath || strings.HasPrefix(resource, minioReservedBucketPath+"/") {
-		s.totalRPCOutputBytes.Add(uint64(n))
-	} else {
-		s.totalOutputBytes.Add(uint64(n))
-	}
+func (s *ConnStats) incOutputBytes(n int) {
+	s.totalOutputBytes.Add(uint64(n))
 }
 
 // Return total input bytes

cmd/http/bufconn.go

@@ -19,28 +19,17 @@ package http
 import (
 	"bufio"
 	"net"
-	"net/http"
 	"time"
 )
 
 // BufConn - is a generic stream-oriented network connection supporting buffered reader and read/write timeout.
 type BufConn struct {
 	QuirkConn
-	bufReader              *bufio.Reader            // buffered reader wraps reader in net.Conn.
-	readTimeout            time.Duration            // sets the read timeout in the connection.
-	writeTimeout           time.Duration            // sets the write timeout in the connection.
-	request                *http.Request            // HTTP request information.
-	updateBytesReadFunc    func(*http.Request, int) // function to be called to update bytes read.
-	updateBytesWrittenFunc func(*http.Request, int) // function to be called to update bytes written.
-}
-
-func (c *BufConn) setRequest(request *http.Request) {
-	c.request = request
-}
-
-func (c *BufConn) setUpdateFuncs(updateBytesReadFunc, updateBytesWrittenFunc func(*http.Request, int)) {
-	c.updateBytesReadFunc = updateBytesReadFunc
-	c.updateBytesWrittenFunc = updateBytesWrittenFunc
+	bufReader              *bufio.Reader // buffered reader wraps reader in net.Conn.
+	readTimeout            time.Duration // sets the read timeout in the connection.
+	writeTimeout           time.Duration // sets the write timeout in the connection.
+	updateBytesReadFunc    func(int)     // function to be called to update bytes read.
+	updateBytesWrittenFunc func(int)     // function to be called to update bytes written.
 }
 
 // Sets read timeout
@@ -81,7 +70,7 @@ func (c *BufConn) Read(b []byte) (n int, err error) {
 	c.setReadTimeout()
 	n, err = c.bufReader.Read(b)
 	if err == nil && c.updateBytesReadFunc != nil {
-		c.updateBytesReadFunc(c.request, n)
+		c.updateBytesReadFunc(n)
 	}
 
 	return n, err
@@ -92,18 +81,20 @@ func (c *BufConn) Write(b []byte) (n int, err error) {
 	c.setWriteTimeout()
 	n, err = c.Conn.Write(b)
 	if err == nil && c.updateBytesWrittenFunc != nil {
-		c.updateBytesWrittenFunc(c.request, n)
+		c.updateBytesWrittenFunc(n)
 	}
 
 	return n, err
 }
 
 // newBufConn - creates a new connection object wrapping net.Conn.
-func newBufConn(c net.Conn, readTimeout, writeTimeout time.Duration, maxHeaderBytes int) *BufConn {
+func newBufConn(c net.Conn, readTimeout, writeTimeout time.Duration, maxHeaderBytes int, updateBytesReadFunc, updateBytesWrittenFunc func(int)) *BufConn {
 	return &BufConn{
-		QuirkConn:    QuirkConn{Conn: c},
-		bufReader:    bufio.NewReaderSize(c, maxHeaderBytes),
-		readTimeout:  readTimeout,
-		writeTimeout: writeTimeout,
+		QuirkConn:              QuirkConn{Conn: c},
+		bufReader:              bufio.NewReaderSize(c, maxHeaderBytes),
+		readTimeout:            readTimeout,
+		writeTimeout:           writeTimeout,
+		updateBytesReadFunc:    updateBytesReadFunc,
+		updateBytesWrittenFunc: updateBytesWrittenFunc,
 	}
 }

cmd/http/bufconn_test.go

@@ -49,7 +49,7 @@ func TestBuffConnReadTimeout(t *testing.T) {
 			t.Errorf("failed to accept new connection. %v", terr)
 			return
 		}
-		bufconn := newBufConn(tcpConn, 1*time.Second, 1*time.Second, 4096)
+		bufconn := newBufConn(tcpConn, 1*time.Second, 1*time.Second, 4096, nil, nil)
 		defer bufconn.Close()
 
 		// Read a line

cmd/http/close.go

@@ -0,0 +1,42 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package http
+
+import (
+	"io"
+	"io/ioutil"
+)
+
+// DrainBody close non nil response with any response Body.
+// convenient wrapper to drain any remaining data on response body.
+//
+// Subsequently this allows golang http RoundTripper
+// to re-use the same connection for future requests.
+func DrainBody(respBody io.ReadCloser) {
+	// Callers should close resp.Body when done reading from it.
+	// If resp.Body is not closed, the Client's underlying RoundTripper
+	// (typically Transport) may not be able to re-use a persistent TCP
+	// connection to the server for a subsequent "keep-alive" request.
+	if respBody != nil {
+		// Drain any remaining Body and then close the connection.
+		// Without this closing connection would disallow re-using
+		// the same connection for future uses.
+		//  - http://stackoverflow.com/a/17961593/4465767
+		defer respBody.Close()
+		io.Copy(ioutil.Discard, respBody)
+	}
+}

cmd/http/listener.go

@@ -23,9 +23,7 @@ import (
 	"io"
 	"net"
 	"net/http"
-	"net/url"
 	"os"
-	"strings"
 	"sync"
 	"syscall"
 	"time"
@@ -33,9 +31,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 )
 
-var sslRequiredErrMsg = []byte("HTTP/1.0 403 Forbidden\r\n\r\nSSL required")
-
-var badRequestMsg = []byte("HTTP/1.0 400 Bad Request\r\n\r\n")
+var sslRequiredErrMsg = []byte("HTTP/1.1 403 Forbidden\r\n\r\nSSL required")
 
 // HTTP methods.
 var methods = []string{
@@ -51,30 +47,6 @@ var methods = []string{
 	"PRI", // HTTP 2 method
 }
 
-// maximum length of above methods + one space.
-var methodMaxLen = getMethodMaxLen() + 1
-
-func getMethodMaxLen() int {
-	maxLen := 0
-	for _, method := range methods {
-		if len(method) > maxLen {
-			maxLen = len(method)
-		}
-	}
-
-	return maxLen
-}
-
-func isHTTPMethod(s string) bool {
-	for _, method := range methods {
-		if s == method {
-			return true
-		}
-	}
-
-	return false
-}
-
 func getPlainText(bufConn *BufConn) (bool, error) {
 	defer bufConn.setReadTimeout()
 
@@ -95,66 +67,6 @@ func getPlainText(bufConn *BufConn) (bool, error) {
 	return false, nil
 }
 
-func getMethodResourceHost(bufConn *BufConn, maxHeaderBytes int) (method string, resource string, host string, err error) {
-	defer bufConn.setReadTimeout()
-
-	var data []byte
-	for dataLen := 1; dataLen < maxHeaderBytes; dataLen++ {
-		if bufConn.canSetReadDeadline() {
-			// Set deadline such that we close the connection quickly
-			// of no data was received from the Peek()
-			bufConn.SetReadDeadline(time.Now().UTC().Add(time.Second * 3))
-		}
-
-		data, err = bufConn.bufReader.Peek(dataLen)
-		if err != nil {
-			return "", "", "", err
-		}
-
-		tokens := strings.Split(string(data), "\n")
-		if len(tokens) < 2 {
-			continue
-		}
-
-		if method == "" && resource == "" {
-			if i := strings.IndexByte(tokens[0], ' '); i == -1 {
-				return "", "", "", fmt.Errorf("malformed HTTP request from '%s'", bufConn.LocalAddr())
-			}
-			httpTokens := strings.SplitN(tokens[0], " ", 3)
-			if len(httpTokens) < 3 {
-				return "", "", "", fmt.Errorf("malformed HTTP request from '%s'", bufConn.LocalAddr())
-			}
-			if !isHTTPMethod(httpTokens[0]) {
-				return "", "", "", fmt.Errorf("malformed HTTP request, invalid HTTP method '%s' from '%s'",
-					httpTokens[0], bufConn.LocalAddr())
-			}
-
-			method = httpTokens[0]
-			resource = httpTokens[1]
-		}
-
-		for _, token := range tokens[1:] {
-			if token == "" || !strings.HasSuffix(token, "\r") {
-				continue
-			}
-
-			// HTTP headers are case insensitive, so we should simply convert
-			// each tokens to their lower case form to match 'host' header.
-			token = strings.ToLower(token)
-			if strings.HasPrefix(token, "host:") {
-				host = strings.TrimPrefix(strings.TrimSuffix(token, "\r"), "host:")
-				return method, resource, host, nil
-			}
-		}
-
-		if tokens[len(tokens)-1] == "\r" {
-			break
-		}
-	}
-
-	return "", "", "", fmt.Errorf("malformed HTTP request from %s", bufConn.LocalAddr())
-}
-
 type acceptResult struct {
 	conn net.Conn
 	err  error
@@ -171,8 +83,8 @@ type httpListener struct {
 	readTimeout            time.Duration
 	writeTimeout           time.Duration
 	maxHeaderBytes         int
-	updateBytesReadFunc    func(*http.Request, int) // function to be called to update bytes read in BufConn.
-	updateBytesWrittenFunc func(*http.Request, int) // function to be called to update bytes written in BufConn.
+	updateBytesReadFunc    func(int) // function to be called to update bytes read in BufConn.
+	updateBytesWrittenFunc func(int) // function to be called to update bytes written in BufConn.
 }
 
 // isRoutineNetErr returns true if error is due to a network timeout,
@@ -222,7 +134,8 @@ func (listener *httpListener) start() {
 		tcpConn.SetKeepAlive(true)
 		tcpConn.SetKeepAlivePeriod(listener.tcpKeepAliveTimeout)
 
-		bufconn := newBufConn(tcpConn, listener.readTimeout, listener.writeTimeout, listener.maxHeaderBytes)
+		bufconn := newBufConn(tcpConn, listener.readTimeout, listener.writeTimeout, listener.maxHeaderBytes,
+			listener.updateBytesReadFunc, listener.updateBytesWrittenFunc)
 		if listener.tlsConfig != nil {
 			ok, err := getPlainText(bufconn)
 			if err != nil {
@@ -248,52 +161,8 @@ func (listener *httpListener) start() {
 				bufconn.Close()
 				return
 			}
-
-			// As the listener is configured with TLS, try to do TLS handshake, drop the connection if it fails.
-			tlsConn := tls.Server(bufconn, listener.tlsConfig)
-
-			if err := tlsConn.Handshake(); err != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", bufconn.RemoteAddr().String())
-				reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String())
-				ctx := logger.SetReqInfo(context.Background(), reqInfo)
-				logger.LogIf(ctx, err)
-				bufconn.Close()
-				return
-			}
-
-			bufconn = newBufConn(tlsConn, listener.readTimeout, listener.writeTimeout, listener.maxHeaderBytes)
 		}
 
-		method, resource, host, err := getMethodResourceHost(bufconn, listener.maxHeaderBytes)
-		if err != nil {
-			// Peek could fail legitimately when clients abruptly close
-			// connection. E.g. Chrome browser opens connections speculatively to
-			// speed up loading of a web page. Peek may also fail due to network
-			// saturation on a transport with read timeout set. All other kind of
-			// errors should be logged for further investigation. Thanks @brendanashworth.
-			if !isRoutineNetErr(err) {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", bufconn.RemoteAddr().String())
-				reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String())
-				ctx := logger.SetReqInfo(context.Background(), reqInfo)
-				logger.LogIf(ctx, err)
-				bufconn.Write(badRequestMsg)
-			}
-			bufconn.Close()
-			return
-		}
-
-		header := make(http.Header)
-		if host != "" {
-			header.Add("Host", host)
-		}
-		bufconn.setRequest(&http.Request{
-			Method: method,
-			URL:    &url.URL{Path: resource},
-			Host:   bufconn.LocalAddr().String(),
-			Header: header,
-		})
-		bufconn.setUpdateFuncs(listener.updateBytesReadFunc, listener.updateBytesWrittenFunc)
-
 		send(acceptResult{bufconn, nil}, doneCh)
 	}
 
@@ -380,8 +249,8 @@ func newHTTPListener(serverAddrs []string,
 	readTimeout time.Duration,
 	writeTimeout time.Duration,
 	maxHeaderBytes int,
-	updateBytesReadFunc func(*http.Request, int),
-	updateBytesWrittenFunc func(*http.Request, int)) (listener *httpListener, err error) {
+	updateBytesReadFunc func(int),
+	updateBytesWrittenFunc func(int)) (listener *httpListener, err error) {
 
 	var tcpListeners []*net.TCPListener
 

cmd/http/listener_test.go

@@ -17,17 +17,14 @@
 package http
 
 import (
-	"bufio"
 	"bytes"
 	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
 	"net"
-	"net/http"
 	"strconv"
 	"strings"
-	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -156,43 +153,6 @@ func getNonLoopBackIP(t *testing.T) string {
 	return nonLoopBackIP
 }
 
-// Test getMethodMaxLen()
-func TestGetMethodMaxLen(t *testing.T) {
-	l := getMethodMaxLen()
-	if l != (methodMaxLen - 1) {
-		t.Fatalf("expected: %v, got: %v", (methodMaxLen - 1), l)
-	}
-}
-
-// Test isHTTPMethod()
-func TestIsHTTPMethod(t *testing.T) {
-	testCases := []struct {
-		method         string
-		expectedResult bool
-	}{
-		{"", false},
-		{"get", false},
-		{"put", false},
-		{"UPLOAD", false},
-		{"OPTIONS", true},
-		{"GET", true},
-		{"HEAD", true},
-		{"POST", true},
-		{"PUT", true},
-		{"DELETE", true},
-		{"TRACE", true},
-		{"CONNECT", true},
-		{"PRI", true},
-	}
-
-	for _, testCase := range testCases {
-		result := isHTTPMethod(testCase.method)
-		if result != testCase.expectedResult {
-			t.Fatalf("expected: %v, got: %v", testCase.expectedResult, result)
-		}
-	}
-}
-
 func TestNewHTTPListener(t *testing.T) {
 	tlsConfig := getTLSConfig(t)
 
@@ -202,8 +162,8 @@ func TestNewHTTPListener(t *testing.T) {
 		tcpKeepAliveTimeout    time.Duration
 		readTimeout            time.Duration
 		writeTimeout           time.Duration
-		updateBytesReadFunc    func(*http.Request, int)
-		updateBytesWrittenFunc func(*http.Request, int)
+		updateBytesReadFunc    func(int)
+		updateBytesWrittenFunc func(int)
 		expectedErr            bool
 	}{
 		{[]string{"93.184.216.34:65432"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, true},
@@ -266,8 +226,7 @@ func TestHTTPListenerStartClose(t *testing.T) {
 			time.Duration(0),
 			time.Duration(0),
 			DefaultMaxHeaderBytes,
-			nil,
-			nil,
+			nil, nil,
 		)
 		if err != nil {
 			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
@@ -314,8 +273,7 @@ func TestHTTPListenerAddr(t *testing.T) {
 			time.Duration(0),
 			time.Duration(0),
 			DefaultMaxHeaderBytes,
-			nil,
-			nil,
+			nil, nil,
 		)
 		if err != nil {
 			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
@@ -359,8 +317,7 @@ func TestHTTPListenerAddrs(t *testing.T) {
 			time.Duration(0),
 			time.Duration(0),
 			DefaultMaxHeaderBytes,
-			nil,
-			nil,
+			nil, nil,
 		)
 		if err != nil {
 			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
@@ -380,149 +337,6 @@ func TestHTTPListenerAddrs(t *testing.T) {
 	}
 }
 
-func TestHTTPListenerAccept(t *testing.T) {
-	tlsConfig := getTLSConfig(t)
-	nonLoopBackIP := getNonLoopBackIP(t)
-
-	testCases := []struct {
-		serverAddrs         []string
-		tlsConfig           *tls.Config
-		request             string
-		reply               string
-		expectedRequestLine string
-	}{
-		{[]string{"localhost:0"}, nil, "GET / HTTP/1.0\r\nHost: example.org\r\n\r\n", "200 OK\r\n", "GET / HTTP/1.0\r\n"},
-		{[]string{nonLoopBackIP + ":0"}, nil, "POST / HTTP/1.0\r\nHost: example.org\r\n\r\n", "200 OK\r\n", "POST / HTTP/1.0\r\n"},
-		{[]string{nonLoopBackIP + ":0"}, nil, "HEAD / HTTP/1.0\r\nhost: example.org\r\n\r\n", "200 OK\r\n", "HEAD / HTTP/1.0\r\n"},
-		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, nil, "CONNECT / HTTP/1.0\r\nHost: www.example.org\r\n\r\n", "200 OK\r\n", "CONNECT / HTTP/1.0\r\n"},
-		{[]string{"localhost:0"}, tlsConfig, "GET / HTTP/1.0\r\nHost: example.org\r\n\r\n", "200 OK\r\n", "GET / HTTP/1.0\r\n"},
-		{[]string{nonLoopBackIP + ":0"}, tlsConfig, "POST / HTTP/1.0\r\nHost: example.org\r\n\r\n", "200 OK\r\n", "POST / HTTP/1.0\r\n"},
-		{[]string{nonLoopBackIP + ":0"}, tlsConfig, "HEAD / HTTP/1.0\r\nhost: example.org\r\n\r\n", "200 OK\r\n", "HEAD / HTTP/1.0\r\n"},
-		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, tlsConfig, "CONNECT / HTTP/1.0\r\nHost: www.example.org\r\n\r\n", "200 OK\r\n", "CONNECT / HTTP/1.0\r\n"},
-	}
-
-	for i, testCase := range testCases {
-		listener, err := newHTTPListener(
-			testCase.serverAddrs,
-			testCase.tlsConfig,
-			time.Duration(0),
-			time.Duration(0),
-			time.Duration(0),
-			DefaultMaxHeaderBytes,
-			nil,
-			nil,
-		)
-		if err != nil {
-			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
-		}
-
-		for _, serverAddr := range listener.Addrs() {
-			var conn net.Conn
-			var err error
-
-			if testCase.tlsConfig == nil {
-				conn, err = net.Dial("tcp", serverAddr.String())
-			} else {
-				conn, err = tls.Dial("tcp", serverAddr.String(), &tls.Config{InsecureSkipVerify: true})
-			}
-			if err != nil {
-				t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
-			}
-
-			if _, err = io.WriteString(conn, testCase.request); err != nil {
-				t.Fatalf("Test %d: request send: expected = <nil>, got = %v", i+1, err)
-			}
-
-			serverConn, err := listener.Accept()
-			if err != nil {
-				t.Fatalf("Test %d: accept: expected = <nil>, got = %v", i+1, err)
-			}
-
-			requestLine, err := bufio.NewReader(serverConn).ReadString('\n')
-			if err != nil {
-				t.Fatalf("Test %d: request read: expected = <nil>, got = %v", i+1, err)
-			}
-
-			if requestLine != testCase.expectedRequestLine {
-				t.Fatalf("Test %d: request: expected = %v, got = %v", i+1, testCase.expectedRequestLine, requestLine)
-			}
-
-			if _, err = io.WriteString(serverConn, testCase.reply); err != nil {
-				t.Fatalf("Test %d: reply send: expected = <nil>, got = %v", i+1, err)
-			}
-
-			reply, err := bufio.NewReader(conn).ReadString('\n')
-			if err != nil {
-				t.Fatalf("Test %d: reply read: expected = <nil>, got = %v", i+1, err)
-			}
-
-			if testCase.reply != reply {
-				t.Fatalf("Test %d: reply: expected = %v, got = %v", i+1, testCase.reply, reply)
-			}
-
-			serverConn.Close()
-			conn.Close()
-		}
-
-		listener.Close()
-	}
-}
-
-func TestHTTPListenerAcceptPeekError(t *testing.T) {
-	tlsConfig := getTLSConfig(t)
-	nonLoopBackIP := getNonLoopBackIP(t)
-
-	testCases := []struct {
-		serverAddrs []string
-		tlsConfig   *tls.Config
-		request     string
-	}{
-		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, nil, "CONN"},
-		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, tlsConfig, "CONN"},
-	}
-
-	for i, testCase := range testCases {
-		listener, err := newHTTPListener(
-			testCase.serverAddrs,
-			testCase.tlsConfig,
-			time.Duration(0),
-			time.Duration(0),
-			time.Duration(0),
-			DefaultMaxHeaderBytes,
-			nil,
-			nil,
-		)
-		if err != nil {
-			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
-		}
-
-		go func() {
-			serverConn, aerr := listener.Accept()
-			if aerr == nil {
-				fail(t, "Test %d: accept: expected = <error>, got = <nil>", i+1)
-			}
-			if serverConn != nil {
-				fail(t, "Test %d: accept: server expected = <nil>, got = %v", i+1, serverConn)
-			}
-		}()
-
-		for _, serverAddr := range listener.Addrs() {
-			conn, err := net.Dial("tcp", serverAddr.String())
-			if err != nil {
-				t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
-			}
-
-			if _, err = io.WriteString(conn, testCase.request); err != nil {
-				t.Fatalf("Test %d: request send: expected = <nil>, got = %v", i+1, err)
-			}
-
-			conn.Close()
-		}
-
-		listener.Close()
-	}
-}
-
 func TestHTTPListenerAcceptTLSError(t *testing.T) {
 	tlsConfig := getTLSConfig(t)
 	nonLoopBackIP := getNonLoopBackIP(t)
@@ -543,8 +357,7 @@ func TestHTTPListenerAcceptTLSError(t *testing.T) {
 			time.Duration(0),
 			time.Duration(0),
 			DefaultMaxHeaderBytes,
-			nil,
-			nil,
+			nil, nil,
 		)
 		if err != nil {
 			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
@@ -587,200 +400,6 @@ func TestHTTPListenerAcceptTLSError(t *testing.T) {
 	}
 }
 
-func TestHTTPListenerAcceptError(t *testing.T) {
-	tlsConfig := getTLSConfig(t)
-	nonLoopBackIP := getNonLoopBackIP(t)
-
-	testCases := []struct {
-		serverAddrs  []string
-		tlsConfig    *tls.Config
-		secureClient bool
-		request      string
-	}{
-		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, nil, false, "CONNECTION"},
-		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, tlsConfig, false, "CONNECTION"},
-		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, tlsConfig, true, "CONNECTION"},
-	}
-
-	for i, testCase := range testCases {
-		listener, err := newHTTPListener(
-			testCase.serverAddrs,
-			testCase.tlsConfig,
-			time.Duration(0),
-			time.Duration(0),
-			time.Duration(0),
-			DefaultMaxHeaderBytes,
-			nil,
-			nil,
-		)
-		if err != nil {
-			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
-		}
-
-		for _, serverAddr := range listener.Addrs() {
-			var conn net.Conn
-			var err error
-
-			if testCase.secureClient {
-				conn, err = tls.Dial("tcp", serverAddr.String(), &tls.Config{InsecureSkipVerify: true})
-			} else {
-				conn, err = net.Dial("tcp", serverAddr.String())
-			}
-			if err != nil {
-				t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
-			}
-
-			if _, err = io.WriteString(conn, testCase.request); err != nil {
-				t.Fatalf("Test %d: request send: expected = <nil>, got = %v", i+1, err)
-			}
-
-			go func() {
-				serverConn, aerr := listener.Accept()
-				if aerr == nil {
-					fail(t, "Test %d: accept: expected = <error>, got = <nil>", i+1)
-				}
-				if serverConn != nil {
-					fail(t, "Test %d: accept: server expected = <nil>, got = %v", i+1, serverConn)
-				}
-			}()
-
-			if !testCase.secureClient && testCase.tlsConfig != nil {
-				buf := make([]byte, len(sslRequiredErrMsg))
-				var n int
-				n, err = io.ReadFull(conn, buf)
-				if err != nil {
-					t.Fatalf("Test %d: reply read: expected = <nil> got = %v", i+1, err)
-				} else if n != len(buf) {
-					t.Fatalf("Test %d: reply length: expected = %v got = %v", i+1, len(buf), n)
-				} else if !bytes.Equal(buf, sslRequiredErrMsg) {
-					t.Fatalf("Test %d: reply: expected = %v got = %v", i+1, string(sslRequiredErrMsg), string(buf))
-				}
-				continue
-			}
-
-			if _, err = bufio.NewReader(conn).ReadString('\n'); err != io.EOF {
-				t.Errorf("Test %d: reply read: expected = io.EOF, got = %s", i+1, err)
-			}
-
-			conn.Close()
-		}
-
-		listener.Close()
-	}
-}
-
-func TestHTTPListenerAcceptParallel(t *testing.T) {
-	tlsConfig := getTLSConfig(t)
-	nonLoopBackIP := getNonLoopBackIP(t)
-
-	testCases := []struct {
-		serverAddrs []string
-		tlsConfig   *tls.Config
-		reply       string
-	}{
-		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, nil, "200 OK\n"},
-		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, tlsConfig, "200 OK\n"},
-	}
-
-	connect := func(i int, serverAddr string, secure bool, delay bool, request, reply string) {
-		var conn net.Conn
-		var err error
-
-		if secure {
-			conn, err = tls.Dial("tcp", serverAddr, &tls.Config{InsecureSkipVerify: true})
-		} else {
-			conn, err = net.Dial("tcp", serverAddr)
-		}
-		if err != nil {
-			fail(t, "Test %d: error: expected = <nil>, got = %v", i+1, err)
-		}
-
-		if delay {
-			if _, err = io.WriteString(conn, request[:3]); err != nil {
-				fail(t, "Test %d: request send: expected = <nil>, got = %v", i+1, err)
-			}
-			time.Sleep(1 * time.Second)
-			if _, err = io.WriteString(conn, request[3:]); err != nil {
-				fail(t, "Test %d: request send: expected = <nil>, got = %v", i+1, err)
-			}
-		} else {
-			if _, err = io.WriteString(conn, request); err != nil {
-				fail(t, "Test %d: request send: expected = <nil>, got = %v", i+1, err)
-			}
-		}
-
-		received, err := bufio.NewReader(conn).ReadString('\n')
-		if err != nil {
-			fail(t, "Test %d: reply read: expected = <nil>, got = %v", i+1, err)
-		}
-		if received != reply {
-			fail(t, "Test %d: reply: expected = %v, got = %v", i+1, reply, received)
-		}
-
-		conn.Close()
-	}
-
-	handleConnection := func(i int, wg *sync.WaitGroup, serverConn net.Conn, request, reply string) {
-		defer wg.Done()
-
-		received, err := bufio.NewReader(serverConn).ReadString('\n')
-		if err != nil {
-			fail(t, "Test %d: request read: expected = <nil>, got = %v", i+1, err)
-		}
-
-		if received != request {
-			fail(t, "Test %d: request: expected = %v, got = %v", i+1, request, received)
-		}
-
-		if _, err := io.WriteString(serverConn, reply); err != nil {
-			fail(t, "Test %d: reply send: expected = <nil>, got = %v", i+1, err)
-		}
-
-		serverConn.Close()
-	}
-
-	for i, testCase := range testCases {
-		listener, err := newHTTPListener(
-			testCase.serverAddrs,
-			testCase.tlsConfig,
-			time.Duration(0),
-			time.Duration(0),
-			time.Duration(0),
-			DefaultMaxHeaderBytes,
-			nil,
-			nil,
-		)
-		if err != nil {
-			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
-		}
-
-		for _, serverAddr := range listener.Addrs() {
-			go connect(i, serverAddr.String(), testCase.tlsConfig != nil, true, "GET /1 HTTP/1.0\r\nHost: example.org\r\nr\n", testCase.reply)
-			go connect(i, serverAddr.String(), testCase.tlsConfig != nil, false, "GET /2 HTTP/1.0\r\nHost: example.org\r\n\r\n", testCase.reply)
-
-			var wg sync.WaitGroup
-
-			serverConn, err := listener.Accept()
-			if err != nil {
-				t.Fatalf("Test %d: accept: expected = <nil>, got = %v", i+1, err)
-			}
-			wg.Add(1)
-			go handleConnection(i, &wg, serverConn, "GET /2 HTTP/1.0\r\n", testCase.reply)
-
-			serverConn, err = listener.Accept()
-			if err != nil {
-				t.Fatalf("Test %d: accept: expected = <nil>, got = %v", i+1, err)
-			}
-			wg.Add(1)
-			go handleConnection(i, &wg, serverConn, "GET /1 HTTP/1.0\r\n", testCase.reply)
-
-			wg.Wait()
-		}
-
-		listener.Close()
-	}
-}
-
 type myTimeoutErr struct {
 	timeout bool
 }

cmd/http/server.go

@@ -51,15 +51,15 @@ const (
 // Server - extended http.Server supports multiple addresses to serve and enhanced connection handling.
 type Server struct {
 	http.Server
-	Addrs                  []string                 // addresses on which the server listens for new connection.
-	ShutdownTimeout        time.Duration            // timeout used for graceful server shutdown.
-	TCPKeepAliveTimeout    time.Duration            // timeout used for underneath TCP connection.
-	UpdateBytesReadFunc    func(*http.Request, int) // function to be called to update bytes read in bufConn.
-	UpdateBytesWrittenFunc func(*http.Request, int) // function to be called to update bytes written in bufConn.
-	listenerMutex          *sync.Mutex              // to guard 'listener' field.
-	listener               *httpListener            // HTTP listener for all 'Addrs' field.
-	inShutdown             uint32                   // indicates whether the server is in shutdown or not
-	requestCount           int32                    // counter holds no. of request in progress.
+	Addrs                  []string      // addresses on which the server listens for new connection.
+	ShutdownTimeout        time.Duration // timeout used for graceful server shutdown.
+	TCPKeepAliveTimeout    time.Duration // timeout used for underneath TCP connection.
+	UpdateBytesReadFunc    func(int)     // function to be called to update bytes read in bufConn.
+	UpdateBytesWrittenFunc func(int)     // function to be called to update bytes written in bufConn.
+	listenerMutex          *sync.Mutex   // to guard 'listener' field.
+	listener               *httpListener // HTTP listener for all 'Addrs' field.
+	inShutdown             uint32        // indicates whether the server is in shutdown or not
+	requestCount           int32         // counter holds no. of request in progress.
 }
 
 // GetRequestCount - returns number of request in progress.
@@ -80,8 +80,6 @@ func (srv *Server) Start() (err error) {
 
 	addrs := set.CreateStringSet(srv.Addrs...).ToSlice() // copy and remove duplicates
 	tcpKeepAliveTimeout := srv.TCPKeepAliveTimeout
-	updateBytesReadFunc := srv.UpdateBytesReadFunc
-	updateBytesWrittenFunc := srv.UpdateBytesWrittenFunc
 
 	// Create new HTTP listener.
 	var listener *httpListener
@@ -92,8 +90,8 @@ func (srv *Server) Start() (err error) {
 		readTimeout,
 		writeTimeout,
 		srv.MaxHeaderBytes,
-		updateBytesReadFunc,
-		updateBytesWrittenFunc,
+		srv.UpdateBytesReadFunc,
+		srv.UpdateBytesWrittenFunc,
 	)
 	if err != nil {
 		return err
@@ -121,6 +119,9 @@ func (srv *Server) Start() (err error) {
 	srv.listenerMutex.Unlock()
 
 	// Start servicing with listener.
+	if tlsConfig != nil {
+		return srv.Server.Serve(tls.NewListener(listener, tlsConfig))
+	}
 	return srv.Server.Serve(listener)
 }
 
@@ -192,7 +193,7 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
 			CipherSuites:             defaultCipherSuites,
 			CurvePreferences:         secureCurves,
 			MinVersion:               tls.VersionTLS12,
-			NextProtos:               []string{"http/1.1"},
+			NextProtos:               []string{"h2", "http/1.1"},
 		}
 		tlsConfig.GetCertificate = getCert
 	}

cmd/iam.go

@@ -94,23 +94,20 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
 	// the following reasons:
 	//  - Read quorum is lost just after the initialization
 	//    of the object layer.
-	retryTimerCh := newRetryTimerSimple(doneCh)
-	for {
-		select {
-		case _ = <-retryTimerCh:
-			// Load IAMSys once during boot.
-			if err := sys.refresh(objAPI); err != nil {
-				if err == errDiskNotFound ||
-					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
-					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
-					logger.Info("Waiting for IAM subsystem to be initialized..")
-					continue
-				}
-				return err
+	for range newRetryTimerSimple(doneCh) {
+		// Load IAMSys once during boot.
+		if err := sys.refresh(objAPI); err != nil {
+			if err == errDiskNotFound ||
+				strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+				strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+				logger.Info("Waiting for IAM subsystem to be initialized..")
+				continue
 			}
-			return nil
+			return err
 		}
+		break
 	}
+	return nil
 }
 
 // DeleteCannedPolicy - deletes a canned policy.

cmd/local-locker.go

@@ -145,11 +145,9 @@ func (l *localLocker) ForceUnlock(args dsync.LockArgs) (reply bool, err error) {
 	if len(args.UID) != 0 {
 		return false, fmt.Errorf("ForceUnlock called with non-empty UID: %s", args.UID)
 	}
-	if _, ok := l.lockMap[args.Resource]; ok {
-		// Only clear lock when it is taken
-		// Remove the lock (irrespective of write or read lock)
-		delete(l.lockMap, args.Resource)
-	}
+	// Only clear lock when it is taken
+	// Remove the lock (irrespective of write or read lock)
+	delete(l.lockMap, args.Resource)
 	return true, nil
 }
 
@@ -159,11 +157,7 @@ func (l *localLocker) DupLockMap() map[string][]lockRequesterInfo {
 
 	lockCopy := make(map[string][]lockRequesterInfo)
 	for k, v := range l.lockMap {
-		var lockSlice []lockRequesterInfo
-		for _, lockInfo := range v {
-			lockSlice = append(lockSlice, lockInfo)
-		}
-		lockCopy[k] = lockSlice
+		lockCopy[k] = append(lockCopy[k], v...)
 	}
 	return lockCopy
 }

cmd/lock-rpc-server-common_test.go

@@ -41,7 +41,7 @@ func TestLockRpcServerRemoveEntryIfExists(t *testing.T) {
 	// first test by simulating item has already been deleted
 	locker.ll.removeEntryIfExists(nlrip)
 	{
-		gotLri, _ := locker.ll.lockMap["name"]
+		gotLri := locker.ll.lockMap["name"]
 		expectedLri := []lockRequesterInfo(nil)
 		if !reflect.DeepEqual(expectedLri, gotLri) {
 			t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
@@ -52,7 +52,7 @@ func TestLockRpcServerRemoveEntryIfExists(t *testing.T) {
 	locker.ll.lockMap["name"] = []lockRequesterInfo{lri} // add item
 	locker.ll.removeEntryIfExists(nlrip)
 	{
-		gotLri, _ := locker.ll.lockMap["name"]
+		gotLri := locker.ll.lockMap["name"]
 		expectedLri := []lockRequesterInfo(nil)
 		if !reflect.DeepEqual(expectedLri, gotLri) {
 			t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
@@ -87,7 +87,7 @@ func TestLockRpcServerRemoveEntry(t *testing.T) {
 		lockRequesterInfo2,
 	}
 
-	lri, _ := locker.ll.lockMap["name"]
+	lri := locker.ll.lockMap["name"]
 
 	// test unknown uid
 	if locker.ll.removeEntry("name", "unknown-uid", &lri) {
@@ -97,7 +97,7 @@ func TestLockRpcServerRemoveEntry(t *testing.T) {
 	if !locker.ll.removeEntry("name", "0123-4567", &lri) {
 		t.Errorf("Expected %#v, got %#v", true, false)
 	} else {
-		gotLri, _ := locker.ll.lockMap["name"]
+		gotLri := locker.ll.lockMap["name"]
 		expectedLri := []lockRequesterInfo{lockRequesterInfo2}
 		if !reflect.DeepEqual(expectedLri, gotLri) {
 			t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
@@ -107,7 +107,7 @@ func TestLockRpcServerRemoveEntry(t *testing.T) {
 	if !locker.ll.removeEntry("name", "89ab-cdef", &lri) {
 		t.Errorf("Expected %#v, got %#v", true, false)
 	} else {
-		gotLri, _ := locker.ll.lockMap["name"]
+		gotLri := locker.ll.lockMap["name"]
 		expectedLri := []lockRequesterInfo(nil)
 		if !reflect.DeepEqual(expectedLri, gotLri) {
 			t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)

cmd/lock-rpc-server_test.go

@@ -94,7 +94,7 @@ func TestLockRpcServerLock(t *testing.T) {
 		if !result {
 			t.Errorf("Expected %#v, got %#v", true, result)
 		} else {
-			gotLri, _ := locker.ll.lockMap["name"]
+			gotLri := locker.ll.lockMap["name"]
 			expectedLri := []lockRequesterInfo{
 				{
 					Writer:          true,
@@ -174,7 +174,7 @@ func TestLockRpcServerUnlock(t *testing.T) {
 		if !result {
 			t.Errorf("Expected %#v, got %#v", true, result)
 		} else {
-			gotLri, _ := locker.ll.lockMap["name"]
+			gotLri := locker.ll.lockMap["name"]
 			expectedLri := []lockRequesterInfo(nil)
 			if !testLockEquality(expectedLri, gotLri) {
 				t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
@@ -210,7 +210,7 @@ func TestLockRpcServerRLock(t *testing.T) {
 		if !result {
 			t.Errorf("Expected %#v, got %#v", true, result)
 		} else {
-			gotLri, _ := locker.ll.lockMap["name"]
+			gotLri := locker.ll.lockMap["name"]
 			expectedLri := []lockRequesterInfo{
 				{
 					Writer:          false,
@@ -312,7 +312,7 @@ func TestLockRpcServerRUnlock(t *testing.T) {
 		if !result {
 			t.Errorf("Expected %#v, got %#v", true, result)
 		} else {
-			gotLri, _ := locker.ll.lockMap["name"]
+			gotLri := locker.ll.lockMap["name"]
 			expectedLri := []lockRequesterInfo{
 				{
 					Writer:          false,
@@ -336,7 +336,7 @@ func TestLockRpcServerRUnlock(t *testing.T) {
 		if !result {
 			t.Errorf("Expected %#v, got %#v", true, result)
 		} else {
-			gotLri, _ := locker.ll.lockMap["name"]
+			gotLri := locker.ll.lockMap["name"]
 			expectedLri := []lockRequesterInfo(nil)
 			if !testLockEquality(expectedLri, gotLri) {
 				t.Errorf("Expected %#v, got %#v", expectedLri, gotLri)
@@ -531,9 +531,6 @@ func TestLockServerInit(t *testing.T) {
 		globalIsDistXL = testCase.isDistXL
 		globalLockServer = nil
 		_, _ = newDsyncNodes(testCase.endpoints)
-		if err != nil {
-			t.Fatalf("Got unexpected error initializing lock servers: %v", err)
-		}
 		if globalLockServer == nil && testCase.isDistXL {
 			t.Errorf("Test %d: Expected initialized lock RPC receiver, but got uninitialized", i+1)
 		}

cmd/lock-stat.go

@@ -1,55 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-// lockStat - encapsulates total, blocked and granted lock counts.
-type lockStat struct {
-	total   int64
-	blocked int64
-	granted int64
-}
-
-// lockWaiting - updates lock stat when a lock becomes blocked.
-func (ls *lockStat) lockWaiting() {
-	ls.blocked++
-	ls.total++
-}
-
-// lockGranted - updates lock stat when a lock is granted.
-func (ls *lockStat) lockGranted() {
-	ls.blocked--
-	ls.granted++
-}
-
-// lockTimedOut - updates lock stat when a lock is timed out.
-func (ls *lockStat) lockTimedOut() {
-	ls.blocked--
-	ls.total--
-}
-
-// lockRemoved - updates lock stat when a lock is removed, by Unlock
-// or ForceUnlock.
-func (ls *lockStat) lockRemoved(granted bool) {
-	if granted {
-		ls.granted--
-		ls.total--
-
-	} else {
-		ls.blocked--
-		ls.total--
-	}
-}

cmd/logger/logonce.go

@@ -56,12 +56,11 @@ func (l *logOnceType) logOnceIf(ctx context.Context, err error, id interface{})
 // Cleanup the map every 30 minutes so that the log message is printed again for the user to notice.
 func (l *logOnceType) cleanupRoutine() {
 	for {
-		select {
-		case <-time.After(time.Minute * 30):
-			l.Lock()
-			l.IDMap = make(map[interface{}]error)
-			l.Unlock()
-		}
+		l.Lock()
+		l.IDMap = make(map[interface{}]error)
+		l.Unlock()
+
+		time.Sleep(30 * time.Minute)
 	}
 }
 

cmd/logger/target/http/http.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ * Minio Cloud Storage, (C) 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,9 +20,9 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
-	"io"
-	"io/ioutil"
 	gohttp "net/http"
+
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
 // Target implements logger.Target and sends the json
@@ -50,17 +50,18 @@ func (h *Target) startHTTPLogger() {
 			}
 
 			req, err := gohttp.NewRequest("POST", h.endpoint, bytes.NewBuffer(logJSON))
+			if err != nil {
+				continue
+			}
 			req.Header.Set("Content-Type", "application/json")
 
 			resp, err := h.client.Do(req)
 			if err != nil {
 				continue
 			}
-			if resp.Body != nil {
-				buf := make([]byte, 512)
-				io.CopyBuffer(ioutil.Discard, resp.Body, buf)
-				resp.Body.Close()
-			}
+
+			// Drain any response.
+			xhttp.DrainBody(resp.Body)
 		}
 	}()
 }

cmd/namespace-lock.go

@@ -102,7 +102,6 @@ func newNSLock(isDistXL bool) *nsLockMap {
 	nsMutex := nsLockMap{
 		isDistXL: isDistXL,
 		lockMap:  make(map[nsParam]*nsLock),
-		counters: &lockStat{},
 	}
 	return &nsMutex
 }
@@ -127,9 +126,6 @@ type nsLock struct {
 // nsLockMap - namespace lock map, provides primitives to Lock,
 // Unlock, RLock and RUnlock.
 type nsLockMap struct {
-	// Lock counter used for lock debugging.
-	counters *lockStat
-
 	// Indicates if namespace is part of a distributed setup.
 	isDistXL     bool
 	lockMap      map[nsParam]*nsLock
@@ -259,11 +255,8 @@ func (n *nsLockMap) ForceUnlock(volume, path string) {
 		dsync.NewDRWMutex(pathJoin(volume, path), globalDsync).ForceUnlock()
 	}
 
-	param := nsParam{volume, path}
-	if _, found := n.lockMap[param]; found {
-		// Remove lock from the map.
-		delete(n.lockMap, param)
-	}
+	// Remove lock from the map.
+	delete(n.lockMap, nsParam{volume, path})
 }
 
 // lockInstance - frontend/top-level interface for namespace locks.

cmd/namespace-lock_test.go

@@ -47,7 +47,6 @@ func TestNamespaceLockTest(t *testing.T) {
 		unlk             func(s1, s2, s3 string)
 		rlk              func(s1, s2, s3 string, t time.Duration) bool
 		runlk            func(s1, s2, s3 string)
-		lkCount          int
 		lockedRefCount   uint
 		unlockedRefCount uint
 		shouldPass       bool

cmd/notification.go

@@ -271,6 +271,9 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 		isDir:   false,
 		sys:     nil,
 	})
+	if zerr != nil {
+		return profilingDataFound
+	}
 
 	zwriter, zerr := zipWriter.CreateHeader(header)
 	if zerr != nil {
@@ -602,22 +605,19 @@ func (sys *NotificationSys) Init(objAPI ObjectLayer) error {
 	// the following reasons:
 	//  - Read quorum is lost just after the initialization
 	//    of the object layer.
-	retryTimerCh := newRetryTimerSimple(doneCh)
-	for {
-		select {
-		case _ = <-retryTimerCh:
-			if err := sys.refresh(objAPI); err != nil {
-				if err == errDiskNotFound ||
-					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
-					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
-					logger.Info("Waiting for notification subsystem to be initialized..")
-					continue
-				}
-				return err
+	for range newRetryTimerSimple(doneCh) {
+		if err := sys.refresh(objAPI); err != nil {
+			if err == errDiskNotFound ||
+				strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+				strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+				logger.Info("Waiting for notification subsystem to be initialized..")
+				continue
 			}
-			return nil
+			return err
 		}
+		break
 	}
+	return nil
 }
 
 // AddRulesMap - adds rules map for bucket name.

cmd/object-api-common.go

@@ -100,7 +100,7 @@ func newStorageAPI(endpoint Endpoint) (storage StorageAPI, err error) {
 		return newPosix(endpoint.Path)
 	}
 
-	return newStorageRESTClient(endpoint), nil
+	return newStorageRESTClient(endpoint)
 }
 
 // Cleanup a directory recursively.

cmd/object-api-deleteobject_test.go

@@ -93,7 +93,7 @@ func testDeleteObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 		for _, object := range testCase.objectToUploads {
 			md5Bytes := md5.Sum([]byte(object.content))
 			_, err = obj.PutObject(context.Background(), testCase.bucketName, object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content),
-				int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), nil, ObjectOptions{})
+				int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{})
 			if err != nil {
 				t.Fatalf("%s : %s", instanceType, err.Error())
 			}

cmd/object-api-getobject_test.go

@@ -75,7 +75,7 @@ func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	// iterate through the above set of inputs and upkoad the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{})
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), ObjectOptions{UserDefined: input.metaData})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -220,7 +220,7 @@ func testGetObjectPermissionDenied(obj ObjectLayer, instanceType string, disks [
 	// iterate through the above set of inputs and upkoad the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{})
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), ObjectOptions{UserDefined: input.metaData})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -333,7 +333,7 @@ func testGetObjectDiskNotFound(obj ObjectLayer, instanceType string, disks []str
 	// iterate through the above set of inputs and upkoad the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{})
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), ObjectOptions{UserDefined: input.metaData})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)

cmd/object-api-getobjectinfo_test.go

@@ -35,13 +35,13 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 	opts := ObjectOptions{}
-	_, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/asiapics.jpg", mustGetPutObjReader(t, bytes.NewBufferString("asiapics"), int64(len("asiapics")), "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/asiapics.jpg", mustGetPutObjReader(t, bytes.NewBufferString("asiapics"), int64(len("asiapics")), "", ""), opts)
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 
 	// Put an empty directory
-	_, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/empty-dir/", mustGetPutObjReader(t, bytes.NewBufferString(""), int64(len("")), "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/empty-dir/", mustGetPutObjReader(t, bytes.NewBufferString(""), int64(len("")), "", ""), opts)
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}

cmd/object-api-interface.go

@@ -29,6 +29,7 @@ import (
 // ObjectOptions represents object options for ObjectLayer operations
 type ObjectOptions struct {
 	ServerSideEncryption encrypt.ServerSide
+	UserDefined          map[string]string
 }
 
 // LockType represents required locking for ObjectLayer operations
@@ -65,13 +66,13 @@ type ObjectLayer interface {
 	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (reader *GetObjectReader, err error)
 	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
 	GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObject(ctx context.Context, bucket, object string) error
 
 	// Multipart operations.
 	ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error)
-	NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
+	NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error)
 	CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int,
 		startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (info PartInfo, err error)
 	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
@@ -82,8 +83,8 @@ type ObjectLayer interface {
 	// Healing operations.
 	ReloadFormat(ctx context.Context, dryRun bool) error
 	HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error)
-	HealBucket(ctx context.Context, bucket string, dryRun bool) ([]madmin.HealResultItem, error)
-	HealObject(ctx context.Context, bucket, object string, dryRun bool) (madmin.HealResultItem, error)
+	HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (madmin.HealResultItem, error)
+	HealObject(ctx context.Context, bucket, object string, dryRun, remove bool) (madmin.HealResultItem, error)
 	ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error)
 	ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error)
 

cmd/object-api-listobjects_test.go

@@ -71,7 +71,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	for _, object := range testObjects {
 		md5Bytes := md5.Sum([]byte(object.content))
 		_, err = obj.PutObject(context.Background(), testBuckets[0], object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content),
-			int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), object.meta, ObjectOptions{})
+			int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{UserDefined: object.meta})
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
@@ -625,7 +625,7 @@ func BenchmarkListObjects(b *testing.B) {
 	// Insert objects to be listed and benchmarked later.
 	for i := 0; i < 20000; i++ {
 		key := "obj" + strconv.Itoa(i)
-		_, err = obj.PutObject(context.Background(), bucket, key, mustGetPutObjReader(b, bytes.NewBufferString(key), int64(len(key)), "", ""), nil, ObjectOptions{})
+		_, err = obj.PutObject(context.Background(), bucket, key, mustGetPutObjReader(b, bytes.NewBufferString(key), int64(len(key)), "", ""), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}

cmd/object-api-multipart_test.go

@@ -41,14 +41,14 @@ func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestEr
 	bucket := "minio-bucket"
 	object := "minio-object"
 	opts := ObjectOptions{}
-	_, err := obj.NewMultipartUpload(context.Background(), "--", object, nil, opts)
+	_, err := obj.NewMultipartUpload(context.Background(), "--", object, opts)
 	if err == nil {
 		t.Fatalf("%s: Expected to fail since bucket name is invalid.", instanceType)
 	}
 
 	errMsg := "Bucket not found: minio-bucket"
 	// opearation expected to fail since the bucket on which NewMultipartUpload is being initiated doesn't exist.
-	_, err = obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts)
+	_, err = obj.NewMultipartUpload(context.Background(), bucket, object, opts)
 	if err == nil {
 		t.Fatalf("%s: Expected to fail since the NewMultipartUpload is intialized on a non-existent bucket.", instanceType)
 	}
@@ -63,12 +63,12 @@ func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestEr
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 
-	_, err = obj.NewMultipartUpload(context.Background(), bucket, "\\", nil, opts)
+	_, err = obj.NewMultipartUpload(context.Background(), bucket, "\\", opts)
 	if err == nil {
 		t.Fatalf("%s: Expected to fail since object name is invalid.", instanceType)
 	}
 
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, opts)
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
@@ -102,7 +102,7 @@ func testObjectAbortMultipartUpload(obj ObjectLayer, instanceType string, t Test
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, opts)
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
@@ -148,7 +148,7 @@ func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestE
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 
-	_, err = obj.NewMultipartUpload(context.Background(), bucket, object, nil, ObjectOptions{})
+	_, err = obj.NewMultipartUpload(context.Background(), bucket, object, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
@@ -184,7 +184,7 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [
 	}
 
 	// Initiate Multipart Upload on the above created bucket.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil, ObjectOptions{})
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], ObjectOptions{})
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -239,8 +239,8 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [
 	if err == nil {
 		t.Fatalf("Test %s: expected to fail but passed instead", instanceType)
 	}
-	// as majority of xl.json are not available, we expect InsufficientReadQuorum while trying to fetch the object quorum
-	expectedErr := InsufficientReadQuorum{}
+	// as majority of xl.json are not available, we expect uploadID to be not available.
+	expectedErr := InvalidUploadID{UploadID: testCase.uploadID}
 	if err.Error() != expectedErr.Error() {
 		t.Fatalf("Test %s: expected error %s, got %s instead.", instanceType, expectedErr, err)
 	}
@@ -264,7 +264,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 	// Initiate Multipart Upload on the above created bucket.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -400,7 +400,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 	// Initiate Multipart Upload on the above created bucket.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil, opts)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -420,7 +420,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 	for i := 0; i < 3; i++ {
 		// Initiate Multipart Upload on bucketNames[1] for the same object 3 times.
 		//  Used to test the listing for the case of multiple uploadID's for a given object.
-		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[1], objectNames[0], nil, opts)
+		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[1], objectNames[0], opts)
 		if err != nil {
 			// Failed to create NewMultipartUpload, abort.
 			t.Fatalf("%s : %s", instanceType, err.Error())
@@ -442,7 +442,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 	//  Used to test the listing for the case of multiple objects for a given bucket.
 	for i := 0; i < 6; i++ {
 		var uploadID string
-		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[2], objectNames[i], nil, opts)
+		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[2], objectNames[i], opts)
 		if err != nil {
 			// Failed to create NewMultipartUpload, abort.
 			t.Fatalf("%s : %s", instanceType, err.Error())
@@ -1275,7 +1275,7 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks
 	}
 	opts := ObjectOptions{}
 	// Initiate Multipart Upload on the above created bucket.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil, opts)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -1519,7 +1519,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 	// Initiate Multipart Upload on the above created bucket.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil, opts)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -1764,9 +1764,8 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err)
 	}
-	opts := ObjectOptions{}
 	// Initiate Multipart Upload on the above created bucket.
-	uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], map[string]string{"X-Amz-Meta-Id": "id"}, opts)
+	uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-Id": "id"}})
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("%s : %s", instanceType, err)
@@ -1799,6 +1798,7 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
 		{bucketNames[0], objectNames[0], uploadIDs[0], 6, string(validPart), validPartMD5, int64(len(string(validPart)))},
 	}
 	sha256sum := ""
+	var opts ObjectOptions
 	// Iterating over creatPartCases to generate multipart chunks.
 	for _, part := range parts {
 		_, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID, mustGetPutObjReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, sha256sum), opts)

cmd/object-api-putobject_test.go

@@ -172,7 +172,7 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
 	}
 
 	for i, testCase := range testCases {
-		objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetPutObjReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], testCase.inputSHA256), testCase.inputMeta, ObjectOptions{})
+		objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetPutObjReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], testCase.inputSHA256), ObjectOptions{UserDefined: testCase.inputMeta})
 		if actualErr != nil && testCase.expectedError == nil {
 			t.Errorf("Test %d: %s: Expected to pass, but failed with: error %s.", i+1, instanceType, actualErr.Error())
 		}
@@ -245,7 +245,7 @@ func testObjectAPIPutObjectDiskNotFound(obj ObjectLayer, instanceType string, di
 
 	sha256sum := ""
 	for i, testCase := range testCases {
-		objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetPutObjReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta, ObjectOptions{})
+		objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetPutObjReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), ObjectOptions{UserDefined: testCase.inputMeta})
 		if actualErr != nil && testCase.shouldPass {
 			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", i+1, instanceType, actualErr.Error())
 		}
@@ -294,7 +294,7 @@ func testObjectAPIPutObjectDiskNotFound(obj ObjectLayer, instanceType string, di
 		InsufficientWriteQuorum{},
 	}
 
-	_, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetPutObjReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta, ObjectOptions{})
+	_, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetPutObjReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), ObjectOptions{UserDefined: testCase.inputMeta})
 	if actualErr != nil && testCase.shouldPass {
 		t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", len(testCases)+1, instanceType, actualErr.Error())
 	}
@@ -326,7 +326,7 @@ func testObjectAPIPutObjectStaleFiles(obj ObjectLayer, instanceType string, disk
 
 	data := []byte("hello, world")
 	// Create object.
-	_, err = obj.PutObject(context.Background(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{})
 	if err != nil {
 		// Failed to create object, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -359,7 +359,7 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str
 	}
 	opts := ObjectOptions{}
 	// Initiate Multipart Upload on the above created bucket.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())

cmd/object-handlers-common.go

@@ -33,8 +33,8 @@ import (
 //  x-amz-copy-source-if-unmodified-since
 //  x-amz-copy-source-if-match
 //  x-amz-copy-source-if-none-match
-func checkCopyObjectPartPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool {
-	return checkCopyObjectPreconditions(w, r, objInfo)
+func checkCopyObjectPartPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool {
+	return checkCopyObjectPreconditions(ctx, w, r, objInfo)
 }
 
 // Validates the preconditions for CopyObject, returns true if CopyObject operation should not proceed.
@@ -43,7 +43,7 @@ func checkCopyObjectPartPreconditions(w http.ResponseWriter, r *http.Request, ob
 //  x-amz-copy-source-if-unmodified-since
 //  x-amz-copy-source-if-match
 //  x-amz-copy-source-if-none-match
-func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool {
+func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool {
 	// Return false for methods other than GET and HEAD.
 	if r.Method != "PUT" {
 		return false
@@ -75,7 +75,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
 			if !ifModifiedSince(objInfo.ModTime, givenTime) {
 				// If the object is not modified since the specified time.
 				writeHeaders()
-				writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL, guessIsBrowserReq(r))
 				return true
 			}
 		}
@@ -89,7 +89,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
 			if ifModifiedSince(objInfo.ModTime, givenTime) {
 				// If the object is modified since the specified time.
 				writeHeaders()
-				writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL, guessIsBrowserReq(r))
 				return true
 			}
 		}
@@ -102,7 +102,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
 		if objInfo.ETag != "" && !isETagEqual(objInfo.ETag, ifMatchETagHeader) {
 			// If the object ETag does not match with the specified ETag.
 			writeHeaders()
-			writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL, guessIsBrowserReq(r))
 			return true
 		}
 	}
@@ -114,7 +114,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
 		if objInfo.ETag != "" && isETagEqual(objInfo.ETag, ifNoneMatchETagHeader) {
 			// If the object ETag matches with the specified ETag.
 			writeHeaders()
-			writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL, guessIsBrowserReq(r))
 			return true
 		}
 	}
@@ -128,7 +128,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
 //  If-Unmodified-Since
 //  If-Match
 //  If-None-Match
-func checkPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool {
+func checkPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool {
 	// Return false for methods other than GET and HEAD.
 	if r.Method != "GET" && r.Method != "HEAD" {
 		return false
@@ -174,7 +174,7 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectIn
 			if ifModifiedSince(objInfo.ModTime, givenTime) {
 				// If the object is modified since the specified time.
 				writeHeaders()
-				writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL, guessIsBrowserReq(r))
 				return true
 			}
 		}
@@ -187,7 +187,7 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectIn
 		if !isETagEqual(objInfo.ETag, ifMatchETagHeader) {
 			// If the object ETag does not match with the specified ETag.
 			writeHeaders()
-			writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL, guessIsBrowserReq(r))
 			return true
 		}
 	}
@@ -211,10 +211,7 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectIn
 func ifModifiedSince(objTime time.Time, givenTime time.Time) bool {
 	// The Date-Modified header truncates sub-second precision, so
 	// use mtime < t+1s instead of mtime <= t to check for unmodified.
-	if objTime.After(givenTime.Add(1 * time.Second)) {
-		return true
-	}
-	return false
+	return objTime.After(givenTime.Add(1 * time.Second))
 }
 
 // canonicalizeETag returns ETag with leading and trailing double-quotes removed,

cmd/object-handlers_test.go

@@ -85,7 +85,7 @@ func testAPIHeadObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 	// iterate through the above set of inputs and upload the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, ObjectOptions{})
+		_, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), ObjectOptions{UserDefined: input.metaData})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -357,13 +357,15 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 	// iterate through the above set of inputs and upload the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, ObjectOptions{})
+		_, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), ObjectOptions{UserDefined: input.metaData})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
 		}
 	}
 
+	ctx := context.Background()
+
 	// test cases with inputs and expected result for GetObject.
 	testCases := []struct {
 		bucketName string
@@ -396,7 +398,9 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			accessKey:  credentials.AccessKey,
 			secretKey:  credentials.SecretKey,
 
-			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey), getGetObjectURL("", bucketName, "abcd"), "")),
+			expectedContent: encodeResponse(getAPIErrorResponse(ctx,
+				getAPIError(ErrNoSuchKey),
+				getGetObjectURL("", bucketName, "abcd"), "", "")),
 			expectedRespStatus: http.StatusNotFound,
 		},
 		// Test case - 3.
@@ -420,7 +424,9 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			accessKey:  credentials.AccessKey,
 			secretKey:  credentials.SecretKey,
 
-			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidRange), getGetObjectURL("", bucketName, objectName), "")),
+			expectedContent: encodeResponse(getAPIErrorResponse(ctx,
+				getAPIError(ErrInvalidRange),
+				getGetObjectURL("", bucketName, objectName), "", "")),
 			expectedRespStatus: http.StatusRequestedRangeNotSatisfiable,
 		},
 		// Test case - 5.
@@ -446,7 +452,9 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			accessKey:  "Invalid-AccessID",
 			secretKey:  credentials.SecretKey,
 
-			expectedContent:    encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidAccessKeyID), getGetObjectURL("", bucketName, objectName), "")),
+			expectedContent: encodeResponse(getAPIErrorResponse(ctx,
+				getAPIError(ErrInvalidAccessKeyID),
+				getGetObjectURL("", bucketName, objectName), "", "")),
 			expectedRespStatus: http.StatusForbidden,
 		},
 		// Test case - 7.
@@ -458,8 +466,9 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			accessKey:  credentials.AccessKey,
 			secretKey:  credentials.SecretKey,
 
-			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidObjectName),
-				getGetObjectURL("", bucketName, "../../etc"), "")),
+			expectedContent: encodeResponse(getAPIErrorResponse(ctx,
+				getAPIError(ErrInvalidObjectName),
+				getGetObjectURL("", bucketName, "../../etc"), "", "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 8.
@@ -471,8 +480,9 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			accessKey:  credentials.AccessKey,
 			secretKey:  credentials.SecretKey,
 
-			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey),
-				"/"+bucketName+"/"+". ./. ./etc", "")),
+			expectedContent: encodeResponse(getAPIErrorResponse(ctx,
+				getAPIError(ErrNoSuchKey),
+				"/"+bucketName+"/"+". ./. ./etc", "", "")),
 			expectedRespStatus: http.StatusNotFound,
 		},
 		// Test case - 9.
@@ -484,8 +494,9 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			accessKey:  credentials.AccessKey,
 			secretKey:  credentials.SecretKey,
 
-			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidObjectName),
-				"/"+bucketName+"/"+". ./../etc", "")),
+			expectedContent: encodeResponse(getAPIErrorResponse(ctx,
+				getAPIError(ErrInvalidObjectName),
+				"/"+bucketName+"/"+". ./../etc", "", "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 10.
@@ -497,8 +508,10 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			accessKey:  credentials.AccessKey,
 			secretKey:  credentials.SecretKey,
 
-			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey),
-				getGetObjectURL("", bucketName, "etc/path/proper/.../etc"), "")),
+			expectedContent: encodeResponse(getAPIErrorResponse(ctx,
+				getAPIError(ErrNoSuchKey),
+				getGetObjectURL("", bucketName, "etc/path/proper/.../etc"),
+				"", "")),
 			expectedRespStatus: http.StatusNotFound,
 		},
 	}
@@ -527,11 +540,28 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 		// read the response body.
 		actualContent, err := ioutil.ReadAll(rec.Body)
 		if err != nil {
+			t.Fatalf("Test %d: %s: Failed reading response body: <ERROR> %v", i+1, instanceType, err)
+		}
+
+		if rec.Code == http.StatusOK || rec.Code == http.StatusPartialContent {
+			if !bytes.Equal(testCase.expectedContent, actualContent) {
+				t.Errorf("Test %d: %s: Object content differs from expected value %s, got %s", i+1, instanceType, testCase.expectedContent, string(actualContent))
+			}
+			continue
+		}
+
+		// Verify whether the bucket obtained object is same as the one created.
+		actualError := &APIErrorResponse{}
+		if err = xml.Unmarshal(actualContent, actualError); err != nil {
 			t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
 		}
-		// Verify whether the bucket obtained object is same as the one created.
-		if !bytes.Equal(testCase.expectedContent, actualContent) {
-			t.Errorf("Test %d: %s: Object content differs from expected value %s, got %s", i+1, instanceType, testCase.expectedContent, string(actualContent))
+
+		if actualError.BucketName != testCase.bucketName {
+			t.Fatalf("Test %d: %s: Unexpected bucket name, expected %s, got %s", i+1, instanceType, testCase.bucketName, actualError.BucketName)
+		}
+
+		if actualError.Key != testCase.objectName {
+			t.Fatalf("Test %d: %s: Unexpected object name, expected %s, got %s", i+1, instanceType, testCase.objectName, actualError.Key)
 		}
 
 		// Verify response of the V2 signed HTTP request.
@@ -559,11 +589,28 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 		// read the response body.
 		actualContent, err = ioutil.ReadAll(recV2.Body)
 		if err != nil {
+			t.Fatalf("Test %d: %s: Failed to read response body: <ERROR> %v", i+1, instanceType, err)
+		}
+
+		if rec.Code == http.StatusOK || rec.Code == http.StatusPartialContent {
+			// Verify whether the bucket obtained object is same as the one created.
+			if !bytes.Equal(testCase.expectedContent, actualContent) {
+				t.Errorf("Test %d: %s: Object content differs from expected value.", i+1, instanceType)
+			}
+			continue
+		}
+
+		actualError = &APIErrorResponse{}
+		if err = xml.Unmarshal(actualContent, actualError); err != nil {
 			t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
 		}
-		// Verify whether the bucket obtained object is same as the one created.
-		if !bytes.Equal(testCase.expectedContent, actualContent) {
-			t.Errorf("Test %d: %s: Object content differs from expected value.", i+1, instanceType)
+
+		if actualError.BucketName != testCase.bucketName {
+			t.Fatalf("Test %d: %s: Unexpected bucket name, expected %s, got %s", i+1, instanceType, testCase.bucketName, actualError.BucketName)
+		}
+
+		if actualError.Key != testCase.objectName {
+			t.Fatalf("Test %d: %s: Unexpected object name, expected %s, got %s", i+1, instanceType, testCase.objectName, actualError.Key)
 		}
 	}
 
@@ -855,6 +902,7 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
 			shouldPass:         true,
+			fault:              None,
 		},
 		// Test case - 2
 		// Small chunk size.
@@ -869,6 +917,7 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
 			shouldPass:         true,
+			fault:              None,
 		},
 		// Test case - 3
 		// Empty data
@@ -897,6 +946,7 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 			accessKey:          "",
 			secretKey:          "",
 			shouldPass:         false,
+			fault:              None,
 		},
 		// Test case - 5
 		// Wrong auth header returns as bad request.
@@ -912,6 +962,7 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 			secretKey:          credentials.SecretKey,
 			shouldPass:         false,
 			removeAuthHeader:   true,
+			fault:              None,
 		},
 		// Test case - 6
 		// Large chunk size.. also passes.
@@ -926,6 +977,7 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
 			shouldPass:         true,
+			fault:              None,
 		},
 		// Test case - 7
 		// Chunk with malformed encoding.
@@ -1017,6 +1069,7 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 			secretKey:          credentials.SecretKey,
 			shouldPass:         true,
 			contentEncoding:    "aws-chunked,gzip",
+			fault:              None,
 		},
 	}
 	// Iterating over the cases, fetching the object validating the response.
@@ -1393,7 +1446,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 	for i, input := range putObjectInputs {
 		// uploading the object.
 		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName,
-			mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
+			mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), ObjectOptions{UserDefined: input.metaData})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -1406,7 +1459,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 	// PutObjectPart API HTTP Handler has to be tested in isolation,
 	// that is without any other handler being registered,
 	// That's why NewMultipartUpload is initiated using ObjectLayer.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil, opts)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("Minio %s : <ERROR>  %s", instanceType, err)
@@ -1503,7 +1556,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 	// iterate through the above set of inputs and upload the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), ObjectOptions{UserDefined: input.metaData})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -1516,7 +1569,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 	// PutObjectPart API HTTP Handler has to be tested in isolation,
 	// that is without any other handler being registered,
 	// That's why NewMultipartUpload is initiated using ObjectLayer.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil, opts)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("Minio %s : <ERROR>  %s", instanceType, err)
@@ -1738,7 +1791,6 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 
 	for i, testCase := range testCases {
 		var req *http.Request
-		var reqV2 *http.Request
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
 		if !testCase.invalidPartNumber || !testCase.maximumPartNumber {
@@ -1782,37 +1834,6 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 				t.Fatalf("Test %d: %s: Expected only one entry returned %d entries", i+1, instanceType, len(results.Parts))
 			}
 		}
-
-		// Verify response of the V2 signed HTTP request.
-		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
-		recV2 := httptest.NewRecorder()
-
-		reqV2, err = newTestRequest("PUT", getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "1"), 0, nil)
-		if err != nil {
-			t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i+1, err)
-		}
-		// "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied.
-		if testCase.copySourceHeader != "" {
-			reqV2.Header.Set("X-Amz-Copy-Source", testCase.copySourceHeader)
-		}
-		if testCase.copySourceVersionID != "" {
-			reqV2.Header.Set("X-Amz-Copy-Source-Version-Id", testCase.copySourceVersionID)
-		}
-		if testCase.copySourceRange != "" {
-			reqV2.Header.Set("X-Amz-Copy-Source-Range", testCase.copySourceRange)
-		}
-
-		err = signRequestV2(reqV2, testCase.accessKey, testCase.secretKey)
-		if err != nil {
-			t.Fatalf("Failed to V2 Sign the HTTP request: %v.", err)
-		}
-
-		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
-		// Call the ServeHTTP to execute the handler.
-		apiRouter.ServeHTTP(recV2, reqV2)
-		if recV2.Code != testCase.expectedRespStatus {
-			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code)
-		}
 	}
 
 	// HTTP request for testing when `ObjectLayer` is set to `nil`.
@@ -1895,7 +1916,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 	for i, input := range putObjectInputs {
 		// uploading the object.
 		var objInfo ObjectInfo
-		objInfo, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
+		objInfo, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), ObjectOptions{UserDefined: input.metaData})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -2535,7 +2556,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 
 	for i := 0; i < 2; i++ {
 		// initiate new multipart uploadID.
-		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil, opts)
+		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketName, objectName, opts)
 		if err != nil {
 			// Failed to create NewMultipartUpload, abort.
 			t.Fatalf("Minio %s : <ERROR>  %s", instanceType, err)
@@ -2672,8 +2693,9 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			accessKey: credentials.AccessKey,
 			secretKey: credentials.SecretKey,
 
-			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(ctx, InvalidPart{})),
-				getGetObjectURL("", bucketName, objectName), "")),
+			expectedContent: encodeResponse(getAPIErrorResponse(ctx,
+				toAPIError(ctx, InvalidPart{}),
+				getGetObjectURL("", bucketName, objectName), "", "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 2.
@@ -2687,8 +2709,9 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			accessKey: credentials.AccessKey,
 			secretKey: credentials.SecretKey,
 
-			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrMalformedXML),
-				getGetObjectURL("", bucketName, objectName), "")),
+			expectedContent: encodeResponse(getAPIErrorResponse(ctx,
+				getAPIError(ErrMalformedXML),
+				getGetObjectURL("", bucketName, objectName), "", "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 3.
@@ -2702,8 +2725,9 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			accessKey: credentials.AccessKey,
 			secretKey: credentials.SecretKey,
 
-			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(ctx, InvalidUploadID{UploadID: "abc"})),
-				getGetObjectURL("", bucketName, objectName), "")),
+			expectedContent: encodeResponse(getAPIErrorResponse(ctx,
+				toAPIError(ctx, InvalidUploadID{UploadID: "abc"}),
+				getGetObjectURL("", bucketName, objectName), "", "")),
 			expectedRespStatus: http.StatusNotFound,
 		},
 		// Test case - 4.
@@ -2717,8 +2741,8 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			secretKey: credentials.SecretKey,
 
 			expectedContent: encodeResponse(completeMultipartAPIError{int64(4), int64(5242880), 1, "e2fc714c4727ee9395f324cd2e7f331f",
-				getAPIErrorResponse(getAPIError(toAPIErrorCode(ctx, PartTooSmall{PartNumber: 1})),
-					getGetObjectURL("", bucketName, objectName), "")}),
+				getAPIErrorResponse(ctx, toAPIError(ctx, PartTooSmall{PartNumber: 1}),
+					getGetObjectURL("", bucketName, objectName), "", "")}),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 5.
@@ -2731,8 +2755,9 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			accessKey: credentials.AccessKey,
 			secretKey: credentials.SecretKey,
 
-			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(ctx, InvalidPart{})),
-				getGetObjectURL("", bucketName, objectName), "")),
+			expectedContent: encodeResponse(getAPIErrorResponse(ctx,
+				toAPIError(ctx, InvalidPart{}),
+				getGetObjectURL("", bucketName, objectName), "", "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 6.
@@ -2746,8 +2771,9 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			accessKey: credentials.AccessKey,
 			secretKey: credentials.SecretKey,
 
-			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidPartOrder),
-				getGetObjectURL("", bucketName, objectName), "")),
+			expectedContent: encodeResponse(getAPIErrorResponse(ctx,
+				getAPIError(ErrInvalidPartOrder),
+				getGetObjectURL("", bucketName, objectName), "", "")),
 			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 7.
@@ -2761,8 +2787,9 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 			accessKey: "Invalid-AccessID",
 			secretKey: credentials.SecretKey,
 
-			expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidAccessKeyID),
-				getGetObjectURL("", bucketName, objectName), "")),
+			expectedContent: encodeResponse(getAPIErrorResponse(ctx,
+				getAPIError(ErrInvalidAccessKeyID),
+				getGetObjectURL("", bucketName, objectName), "", "")),
 			expectedRespStatus: http.StatusForbidden,
 		},
 		// Test case - 8.
@@ -2814,11 +2841,27 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 		if err != nil {
 			t.Fatalf("Test %d : Minio %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
 		}
-		// Verify whether the bucket obtained object is same as the one created.
-		if !bytes.Equal(testCase.expectedContent, actualContent) {
-			t.Errorf("Test %d : Minio %s: Object content differs from expected value.", i+1, instanceType)
+
+		if rec.Code == http.StatusOK {
+			// Verify whether the bucket obtained object is same as the one created.
+			if !bytes.Equal(testCase.expectedContent, actualContent) {
+				t.Errorf("Test %d : Minio %s: Object content differs from expected value.", i+1, instanceType)
+			}
+			continue
 		}
 
+		actualError := &APIErrorResponse{}
+		if err = xml.Unmarshal(actualContent, actualError); err != nil {
+			t.Errorf("Minio %s: error response failed to parse error XML", instanceType)
+		}
+
+		if actualError.BucketName != bucketName {
+			t.Errorf("Minio %s: error response bucket name differs from expected value", instanceType)
+		}
+
+		if actualError.Key != objectName {
+			t.Errorf("Minio %s: error response object name differs from expected value", instanceType)
+		}
 	}
 
 	// Testing for anonymous API request.
@@ -2886,7 +2929,7 @@ func testAPIAbortMultipartHandler(obj ObjectLayer, instanceType, bucketName stri
 
 	for i := 0; i < 2; i++ {
 		// initiate new multipart uploadID.
-		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil, opts)
+		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketName, objectName, opts)
 		if err != nil {
 			// Failed to create NewMultipartUpload, abort.
 			t.Fatalf("Minio %s : <ERROR>  %s", instanceType, err)
@@ -3038,7 +3081,6 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
 	credentials auth.Credentials, t *testing.T) {
 
 	var err error
-	var opts ObjectOptions
 	objectName := "test-object"
 	// Object used for anonymous API request test.
 	anonObjectName := "test-anon-obj"
@@ -3066,7 +3108,7 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
 	// iterate through the above set of inputs and upload the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), ObjectOptions{UserDefined: input.metaData})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -3189,73 +3231,6 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
 	ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq)
 }
 
-// TestAPIPutObjectPartHandlerPreSign - Tests validate the response of PutObjectPart HTTP handler
-// when the request signature type is PreSign.
-func TestAPIPutObjectPartHandlerPreSign(t *testing.T) {
-	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIPutObjectPartHandlerPreSign, []string{"NewMultipart", "PutObjectPart"})
-}
-
-func testAPIPutObjectPartHandlerPreSign(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
-	testObject := "testobject"
-	rec := httptest.NewRecorder()
-	req, err := newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, "testobject"),
-		0, nil, credentials.AccessKey, credentials.SecretKey, nil)
-	if err != nil {
-		t.Fatalf("[%s] - Failed to create a signed request to initiate multipart upload for %s/%s: <ERROR> %v",
-			instanceType, bucketName, testObject, err)
-	}
-	apiRouter.ServeHTTP(rec, req)
-
-	// Get uploadID of the mulitpart upload initiated.
-	var mpartResp InitiateMultipartUploadResponse
-	mpartRespBytes, err := ioutil.ReadAll(rec.Result().Body)
-	if err != nil {
-		t.Fatalf("[%s] Failed to read NewMultipartUpload response <ERROR> %v", instanceType, err)
-
-	}
-	err = xml.Unmarshal(mpartRespBytes, &mpartResp)
-	if err != nil {
-		t.Fatalf("[%s] Failed to unmarshal NewMultipartUpload response <ERROR> %v", instanceType, err)
-	}
-
-	rec = httptest.NewRecorder()
-	req, err = newTestRequest("PUT", getPutObjectPartURL("", bucketName, testObject, mpartResp.UploadID, "1"),
-		int64(len("hello")), bytes.NewReader([]byte("hello")))
-	if err != nil {
-		t.Fatalf("[%s] - Failed to create an unsigned request to put object part for %s/%s <ERROR> %v",
-			instanceType, bucketName, testObject, err)
-	}
-
-	err = preSignV2(req, credentials.AccessKey, credentials.SecretKey, int64(10*60*60))
-	if err != nil {
-		t.Fatalf("[%s] - Failed to presign an unsigned request to put object part for %s/%s <ERROR> %v",
-			instanceType, bucketName, testObject, err)
-	}
-	apiRouter.ServeHTTP(rec, req)
-	if rec.Code != http.StatusOK {
-		t.Errorf("Test %d %s expected to succeed but failed with HTTP status code %d", 1, instanceType, rec.Code)
-	}
-
-	rec = httptest.NewRecorder()
-	req, err = newTestRequest("PUT", getPutObjectPartURL("", bucketName, testObject, mpartResp.UploadID, "1"),
-		int64(len("hello")), bytes.NewReader([]byte("hello")))
-	if err != nil {
-		t.Fatalf("[%s] - Failed to create an unsigned request to put object part for %s/%s <ERROR> %v",
-			instanceType, bucketName, testObject, err)
-	}
-	err = preSignV4(req, credentials.AccessKey, credentials.SecretKey, int64(10*60*60))
-	if err != nil {
-		t.Fatalf("[%s] - Failed to presign an unsigned request to put object part for %s/%s <ERROR> %v",
-			instanceType, bucketName, testObject, err)
-	}
-	apiRouter.ServeHTTP(rec, req)
-	if rec.Code != http.StatusOK {
-		t.Errorf("Test %d %s expected to succeed but failed with HTTP status code %d", 1, instanceType, rec.Code)
-	}
-}
-
 // TestAPIPutObjectPartHandlerStreaming - Tests validate the response of PutObjectPart HTTP handler
 // when the request signature type is `streaming signature`.
 func TestAPIPutObjectPartHandlerStreaming(t *testing.T) {
@@ -3360,7 +3335,7 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin
 	// PutObjectPart API HTTP Handler has to be tested in isolation,
 	// that is without any other handler being registered,
 	// That's why NewMultipartUpload is initiated using ObjectLayer.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil, opts)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("Minio %s : <ERROR>  %s", instanceType, err)
@@ -3763,7 +3738,7 @@ func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName str
 	// PutObjectPart API HTTP Handler has to be tested in isolation,
 	// that is without any other handler being registered,
 	// That's why NewMultipartUpload is initiated using ObjectLayer.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil, opts)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("Minio %s : <ERROR>  %s", instanceType, err)

cmd/object_api_suite_test.go

@@ -95,7 +95,7 @@ func testMultipartObjectCreation(obj ObjectLayer, instanceType string, t TestErr
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
-	uploadID, err := obj.NewMultipartUpload(context.Background(), "bucket", "key", nil, opts)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), "bucket", "key", opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -139,7 +139,7 @@ func testMultipartObjectAbort(obj ObjectLayer, instanceType string, t TestErrHan
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
-	uploadID, err := obj.NewMultipartUpload(context.Background(), "bucket", "key", nil, opts)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), "bucket", "key", opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -180,8 +180,8 @@ func (s *ObjectLayerAPISuite) TestMultipleObjectCreation(t *testing.T) {
 // Tests validate object creation.
 func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	objects := make(map[string][]byte)
-	err := obj.MakeBucketWithLocation(context.Background(), "bucket", "")
 	var opts ObjectOptions
+	err := obj.MakeBucketWithLocation(context.Background(), "bucket", "")
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -199,7 +199,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH
 		metadata := make(map[string]string)
 		metadata["etag"] = expectedETaghex
 		var objInfo ObjectInfo
-		objInfo, err = obj.PutObject(context.Background(), "bucket", key, mustGetPutObjReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), metadata["etag"], ""), metadata, opts)
+		objInfo, err = obj.PutObject(context.Background(), "bucket", key, mustGetPutObjReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), metadata["etag"], ""), ObjectOptions{UserDefined: metadata})
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -253,7 +253,7 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	// check before paging occurs.
 	for i := 0; i < 5; i++ {
 		key := "obj" + strconv.Itoa(i)
-		_, err = obj.PutObject(context.Background(), "bucket", key, mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
+		_, err = obj.PutObject(context.Background(), "bucket", key, mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -273,7 +273,7 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	// check after paging occurs pages work.
 	for i := 6; i <= 10; i++ {
 		key := "obj" + strconv.Itoa(i)
-		_, err = obj.PutObject(context.Background(), "bucket", key, mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
+		_, err = obj.PutObject(context.Background(), "bucket", key, mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -290,11 +290,11 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 	// check paging with prefix at end returns less objects.
 	{
-		_, err = obj.PutObject(context.Background(), "bucket", "newPrefix", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
+		_, err = obj.PutObject(context.Background(), "bucket", "newPrefix", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
-		_, err = obj.PutObject(context.Background(), "bucket", "newPrefix2", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
+		_, err = obj.PutObject(context.Background(), "bucket", "newPrefix2", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -332,11 +332,11 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 
 	// check delimited results with delimiter and prefix.
 	{
-		_, err = obj.PutObject(context.Background(), "bucket", "this/is/delimited", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
+		_, err = obj.PutObject(context.Background(), "bucket", "this/is/delimited", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
-		_, err = obj.PutObject(context.Background(), "bucket", "this/is/also/a/delimited/file", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
+		_, err = obj.PutObject(context.Background(), "bucket", "this/is/also/a/delimited/file", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -448,14 +448,14 @@ func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, t TestErrHan
 	var opts ObjectOptions
 	uploadContent := "The list of parts was not in ascending order. The parts list must be specified in order by part number."
 	length := int64(len(uploadContent))
-	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), length, "", ""), opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
 
 	uploadContent = "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
 	length = int64(len(uploadContent))
-	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), length, "", ""), opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -465,7 +465,7 @@ func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, t TestErrHan
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
-	if string(bytesBuffer.Bytes()) != "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed." {
+	if bytesBuffer.String() != "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed." {
 		t.Errorf("%s: Invalid upload ID error mismatch.", instanceType)
 	}
 }
@@ -478,7 +478,7 @@ func (s *ObjectLayerAPISuite) TestNonExistantBucketOperations(t *testing.T) {
 // Tests validate that bucket operation on non-existent bucket fails.
 func testNonExistantBucketOperations(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	var opts ObjectOptions
-	_, err := obj.PutObject(context.Background(), "bucket1", "object", mustGetPutObjReader(t, bytes.NewBufferString("one"), int64(len("one")), "", ""), nil, opts)
+	_, err := obj.PutObject(context.Background(), "bucket1", "object", mustGetPutObjReader(t, bytes.NewBufferString("one"), int64(len("one")), "", ""), opts)
 	if err == nil {
 		t.Fatal("Expected error but found nil")
 	}
@@ -526,7 +526,7 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 
 	var bytesBuffer1 bytes.Buffer
 	var opts ObjectOptions
-	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, readerEOF, length, "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, readerEOF, length, "", ""), opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -539,7 +539,7 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 
 	var bytesBuffer2 bytes.Buffer
-	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, readerNoEOF, length, "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, readerNoEOF, length, "", ""), opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -568,7 +568,7 @@ func testPutObjectInSubdir(obj ObjectLayer, instanceType string, t TestErrHandle
 	uploadContent := `The specified multipart upload does not exist. The upload ID might be invalid, or the multipart
  upload might have been aborted or completed.`
 	length := int64(len(uploadContent))
-	_, err = obj.PutObject(context.Background(), "bucket", "dir1/dir2/object", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "bucket", "dir1/dir2/object", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), length, "", ""), opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -741,7 +741,7 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string,
 	content := "One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag."
 	length := int64(len(content))
 	var opts ObjectOptions
-	_, err = obj.PutObject(context.Background(), bucketName, "dir1/dir3/object", mustGetPutObjReader(t, bytes.NewBufferString(content), length, "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), bucketName, "dir1/dir3/object", mustGetPutObjReader(t, bytes.NewBufferString(content), length, "", ""), opts)
 
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
@@ -783,7 +783,7 @@ func testContentType(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	var opts ObjectOptions
 	uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
 	// Test empty.
-	_, err = obj.PutObject(context.Background(), "bucket", "minio.png", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
+	_, err = obj.PutObject(context.Background(), "bucket", "minio.png", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}

cmd/policy.go

@@ -158,23 +158,20 @@ func (sys *PolicySys) Init(objAPI ObjectLayer) error {
 	// the following reasons:
 	//  - Read quorum is lost just after the initialization
 	//    of the object layer.
-	retryTimerCh := newRetryTimerSimple(doneCh)
-	for {
-		select {
-		case _ = <-retryTimerCh:
-			// Load PolicySys once during boot.
-			if err := sys.refresh(objAPI); err != nil {
-				if err == errDiskNotFound ||
-					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
-					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
-					logger.Info("Waiting for policy subsystem to be initialized..")
-					continue
-				}
-				return err
+	for range newRetryTimerSimple(doneCh) {
+		// Load PolicySys once during boot.
+		if err := sys.refresh(objAPI); err != nil {
+			if err == errDiskNotFound ||
+				strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+				strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+				logger.Info("Waiting for policy subsystem to be initialized..")
+				continue
 			}
-			return nil
+			return err
 		}
+		break
 	}
+	return nil
 }
 
 // NewPolicySys - creates new policy system.

cmd/posix.go

@@ -73,6 +73,7 @@ type posix struct {
 	diskMount bool // indicates if the path is an actual mount.
 	driveSync bool // indicates if the backend is synchronous.
 
+	diskFileInfo os.FileInfo
 	// Disk usage metrics
 	stopUsageCh chan struct{}
 }
@@ -177,7 +178,10 @@ func newPosix(path string) (*posix, error) {
 	if path, err = getValidPath(path); err != nil {
 		return nil, err
 	}
-
+	fi, err := os.Stat(path)
+	if err != nil {
+		return nil, err
+	}
 	p := &posix{
 		connected: true,
 		diskPath:  path,
@@ -188,8 +192,9 @@ func newPosix(path string) (*posix, error) {
 				return &b
 			},
 		},
-		stopUsageCh: make(chan struct{}),
-		diskMount:   mountinfo.IsLikelyMountPoint(path),
+		stopUsageCh:  make(chan struct{}),
+		diskFileInfo: fi,
+		diskMount:    mountinfo.IsLikelyMountPoint(path),
 	}
 
 	var pf BoolFlag
@@ -351,7 +356,7 @@ func (s *posix) checkDiskFound() (err error) {
 	if !s.IsOnline() {
 		return errDiskNotFound
 	}
-	_, err = os.Stat(s.diskPath)
+	fi, err := os.Stat(s.diskPath)
 	if err != nil {
 		switch {
 		case os.IsNotExist(err):
@@ -364,6 +369,10 @@ func (s *posix) checkDiskFound() (err error) {
 			return err
 		}
 	}
+	if !os.SameFile(s.diskFileInfo, fi) {
+		s.connected = false
+		return errDiskNotFound
+	}
 	return nil
 }
 
@@ -860,7 +869,7 @@ func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verif
 		return 0, err
 	}
 
-	if bytes.Compare(h.Sum(nil), verifier.sum) != 0 {
+	if !bytes.Equal(h.Sum(nil), verifier.sum) {
 		return 0, hashMismatchError{hex.EncodeToString(verifier.sum), hex.EncodeToString(h.Sum(nil))}
 	}
 
@@ -1105,8 +1114,9 @@ func (s *posix) WriteAll(volume, path string, buf []byte) (err error) {
 		return errFaultyDisk
 	}
 
-	// Create file if not found
-	w, err := s.openFile(volume, path, os.O_CREATE|os.O_SYNC|os.O_WRONLY)
+	// Create file if not found. Note that it is created with os.O_EXCL flag as the file
+	// always is supposed to be created in the tmp directory with a unique file name.
+	w, err := s.openFile(volume, path, os.O_CREATE|os.O_SYNC|os.O_WRONLY|os.O_EXCL)
 	if err != nil {
 		return err
 	}

cmd/posix_test.go

@@ -188,7 +188,7 @@ func TestPosixIsDirEmpty(t *testing.T) {
 
 	// Should give false on non-existent directory.
 	dir1 := slashpath.Join(tmp, "non-existent-directory")
-	if isDirEmpty(dir1) != false {
+	if isDirEmpty(dir1) {
 		t.Error("expected false for non-existent directory, got true")
 	}
 
@@ -199,7 +199,7 @@ func TestPosixIsDirEmpty(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if isDirEmpty(dir2) != false {
+	if isDirEmpty(dir2) {
 		t.Error("expected false for a file, got true")
 	}
 
@@ -210,7 +210,7 @@ func TestPosixIsDirEmpty(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if isDirEmpty(dir3) != true {
+	if !isDirEmpty(dir3) {
 		t.Error("expected true for empty dir, got false")
 	}
 }

cmd/rest/client.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	xhttp "github.com/minio/minio/cmd/http"
+	"golang.org/x/net/http2"
 )
 
 // DefaultRESTTimeout - default RPC timeout is one minute.
@@ -59,9 +60,9 @@ func (c *Client) Call(method string, values url.Values, body io.Reader, length i
 	}
 
 	if resp.StatusCode != http.StatusOK {
+		defer xhttp.DrainBody(resp.Body)
 		// Limit the ReadAll(), just in case, because of a bug, the server responds with large data.
-		r := io.LimitReader(resp.Body, 1024)
-		b, err := ioutil.ReadAll(r)
+		b, err := ioutil.ReadAll(io.LimitReader(resp.Body, 4096))
 		if err != nil {
 			return nil, err
 		}
@@ -95,7 +96,7 @@ func newCustomDialContext(timeout time.Duration) func(ctx context.Context, netwo
 }
 
 // NewClient - returns new REST client.
-func NewClient(url *url.URL, tlsConfig *tls.Config, timeout time.Duration, newAuthToken func() string) *Client {
+func NewClient(url *url.URL, tlsConfig *tls.Config, timeout time.Duration, newAuthToken func() string) (*Client, error) {
 	// Transport is exactly same as Go default in https://golang.org/pkg/net/http/#RoundTripper
 	// except custom DialContext and TLSClientConfig.
 	tr := &http.Transport{
@@ -103,17 +104,22 @@ func NewClient(url *url.URL, tlsConfig *tls.Config, timeout time.Duration, newAu
 		DialContext:           newCustomDialContext(timeout),
 		MaxIdleConnsPerHost:   4096,
 		MaxIdleConns:          4096,
-		IdleConnTimeout:       90 * time.Second,
-		TLSHandshakeTimeout:   10 * time.Second,
-		ExpectContinueTimeout: 1 * time.Second,
+		IdleConnTimeout:       120 * time.Second,
+		TLSHandshakeTimeout:   30 * time.Second,
+		ExpectContinueTimeout: 10 * time.Second,
 		TLSClientConfig:       tlsConfig,
 		DisableCompression:    true,
 	}
-
+	if tlsConfig != nil {
+		// If TLS is enabled configure http2
+		if err := http2.ConfigureTransport(tr); err != nil {
+			return nil, err
+		}
+	}
 	return &Client{
 		httpClient:          &http.Client{Transport: tr},
 		httpIdleConnsCloser: tr.CloseIdleConnections,
 		url:                 url,
 		newAuthToken:        newAuthToken,
-	}
+	}, nil
 }

cmd/routers.go

@@ -55,8 +55,8 @@ var globalHandlers = []HandlerFunc{
 	setBucketForwardingHandler,
 	// Ratelimit the incoming requests using a token bucket algorithm
 	setRateLimitHandler,
-	// Validate all the incoming paths.
-	setPathValidityHandler,
+	// Validate all the incoming requests.
+	setRequestValidityHandler,
 	// Network statistics
 	setHTTPStatsHandler,
 	// Limits all requests size to a maximum fixed limit