Fix writing 'format.json' and make it atomic (#8296)

- Choose a unique uuid such that under situations of duplicate
  mounts we do not append to an existing json entry.
- Avoid AppendFile instead use WriteAll() to write the entire
  byte array atomically.

.dockerignore

@@ -0,0 +1,2 @@
+.git
+.github

.travis.yml

@@ -26,8 +26,7 @@ matrix:
         - ARCH=x86_64
         - CGO_ENABLED=0
         - GO111MODULE=on
-        - GOPROXY=https://proxy.golang.org
-      go: 1.12.x
+      go: 1.13.x
       script:
         - make
         - diff -au <(gofmt -s -d cmd) <(printf "")
@@ -45,8 +44,7 @@ matrix:
         - ARCH=x86_64
         - CGO_ENABLED=0
         - GO111MODULE=on
-        - GOPROXY=https://proxy.golang.org
-      go: 1.12.x
+      go: 1.13.x
       script:
         - go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
         - bash buildscripts/go-coverage.sh
@@ -58,6 +56,3 @@ before_script:
 
 before_install:
  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install 11.10.1 ; fi
-
-after_success:
-- bash <(curl -s https://codecov.io/bash)

Dockerfile

@@ -1,10 +1,11 @@
-FROM golang:1.12-alpine
+FROM golang:1.13-alpine
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
 ENV GOPATH /go
 ENV CGO_ENABLED 0
 ENV GO111MODULE on
+ENV GOPROXY https://proxy.golang.org
 
 RUN  \
      apk add --no-cache git && \

Dockerfile.release

@@ -1,4 +1,4 @@
-FROM golang:1.12-alpine
+FROM golang:1.13-alpine
 
 ENV GOPATH /go
 ENV CGO_ENABLED 0

Dockerfile.simpleci

@@ -1,14 +1,13 @@
 #-------------------------------------------------------------
 # Stage 1: Build and Unit tests
 #-------------------------------------------------------------
-FROM golang:1.12
+FROM golang:1.13
 
 COPY . /go/src/github.com/minio/minio
 WORKDIR /go/src/github.com/minio/minio
 
 RUN apt-get update && apt-get install -y jq
 ENV GO111MODULE=on
-ENV GOPROXY=https://proxy.golang.org
 
 RUN git config --global http.cookiefile /gitcookie/.gitcookie
 

MAINTAINERS.md

@@ -1,37 +0,0 @@
-# For maintainers only
-
-### Setup your minio GitHub Repository
-
-Fork [minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository.
-```bash
-$ mkdir -p $GOPATH/src/github.com/minio
-$ cd $GOPATH/src/github.com/minio
-$ git clone https://github.com/$USER_ID/minio
-$ 
-```
-
-``minio`` uses [govendor](https://github.com/kardianos/govendor) for its dependency management.
-
-### To manage dependencies
-
-#### Add new dependencies
-
-  - Run `go get foo/bar`
-  - Edit your code to import foo/bar
-  - Run `govendor add foo/bar` from top-level directory
-
-#### Remove dependencies 
-
-  - Run `govendor remove foo/bar`
-
-#### Update dependencies
-
-  - Run `govendor remove +vendor`
-  - Run to update the dependent package `go get -u foo/bar`
-  - Run `govendor add +external`
-
-### Making new releases 
-
-`minio` doesn't follow semantic versioning style, `minio` instead uses the release date and time as the release versions.
-
-`make release` will generate new binary into `release` directory.

Makefile

@@ -2,13 +2,12 @@ PWD := $(shell pwd)
 GOPATH := $(shell go env GOPATH)
 LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
 
+GOARCH := $(shell go env GOARCH)
 GOOS := $(shell go env GOOS)
-GOOSALT ?= 'linux'
-ifeq ($(GOOS),'darwin')
-  GOOSALT = 'mac'
-endif
 
-TAG ?= $(USER)
+VERSION ?= $(shell git describe --tags)
+TAG ?= "minio/minio:$(VERSION)"
+
 BUILD_LDFLAGS := '$(LDFLAGS)'
 
 all: build
@@ -19,9 +18,13 @@ checks:
 
 getdeps:
 	@mkdir -p ${GOPATH}/bin
-	@which golint 1>/dev/null || (echo "Installing golint" && go get -u golang.org/x/lint/golint)
-	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet -O ${GOPATH}/bin/staticcheck https://github.com/dominikh/go-tools/releases/download/2019.1/staticcheck_${GOOS}_amd64 && chmod +x ${GOPATH}/bin/staticcheck)
-	@which misspell 1>/dev/null || (echo "Installing misspell" && wget --quiet https://github.com/client9/misspell/releases/download/v0.3.4/misspell_0.3.4_${GOOSALT}_64bit.tar.gz && tar xf misspell_0.3.4_${GOOSALT}_64bit.tar.gz && mv misspell ${GOPATH}/bin/misspell && chmod +x ${GOPATH}/bin/misspell && rm -f misspell_0.3.4_${GOOSALT}_64bit.tar.gz)
+	@which golint 1>/dev/null || (echo "Installing golint" && GO111MODULE=off go get -u golang.org/x/lint/golint)
+ifeq ($(GOARCH),s390x)
+	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && GO111MODULE=off go get honnef.co/go/tools/cmd/staticcheck)
+else
+	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet https://github.com/dominikh/go-tools/releases/download/2019.2.3/staticcheck_${GOOS}_${GOARCH}.tar.gz && tar xf staticcheck_${GOOS}_${GOARCH}.tar.gz && mv staticcheck/staticcheck ${GOPATH}/bin/staticcheck && chmod +x ${GOPATH}/bin/staticcheck && rm -f staticcheck_${GOOS}_${GOARCH}.tar.gz && rm -rf staticcheck)
+endif
+	@which misspell 1>/dev/null || (echo "Installing misspell" && GO111MODULE=off go get -u github.com/client9/misspell/cmd/misspell)
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
@@ -29,36 +32,37 @@ crosscompile:
 verifiers: getdeps vet fmt lint staticcheck spelling
 
 vet:
-	@echo "Running $@"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on go vet github.com/minio/minio/...
+	@echo "Running $@ check"
+	@GO111MODULE=on go vet github.com/minio/minio/...
 
 fmt:
-	@echo "Running $@"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on gofmt -d cmd/
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on gofmt -d pkg/
+	@echo "Running $@ check"
+	@GO111MODULE=on gofmt -d cmd/
+	@GO111MODULE=on gofmt -d pkg/
 
 lint:
-	@echo "Running $@"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd/...
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg/...
+	@echo "Running $@ check"
+	@GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd/...
+	@GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg/...
 
 staticcheck:
-	@echo "Running $@"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/cmd/...
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/pkg/...
+	@echo "Running $@ check"
+	@GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/cmd/...
+	@GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/pkg/...
 
 spelling:
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find cmd/`
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find pkg/`
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find docs/`
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
+	@echo "Running $@ check"
+	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find cmd/`
+	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find pkg/`
+	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find docs/`
+	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
+	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
 
 # Builds minio, runs the verifiers then runs the tests.
 check: test
 test: verifiers build
 	@echo "Running unit tests"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
 
 verify: build
 	@echo "Verifying build"
@@ -71,7 +75,7 @@ coverage: build
 # Builds minio locally.
 build: checks
 	@echo "Building minio binary to './minio'"
-	@GOPROXY=https://proxy.golang.org GO111MODULE=on GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
 
 docker: build
 	@docker build -t $(TAG) . -f Dockerfile.dev

README.md

@@ -5,7 +5,7 @@ MinIO is an object storage server released under Apache License v2.0. It is comp
 
 MinIO server is light enough to be bundled with the application stack, similar to NodeJS, Redis and MySQL.
 
-[1]: MinIO in its default mode is faster and does not calculate MD5Sum unless passed by client. This may lead to incompatibility with few S3 clients like s3ql that heavily depend on MD5Sum. For full compatibility with Amazon S3 API, start MinIO with `--compat` option.
+[1] MinIO in its default mode is faster and does not calculate MD5Sum unless passed by client. This may lead to incompatibility with some S3 clients like s3ql that heavily depend on MD5Sum. For such applications start MinIO with `--compat` option.
 ```sh
 minio --compat server /data
 ```
@@ -91,7 +91,7 @@ service minio start
 Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.12](https://golang.org/dl/#stable)
 
 ```sh
-GOPROXY=https://proxy.golang.org GO111MODULE=on go get github.com/minio/minio
+GO111MODULE=on go get github.com/minio/minio
 ```
 
 ## Allow port access for Firewalls
@@ -162,6 +162,22 @@ When deployed on a single drive, MinIO server lets clients access any pre-existi
 
 The above statement is also valid for all gateway backends.
 
+## Upgrading MinIO
+MinIO server supports rolling upgrades, i.e. you can update one MinIO instance at a time in a distributed cluster. This allows upgrades with no downtime. Upgrades can be done manually by replacing the binary with the latest release and restarting all servers in a rolling fashion. However, we recommend all our users to use [`mc admin update`](https://docs.min.io/docs/minio-admin-complete-guide.html#update) from the client. This will update all the nodes in the cluster and restart them, as shown in the following command from the MinIO client (mc):
+
+```
+mc admin update <minio alias, e.g., myminio>
+```
+
+**Important things to remember during upgrades**:
+
+- `mc admin update` will only work if the user running MinIO has write access to the parent directory where the binary is located, for example if the current binary is at `/usr/local/bin/minio`, you would need write access to `/usr/local/bin`.
+- In the case of federated setups `mc admin update` should be run against each cluster individually. Avoid updating `mc` until all clusters have been updated.
+- If you are updating the server it is always recommended (unless explicitly mentioned in MinIO server release notes), to update `mc` once all the servers have been upgraded using `mc update`.
+- `mc admin update` is disabled in docker/container environments, container environments provide their own mechanisms for updating running containers.
+- If you are using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
+- If you are using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
+
 ## Explore Further
 - [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
 - [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide)

buildscripts/checkdeps.sh

@@ -21,7 +21,7 @@ _init() {
 
     ## Minimum required versions for build dependencies
     GIT_VERSION="1.0"
-    GO_VERSION="1.12"
+    GO_VERSION="1.13"
     OSX_VERSION="10.8"
     KNAME=$(uname -s)
     ARCH=$(uname -m)

buildscripts/cross-compile.sh

@@ -23,7 +23,6 @@ function _build() {
     export GOOS=$os
     export GOARCH=$arch
     export GO111MODULE=on
-    export GOPROXY=https://proxy.golang.org
     go build -tags kqueue -o /dev/null
 }
 

buildscripts/go-coverage.sh

@@ -2,4 +2,4 @@
 
 set -e
 
-GOPROXY=https://proxy.golang.org GO111MODULE=on CGO_ENABLED=0 go test -v -coverprofile=coverage.txt -covermode=atomic ./...
+GO111MODULE=on CGO_ENABLED=0 go test -v -coverprofile=coverage.txt -covermode=atomic ./...

buildscripts/verify-build.sh

@@ -33,7 +33,6 @@ export ACCESS_KEY="minio"
 export SECRET_KEY="minio123"
 export ENABLE_HTTPS=0
 export GO111MODULE=on
-export GOPROXY=https://proxy.golang.org
 
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
 MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" )

cmd/admin-handlers.go

@@ -17,12 +17,9 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"crypto/subtle"
-	"encoding/base64"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -38,14 +35,11 @@ import (
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
-	"github.com/tidwall/gjson"
-	"github.com/tidwall/sjson"
 
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/cpu"
-	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/handlers"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
@@ -318,15 +312,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
-// ServerDrivesPerfInfo holds information about address, performance
-// of all drives on one server. It also reports any errors if encountered
-// while trying to reach this server.
-type ServerDrivesPerfInfo struct {
-	Addr  string             `json:"addr"`
-	Error string             `json:"error,omitempty"`
-	Perf  []disk.Performance `json:"perf"`
-}
-
 // ServerCPULoadInfo holds informantion about cpu utilization
 // of one minio node. It also reports any errors if encountered
 // while trying to reach this server.
@@ -406,16 +391,25 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 		writeSuccessResponseJSON(w, jsonBytes)
 
 	case "drive":
-		info := objectAPI.StorageInfo(ctx)
-		if !(info.Backend.Type == BackendFS || info.Backend.Type == BackendErasure) {
+		// Drive Perf is only implemented for Erasure coded backends
+		if !globalIsXL {
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 			return
 		}
+
+		var size int64 = madmin.DefaultDrivePerfSize
+		if sizeStr, found := vars["size"]; found {
+			var err error
+			if size, err = strconv.ParseInt(sizeStr, 10, 64); err != nil || size <= 0 {
+				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
+				return
+			}
+		}
 		// Get drive performance details from local server's drive(s)
-		dp := localEndpointsDrivePerf(globalEndpoints, r)
+		dp := getLocalDrivesPerf(globalEndpoints, size, r)
 
 		// Notify all other MinIO peers to report drive performance numbers
-		dps := globalNotificationSys.DrivePerfInfo()
+		dps := globalNotificationSys.DrivePerfInfo(size)
 		dps = append(dps, dp)
 
 		// Marshal API response
@@ -430,7 +424,7 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 		writeSuccessResponseJSON(w, jsonBytes)
 	case "cpu":
 		// Get CPU load details from local server's cpu(s)
-		cpu := localEndpointsCPULoad(globalEndpoints, r)
+		cpu := getLocalCPULoad(globalEndpoints, r)
 		// Notify all other MinIO peers to report cpu load numbers
 		cpus := globalNotificationSys.CPULoadInfo()
 		cpus = append(cpus, cpu)
@@ -447,7 +441,7 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 		writeSuccessResponseJSON(w, jsonBytes)
 	case "mem":
 		// Get mem usage details from local server(s)
-		m := localEndpointsMemUsage(globalEndpoints, r)
+		m := getLocalMemUsage(globalEndpoints, r)
 		// Notify all other MinIO peers to report mem usage numbers
 		mems := globalNotificationSys.MemUsageInfo()
 		mems = append(mems, m)
@@ -953,25 +947,6 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 	writeSuccessResponseJSON(w, econfigData)
 }
 
-// Disable tidwall json array notation in JSON key path so
-// users can set json with a key as a number.
-// In tidwall json, notify.webhook.0 = val means { "notify" : { "webhook" : [val] }}
-// In MinIO, notify.webhook.0 = val means { "notify" : { "webhook" : {"0" : val}}}
-func normalizeJSONKey(input string) (key string) {
-	subKeys := strings.Split(input, ".")
-	for i, k := range subKeys {
-		if i > 0 {
-			key += "."
-		}
-		if _, err := strconv.Atoi(k); err == nil {
-			key += ":" + k
-		} else {
-			key += k
-		}
-	}
-	return
-}
-
 func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request) ObjectLayer {
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -990,60 +965,6 @@ func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Reques
 	return objectAPI
 }
 
-// GetConfigHandler - GET /minio/admin/v1/config-keys
-// Get some keys in config.json of this minio setup.
-func (a adminAPIHandlers) GetConfigKeysHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetConfigKeysHandler")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	var keys []string
-	queries := r.URL.Query()
-
-	for k := range queries {
-		keys = append(keys, k)
-	}
-
-	config, err := readServerConfig(ctx, objectAPI)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	configData, err := json.Marshal(config)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	configStr := string(configData)
-	newConfigStr := `{}`
-
-	for _, key := range keys {
-		// sjson.Set does not return an error if key is empty
-		// we should check by ourselves here
-		if key == "" {
-			continue
-		}
-		val := gjson.Get(configStr, key)
-		if j, ierr := sjson.Set(newConfigStr, normalizeJSONKey(key), val.Value()); ierr == nil {
-			newConfigStr = j
-		}
-	}
-
-	password := config.GetCredential().SecretKey
-	econfigData, err := madmin.EncryptData(password, []byte(newConfigStr))
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	writeSuccessResponseJSON(w, []byte(econfigData))
-}
-
 // AdminError - is a generic error for all admin APIs.
 type AdminError struct {
 	Code       string
@@ -1260,7 +1181,12 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	groups := globalIAMSys.ListGroups()
+	groups, err := globalIAMSys.ListGroups()
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
 	body, err := json.Marshal(groups)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -1620,130 +1546,6 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	writeSuccessResponseHeadersOnly(w)
 }
 
-func convertValueType(elem []byte, jsonType gjson.Type) (interface{}, error) {
-	str := string(elem)
-	switch jsonType {
-	case gjson.False, gjson.True:
-		return strconv.ParseBool(str)
-	case gjson.JSON:
-		if gjson.Valid(str) {
-			return gjson.Parse(str).Value(), nil
-		}
-		return nil, errors.New("invalid json")
-	case gjson.String:
-		return str, nil
-	case gjson.Number:
-		return strconv.ParseFloat(str, 64)
-	default:
-		return nil, nil
-	}
-}
-
-// SetConfigKeysHandler - PUT /minio/admin/v1/config-keys
-func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetConfigKeysHandler")
-
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
-		return
-	}
-
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
-	// Load config
-	configStruct, err := readServerConfig(ctx, objectAPI)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Convert config to json bytes
-	configBytes, err := json.Marshal(configStruct)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	configStr := string(configBytes)
-
-	queries := r.URL.Query()
-	password := globalServerConfig.GetCredential().SecretKey
-
-	// Set key values in the JSON config
-	for k := range queries {
-		// Decode encrypted data associated to the current key
-		encryptedElem, dErr := base64.StdEncoding.DecodeString(queries.Get(k))
-		if dErr != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("key", k)
-			ctx = logger.SetReqInfo(ctx, reqInfo)
-			logger.LogIf(ctx, dErr)
-			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), dErr.Error(), r.URL)
-			return
-		}
-
-		elem, dErr := madmin.DecryptData(password, bytes.NewBuffer([]byte(encryptedElem)))
-		if dErr != nil {
-			logger.LogIf(ctx, dErr)
-			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), dErr.Error(), r.URL)
-			return
-		}
-
-		// Calculate the type of the current key from the
-		// original config json
-		jsonFieldType := gjson.Get(configStr, k).Type
-		// Convert passed value to json filed type
-		val, cErr := convertValueType(elem, jsonFieldType)
-		if cErr != nil {
-			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), cErr.Error(), r.URL)
-			return
-		}
-		// Set the key/value in the new json document
-		if s, sErr := sjson.Set(configStr, normalizeJSONKey(k), val); sErr == nil {
-			configStr = s
-		}
-	}
-
-	configBytes = []byte(configStr)
-
-	// Validate config
-	var config serverConfig
-	if err = json.Unmarshal(configBytes, &config); err != nil {
-		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
-		return
-	}
-
-	if err = config.Validate(); err != nil {
-		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
-		return
-	}
-
-	if err = config.TestNotificationTargets(); err != nil {
-		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
-		return
-	}
-
-	// If credentials for the server are provided via environment,
-	// then credentials in the provided configuration must match.
-	if globalIsEnvCreds {
-		if !globalServerConfig.GetCredential().Equal(config.Credential) {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminCredentialsMismatch), r.URL)
-			return
-		}
-	}
-
-	if err = saveServerConfig(ctx, objectAPI, &config); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Send success response
-	writeSuccessResponseHeadersOnly(w)
-}
-
 // Returns true if the trace.Info should be traced,
 // false if certain conditions are not met.
 // - input entry is not of the type *trace.Info*

cmd/admin-handlers_test.go

@@ -25,7 +25,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
-	"reflect"
 	"strings"
 	"sync"
 	"testing"
@@ -34,7 +33,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/madmin"
-	"github.com/tidwall/gjson"
 )
 
 var (
@@ -750,52 +748,3 @@ func TestExtractHealInitParams(t *testing.T) {
 	}
 
 }
-
-func TestNormalizeJSONKey(t *testing.T) {
-	cases := []struct {
-		input         string
-		correctResult string
-	}{
-		{"notify.webhook.1", "notify.webhook.:1"},
-		{"notify.webhook.ok", "notify.webhook.ok"},
-		{"notify.webhook.1.2", "notify.webhook.:1.:2"},
-		{"abc", "abc"},
-	}
-	for i, tcase := range cases {
-		res := normalizeJSONKey(tcase.input)
-		if res != tcase.correctResult {
-			t.Errorf("Case %d: failed to match", i)
-		}
-	}
-}
-
-func TestConvertValueType(t *testing.T) {
-	cases := []struct {
-		elem          []byte
-		jt            gjson.Type
-		correctResult interface{}
-		isError       bool
-	}{
-		{[]byte(""), gjson.Null, nil, false},
-		{[]byte("t"), gjson.False, true, false},
-		{[]byte("f"), gjson.False, false, false},
-		{[]byte(`{"a": 1}`), gjson.JSON, map[string]interface{}{"a": float64(1)}, false},
-		{[]byte(`abc`), gjson.String, "abc", false},
-		{[]byte(`1`), gjson.Number, float64(1), false},
-		{[]byte(`a1`), gjson.Number, nil, true},
-		{[]byte(`{"A": `), gjson.JSON, nil, true},
-		{[]byte(`2`), gjson.False, nil, true},
-	}
-	for i, tc := range cases {
-		res, err := convertValueType(tc.elem, tc.jt)
-		if err != nil {
-			if !tc.isError {
-				t.Errorf("Case %d: got an error when none was expected", i)
-			}
-		} else if err == nil && tc.isError {
-			t.Errorf("Case %d: got no error though we expected one", i)
-		} else if !reflect.DeepEqual(res, tc.correctResult) {
-			t.Errorf("Case %d: result mismatch - got %#v", i, res)
-		}
-	}
-}

cmd/admin-router.go

@@ -77,11 +77,6 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
 		// Set config
 		adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
-
-		// Get config keys/values
-		adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
-		// Set config keys/values
-		adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
 	}
 
 	if enableIAMOps {

cmd/admin-server-info.go

@@ -0,0 +1,114 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/http"
+	"os"
+
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/pkg/cpu"
+	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/mem"
+)
+
+// getLocalMemUsage - returns ServerMemUsageInfo for only the
+// local endpoints from given list of endpoints
+func getLocalMemUsage(endpoints EndpointList, r *http.Request) ServerMemUsageInfo {
+	var memUsages []mem.Usage
+	var historicUsages []mem.Usage
+	seenHosts := set.NewStringSet()
+	for _, endpoint := range endpoints {
+		if seenHosts.Contains(endpoint.Host) {
+			continue
+		}
+		seenHosts.Add(endpoint.Host)
+
+		// Only proceed for local endpoints
+		if endpoint.IsLocal {
+			memUsages = append(memUsages, mem.GetUsage())
+			historicUsages = append(historicUsages, mem.GetHistoricUsage())
+		}
+	}
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpoints)
+	}
+	return ServerMemUsageInfo{
+		Addr:          addr,
+		Usage:         memUsages,
+		HistoricUsage: historicUsages,
+	}
+}
+
+// getLocalCPULoad - returns ServerCPULoadInfo for only the
+// local endpoints from given list of endpoints
+func getLocalCPULoad(endpoints EndpointList, r *http.Request) ServerCPULoadInfo {
+	var cpuLoads []cpu.Load
+	var historicLoads []cpu.Load
+	seenHosts := set.NewStringSet()
+	for _, endpoint := range endpoints {
+		if seenHosts.Contains(endpoint.Host) {
+			continue
+		}
+		seenHosts.Add(endpoint.Host)
+
+		// Only proceed for local endpoints
+		if endpoint.IsLocal {
+			cpuLoads = append(cpuLoads, cpu.GetLoad())
+			historicLoads = append(historicLoads, cpu.GetHistoricLoad())
+		}
+	}
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpoints)
+	}
+	return ServerCPULoadInfo{
+		Addr:         addr,
+		Load:         cpuLoads,
+		HistoricLoad: historicLoads,
+	}
+}
+
+// getLocalDrivesPerf - returns ServerDrivesPerfInfo for only the
+// local endpoints from given list of endpoints
+func getLocalDrivesPerf(endpoints EndpointList, size int64, r *http.Request) madmin.ServerDrivesPerfInfo {
+	var dps []disk.Performance
+	for _, endpoint := range endpoints {
+		// Only proceed for local endpoints
+		if endpoint.IsLocal {
+			if _, err := os.Stat(endpoint.Path); err != nil {
+				// Since this drive is not available, add relevant details and proceed
+				dps = append(dps, disk.Performance{Path: endpoint.Path, Error: err.Error()})
+				continue
+			}
+			dp := disk.GetPerformance(pathJoin(endpoint.Path, minioMetaTmpBucket, mustGetUUID()), size)
+			dp.Path = endpoint.Path
+			dps = append(dps, dp)
+		}
+	}
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpoints)
+	}
+	return madmin.ServerDrivesPerfInfo{
+		Addr: addr,
+		Perf: dps,
+		Size: size,
+	}
+}

cmd/api-errors.go

@@ -315,6 +315,7 @@ const (
 	ErrAdminProfilerNotEnabled
 	ErrInvalidDecompressedSize
 	ErrAddUserInvalidArgument
+	ErrPostPolicyConditionInvalidFormat
 )
 
 type errorCodeMap map[APIErrorCode]APIError
@@ -1496,6 +1497,11 @@ var errorCodes = errorCodeMap{
 		Description:    "User is not allowed to be same as admin access key",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrPostPolicyConditionInvalidFormat: {
+		Code:           "PostPolicyInvalidKeyName",
+		Description:    "Invalid according to Policy: Policy Condition failed",
+		HTTPStatusCode: http.StatusForbidden,
+	},
 	// Add your error structure here.
 }
 
@@ -1723,6 +1729,12 @@ func toAPIError(ctx context.Context, err error) APIError {
 		// their internal error types. This code is only
 		// useful with gateway implementations.
 		switch e := err.(type) {
+		case crypto.Error:
+			apiErr = APIError{
+				Code:           "XKMSInternalError",
+				Description:    e.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case minio.ErrorResponse:
 			apiErr = APIError{
 				Code:           e.Code,

cmd/api-response.go

@@ -498,8 +498,8 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 	data.Delimiter = s3EncodeName(delimiter, encodingType)
 	data.Prefix = s3EncodeName(prefix, encodingType)
 	data.MaxKeys = maxKeys
-	data.ContinuationToken = token
-	data.NextContinuationToken = nextToken
+	data.ContinuationToken = s3EncodeName(token, encodingType)
+	data.NextContinuationToken = s3EncodeName(nextToken, encodingType)
 	data.IsTruncated = isTruncated
 	for _, prefix := range prefixes {
 		var prefixItem = CommonPrefix{}

cmd/auth-handler.go

@@ -203,6 +203,12 @@ func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
 	}
 
 	if globalPolicyOPA == nil {
+		// If OPA is not set and if ldap claim key is set,
+		// allow the claim.
+		if _, ok := claims[ldapUser]; ok {
+			return claims, nil
+		}
+
 		// If OPA is not set, session token should
 		// have a policy and its mandatory, reject
 		// requests without policy claim.

cmd/bitrot.go

@@ -155,3 +155,11 @@ func bitrotWriterSum(w io.Writer) []byte {
 	}
 	return nil
 }
+
+// Returns the size of the file with bitrot protection
+func bitrotShardFileSize(size int64, shardSize int64, algo BitrotAlgorithm) int64 {
+	if algo != HighwayHash256S {
+		return size
+	}
+	return ceilFrac(size, shardSize)*int64(algo.New().Size()) + size
+}

cmd/bucket-handlers.go

@@ -557,8 +557,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
-
-	if !api.EncryptionEnabled() && hasServerSideEncryptionHeader(r.Header) {
+	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -657,9 +656,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	// Handle policy if it is set.
 	if len(policyBytes) > 0 {
+
 		postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
 		if err != nil {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPostPolicyConditionInvalidFormat), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -715,7 +715,11 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 	if objectAPI.IsEncryptionSupported() {
-		if hasServerSideEncryptionHeader(formValues) && !hasSuffix(object, SlashSeparator) { // handle SSE-C and SSE-S3 requests
+		if crypto.IsRequested(formValues) && !hasSuffix(object, SlashSeparator) { // handle SSE requests
+			if crypto.SSECopy.IsRequested(r.Header) {
+				writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL, guessIsBrowserReq(r))
+				return
+			}
 			var reader io.Reader
 			var key []byte
 			if crypto.SSEC.IsRequested(formValues) {

cmd/common-main.go

@@ -64,9 +64,9 @@ func checkUpdate(mode string) {
 			return
 		}
 		if globalInplaceUpdateDisabled {
-			logger.StartupMessage(updateMsg)
+			logStartupMessage(updateMsg)
 		} else {
-			logger.StartupMessage(prepareUpdateMessage("Run `mc admin update`", latestReleaseTime.Sub(currentReleaseTime)))
+			logStartupMessage(prepareUpdateMessage("Run `mc admin update`", latestReleaseTime.Sub(currentReleaseTime)))
 		}
 	}
 }
@@ -438,3 +438,10 @@ func handleCommonEnvVars() {
 		}
 	}
 }
+
+func logStartupMessage(msg string, data ...interface{}) {
+	if globalConsoleSys != nil {
+		globalConsoleSys.Send(msg)
+	}
+	logger.StartupMessage(msg, data...)
+}

cmd/config-current.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -303,6 +303,12 @@ func (s *serverConfig) loadFromEnvs() {
 		s.Policy.OPA.URL = opaArgs.URL
 		s.Policy.OPA.AuthToken = opaArgs.AuthToken
 	}
+
+	var err error
+	s.LDAPServerConfig, err = newLDAPConfigFromEnv()
+	if err != nil {
+		logger.FatalIf(err, "Unable to parse LDAP configuration from env")
+	}
 }
 
 // TestNotificationTargets tries to establish connections to all notification
@@ -335,6 +341,9 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
+		if v.TLS.Enable {
+			v.TLS.RootCAs = globalRootCAs
+		}
 		t, err := target.NewKafkaTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("kafka(%s): %s", k, err.Error())
@@ -346,6 +355,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
+		v.RootCAs = globalRootCAs
 		t, err := target.NewMQTTTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("mqtt(%s): %s", k, err.Error())
@@ -684,6 +694,9 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.Kafka {
 		if args.Enable {
+			if args.TLS.Enable {
+				args.TLS.RootCAs = globalRootCAs
+			}
 			newTarget, err := target.NewKafkaTarget(id, args, GlobalServiceDoneCh)
 			if err != nil {
 				logger.LogIf(context.Background(), err)

cmd/config-versions.go

@@ -926,4 +926,6 @@ type serverConfigV33 struct {
 
 		// Add new external policy enforcements here.
 	} `json:"policy"`
+
+	LDAPServerConfig ldapServerConfig `json:"ldapserverconfig"`
 }

cmd/consolelogger.go

@@ -113,9 +113,14 @@ func (sys *HTTPConsoleLoggerSys) Console() *HTTPConsoleLoggerSys {
 // Send log message 'e' to console and publish to console
 // log pubsub system
 func (sys *HTTPConsoleLoggerSys) Send(e interface{}) error {
-	lg := madmin.LogInfo{}
-	lg.Entry = e.(log.Entry)
-	lg.NodeName = sys.nodeName
+	var lg madmin.LogInfo
+	switch e := e.(type) {
+	case log.Entry:
+		lg = madmin.LogInfo{Entry: e, NodeName: sys.nodeName}
+	case string:
+		lg = madmin.LogInfo{ConsoleMsg: e, NodeName: sys.nodeName}
+	}
+
 	sys.pubsub.Publish(lg)
 	// add log to ring buffer
 	sys.logBuf.Value = lg

cmd/crypto/error.go

@@ -19,9 +19,9 @@ import "errors"
 // Error is the generic type for any error happening during decrypting
 // an object. It indicates that the object itself or its metadata was
 // modified accidentally or maliciously.
-type Error struct{ msg string }
+type Error string
 
-func (e Error) Error() string { return e.msg }
+func (e Error) Error() string { return string(e) }
 
 var (
 	// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
@@ -56,14 +56,14 @@ var (
 	ErrIncompatibleEncryptionMethod = errors.New("Server side encryption specified with both SSE-C and SSE-S3 headers")
 )
 
-var (
-	errMissingInternalIV            = Error{"The object metadata is missing the internal encryption IV"}
-	errMissingInternalSealAlgorithm = Error{"The object metadata is missing the internal seal algorithm"}
+const (
+	errMissingInternalIV            Error = "The object metadata is missing the internal encryption IV"
+	errMissingInternalSealAlgorithm Error = "The object metadata is missing the internal seal algorithm"
 
-	errInvalidInternalIV            = Error{"The internal encryption IV is malformed"}
-	errInvalidInternalSealAlgorithm = Error{"The internal seal algorithm is invalid and not supported"}
+	errInvalidInternalIV            Error = "The internal encryption IV is malformed"
+	errInvalidInternalSealAlgorithm Error = "The internal seal algorithm is invalid and not supported"
 
-	errMissingUpdatedKey = Error{"The key update returned no error but also no sealed key"}
+	errMissingUpdatedKey Error = "The key update returned no error but also no sealed key"
 )
 
 var (

cmd/crypto/header.go

@@ -83,6 +83,13 @@ func RemoveSensitiveHeaders(h http.Header) {
 	h.Del(SSECopyKey)
 }
 
+// IsRequested returns true if the HTTP headers indicates
+// that any form server-side encryption (SSE-C, SSE-S3 or SSE-KMS)
+// is requested.
+func IsRequested(h http.Header) bool {
+	return S3.IsRequested(h) || SSEC.IsRequested(h) || SSECopy.IsRequested(h) || S3KMS.IsRequested(h)
+}
+
 // S3 represents AWS SSE-S3. It provides functionality to handle
 // SSE-S3 requests.
 var S3 = s3{}

cmd/crypto/header_test.go

@@ -20,6 +20,29 @@ import (
 	"testing"
 )
 
+func TestIsRequested(t *testing.T) {
+	for i, test := range kmsIsRequestedTests {
+		if got := IsRequested(test.Header) && S3KMS.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("SSE-KMS: Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+	for i, test := range s3IsRequestedTests {
+		if got := IsRequested(test.Header) && S3.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("SSE-S3: Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+	for i, test := range ssecIsRequestedTests {
+		if got := IsRequested(test.Header) && SSEC.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("SSE-C: Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+	for i, test := range ssecCopyIsRequestedTests {
+		if got := IsRequested(test.Header) && SSECopy.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("SSE-C: Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+}
+
 var kmsIsRequestedTests = []struct {
 	Header   http.Header
 	Expected bool

cmd/crypto/key.go

@@ -108,7 +108,7 @@ func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucke
 	)
 	switch sealedKey.Algorithm {
 	default:
-		return Error{fmt.Sprintf("The sealing algorithm '%s' is not supported", sealedKey.Algorithm)}
+		return Error(fmt.Sprintf("The sealing algorithm '%s' is not supported", sealedKey.Algorithm))
 	case SealAlgorithm:
 		mac := hmac.New(sha256.New, extKey[:])
 		mac.Write(sealedKey.IV[:])

cmd/crypto/metadata.go

@@ -17,6 +17,7 @@ package crypto
 import (
 	"context"
 	"encoding/base64"
+	"errors"
 	"fmt"
 
 	"github.com/minio/minio/cmd/logger"
@@ -119,28 +120,46 @@ func CreateMultipartMetadata(metadata map[string]string) map[string]string {
 	return metadata
 }
 
-// CreateMetadata encodes the keyID, the sealed kms data key and the sealed key
-// into the metadata and returns the modified metadata. It allocates a new
-// metadata map if metadata is nil.
+// CreateMetadata encodes the sealed object key into the metadata and returns
+// the modified metadata. If the keyID and the kmsKey is not empty it encodes
+// both into the metadata as well. It allocates a new metadata map if metadata
+// is nil.
 func (s3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
 	if sealedKey.Algorithm != SealAlgorithm {
 		logger.CriticalIf(context.Background(), fmt.Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
 	}
 
+	// There are two possibilites:
+	// - We use a KMS -> There must be non-empty key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the caller has passed an invalid argument combination.
+	if keyID == "" && len(kmsKey) != 0 {
+		logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
+	}
+	if keyID != "" && len(kmsKey) == 0 {
+		logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
+	}
+
 	if metadata == nil {
 		metadata = map[string]string{}
 	}
-	metadata[S3KMSKeyID] = keyID
+
 	metadata[SSESealAlgorithm] = sealedKey.Algorithm
 	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
 	metadata[S3SealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
-	metadata[S3KMSSealedKey] = base64.StdEncoding.EncodeToString(kmsKey)
+	if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
+		metadata[S3KMSKeyID] = keyID
+		metadata[S3KMSSealedKey] = base64.StdEncoding.EncodeToString(kmsKey)
+	}
 	return metadata
 }
 
 // ParseMetadata extracts all SSE-S3 related values from the object metadata
-// and checks whether they are well-formed. It returns the KMS key-ID, the
-// sealed KMS key and the sealed object key on success.
+// and checks whether they are well-formed. It returns the sealed object key
+// on success. If the metadata contains both, a KMS master key ID and a sealed
+// KMS data key it returns both. If the metadata does not contain neither a
+// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
+// KMS data key. Otherwise, it returns an error.
 func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
 	// Extract all required values from object metadata
 	b64IV, ok := metadata[SSEIV]
@@ -153,15 +172,20 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 	b64SealedKey, ok := metadata[S3SealedKey]
 	if !ok {
-		return keyID, kmsKey, sealedKey, Error{"The object metadata is missing the internal sealed key for SSE-S3"}
+		return keyID, kmsKey, sealedKey, Error("The object metadata is missing the internal sealed key for SSE-S3")
 	}
-	keyID, ok = metadata[S3KMSKeyID]
-	if !ok {
-		return keyID, kmsKey, sealedKey, Error{"The object metadata is missing the internal KMS key-ID for SSE-S3"}
+
+	// There are two possibilites:
+	// - We use a KMS -> There must be a key ID and a KMS data key.
+	// - We use a K/V -> There must be no key ID and no KMS data key.
+	// Otherwise, the metadata is corrupted.
+	keyID, idPresent := metadata[S3KMSKeyID]
+	b64KMSSealedKey, kmsKeyPresent := metadata[S3KMSSealedKey]
+	if !idPresent && kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Error("The object metadata is missing the internal KMS key-ID for SSE-S3")
 	}
-	b64KMSSealedKey, ok := metadata[S3KMSSealedKey]
-	if !ok {
-		return keyID, kmsKey, sealedKey, Error{"The object metadata is missing the internal sealed KMS data key for SSE-S3"}
+	if idPresent && !kmsKeyPresent {
+		return keyID, kmsKey, sealedKey, Error("The object metadata is missing the internal sealed KMS data key for SSE-S3")
 	}
 
 	// Check whether all extracted values are well-formed
@@ -174,11 +198,13 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
 	if err != nil || len(encryptedKey) != 64 {
-		return keyID, kmsKey, sealedKey, Error{"The internal sealed key for SSE-S3 is invalid"}
+		return keyID, kmsKey, sealedKey, Error("The internal sealed key for SSE-S3 is invalid")
 	}
-	kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
-	if err != nil {
-		return keyID, kmsKey, sealedKey, Error{"The internal sealed KMS data key for SSE-S3 is invalid"}
+	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
+		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
+		if err != nil {
+			return keyID, kmsKey, sealedKey, Error("The internal sealed KMS data key for SSE-S3 is invalid")
+		}
 	}
 
 	sealedKey.Algorithm = algorithm
@@ -218,7 +244,7 @@ func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err
 	}
 	b64SealedKey, ok := metadata[SSECSealedKey]
 	if !ok {
-		return sealedKey, Error{"The object metadata is missing the internal sealed key for SSE-C"}
+		return sealedKey, Error("The object metadata is missing the internal sealed key for SSE-C")
 	}
 
 	// Check whether all extracted values are well-formed
@@ -231,7 +257,7 @@ func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err
 	}
 	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
 	if err != nil || len(encryptedKey) != 64 {
-		return sealedKey, Error{"The internal sealed key for SSE-C is invalid"}
+		return sealedKey, Error("The internal sealed key for SSE-C is invalid")
 	}
 
 	sealedKey.Algorithm = algorithm

cmd/crypto/metadata_test.go

@@ -125,15 +125,15 @@ var s3ParseMetadataTests = []struct {
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 1
 	{
-		ExpectedErr: Error{"The object metadata is missing the internal sealed key for SSE-S3"},
+		ExpectedErr: Error("The object metadata is missing the internal sealed key for SSE-S3"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 2
 	{
-		ExpectedErr: Error{"The object metadata is missing the internal KMS key-ID for SSE-S3"},
-		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
+		ExpectedErr: Error("The object metadata is missing the internal KMS key-ID for SSE-S3"),
+		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSSealedKey: "IAAF0b=="}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 3
 	{
-		ExpectedErr: Error{"The object metadata is missing the internal sealed KMS data key for SSE-S3"},
+		ExpectedErr: Error("The object metadata is missing the internal sealed KMS data key for SSE-S3"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: ""},
 		DataKey:     []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 4
@@ -150,7 +150,7 @@ var s3ParseMetadataTests = []struct {
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 6
 	{
-		ExpectedErr: Error{"The internal sealed key for SSE-S3 is invalid"},
+		ExpectedErr: Error("The internal sealed key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
 			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, S3SealedKey: "",
 			S3KMSKeyID: "", S3KMSSealedKey: "",
@@ -158,7 +158,7 @@ var s3ParseMetadataTests = []struct {
 		DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
 	}, // 7
 	{
-		ExpectedErr: Error{"The internal sealed KMS data key for SSE-S3 is invalid"},
+		ExpectedErr: Error("The internal sealed KMS data key for SSE-S3 is invalid"),
 		Metadata: map[string]string{
 			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
 			S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "key-1",
@@ -218,7 +218,7 @@ var ssecParseMetadataTests = []struct {
 	{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}},                     // 0
 	{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""}, SealedKey: SealedKey{}}, // 1
 	{
-		ExpectedErr: Error{"The object metadata is missing the internal sealed key for SSE-C"},
+		ExpectedErr: Error("The object metadata is missing the internal sealed key for SSE-C"),
 		Metadata:    map[string]string{SSEIV: "", SSESealAlgorithm: ""}, SealedKey: SealedKey{},
 	}, // 2
 	{
@@ -233,7 +233,7 @@ var ssecParseMetadataTests = []struct {
 		SealedKey: SealedKey{},
 	}, // 4
 	{
-		ExpectedErr: Error{"The internal sealed key for SSE-C is invalid"},
+		ExpectedErr: Error("The internal sealed key for SSE-C is invalid"),
 		Metadata: map[string]string{
 			SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, SSECSealedKey: "",
 		},
@@ -287,7 +287,9 @@ var s3CreateMetadataTests = []struct {
 	SealedDataKey []byte
 	SealedKey     SealedKey
 }{
-	{KeyID: "", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+
+	{KeyID: "", SealedDataKey: nil, SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "my-minio-key", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
 	{KeyID: "cafebabe", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
 	{KeyID: "deadbeef", SealedDataKey: make([]byte, 32), SealedKey: SealedKey{IV: [32]byte{0xf7}, Key: [64]byte{0xea}, Algorithm: SealAlgorithm}},
 }

cmd/daily-lifecycle-ops.go

@@ -139,22 +139,24 @@ func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 			if err != nil {
 				continue
 			}
+			var objects []string
 			for _, obj := range res.Objects {
 				// Find the action that need to be executed
 				action := l.ComputeAction(obj.Name, obj.ModTime)
 				switch action {
 				case lifecycle.DeleteAction:
-					objAPI.DeleteObject(ctx, bucket.Name, obj.Name)
+					objects = append(objects, obj.Name)
 				default:
-					// Nothing
-
+					// Do nothing, for now.
 				}
 			}
+			// Deletes a list of objects.
+			objAPI.DeleteObjects(ctx, bucket.Name, objects)
 			if !res.IsTruncated {
+				// We are done here, proceed to next bucket.
 				break
-			} else {
-				marker = res.NextMarker
 			}
+			marker = res.NextMarker
 		}
 	}
 

cmd/daily-sweeper.go

@@ -91,6 +91,7 @@ func sweepRound(ctx context.Context, objAPI ObjectLayer) error {
 			if err != nil {
 				continue
 			}
+
 			for _, obj := range res.Objects {
 				for _, l := range copyDailySweepListeners() {
 					l <- pathJoin(bucket.Name, obj.Name)
@@ -133,7 +134,6 @@ func dailySweeper() {
 	// Start with random sleep time, so as to avoid "synchronous checks" between servers
 	time.Sleep(time.Duration(rand.Float64() * float64(time.Hour)))
 
-	// Perform a sweep round each month
 	for {
 		if time.Since(lastSweepTime) < 30*24*time.Hour {
 			time.Sleep(time.Hour)
@@ -147,13 +147,12 @@ func dailySweeper() {
 			// instance doing the sweep round
 			case OperationTimedOut:
 				lastSweepTime = time.Now()
-			default:
-				logger.LogIf(ctx, err)
-				time.Sleep(time.Minute)
 				continue
 			}
-		} else {
-			lastSweepTime = time.Now()
+			logger.LogIf(ctx, err)
+			time.Sleep(time.Minute)
+			continue
 		}
+		lastSweepTime = time.Now()
 	}
 }

cmd/disk-cache-backend.go

@@ -333,28 +333,6 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 	return err
 }
 
-// updates ETag in cache metadata file to the backend ETag.
-func (c *diskCache) updateETag(ctx context.Context, bucket, object string, etag string) error {
-	metaPath := path.Join(getCacheSHADir(c.dir, bucket, object), cacheMetaJSONFile)
-	f, err := os.OpenFile(metaPath, os.O_RDWR, 0)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-
-	meta := &cacheMeta{Version: cacheMetaVersion}
-	if err := jsonLoad(f, meta); err != nil {
-		return err
-	}
-	meta.Meta["etag"] = etag
-	jsonData, err := json.Marshal(meta)
-	if err != nil {
-		return err
-	}
-	_, err = f.Write(jsonData)
-	return err
-}
-
 // Backend metadata could have changed through server side copy - reset cache metadata if that is the case
 func (c *diskCache) updateMetadataIfChanged(ctx context.Context, bucket, object string, bkObjectInfo, cacheObjInfo ObjectInfo) error {
 
@@ -418,7 +396,7 @@ func (c *diskCache) bitrotWriteToCache(ctx context.Context, cachePath string, re
 	bufp := c.pool.Get().(*[]byte)
 	defer c.pool.Put(bufp)
 
-	var n int
+	var n, n2 int
 	for {
 		n, err = io.ReadFull(reader, *bufp)
 		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
@@ -437,10 +415,10 @@ func (c *diskCache) bitrotWriteToCache(ctx context.Context, cachePath string, re
 		if _, err = f.Write(hashBytes); err != nil {
 			return 0, err
 		}
-		if _, err = f.Write((*bufp)[:n]); err != nil {
+		if n2, err = f.Write((*bufp)[:n]); err != nil {
 			return 0, err
 		}
-		bytesWritten += int64(n)
+		bytesWritten += int64(n2)
 		if eof {
 			break
 		}
@@ -521,9 +499,11 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 		c.setOnline(false)
 	}
 	if err != nil {
+		removeAll(cachePath)
 		return err
 	}
 	if actualSize != uint64(n) {
+		removeAll(cachePath)
 		return IncompleteBody{}
 	}
 	return c.saveMetadata(ctx, bucket, object, metadata, n)
@@ -648,7 +628,11 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 	filePath := path.Join(cacheObjPath, cacheDataFile)
 	pr, pw := io.Pipe()
 	go func() {
-		pw.CloseWithError(c.bitrotReadFromCache(ctx, filePath, off, length, pw))
+		err := c.bitrotReadFromCache(ctx, filePath, off, length, pw)
+		if err != nil {
+			removeAll(cacheObjPath)
+		}
+		pw.CloseWithError(err)
 	}()
 	// Cleanup function to cause the go routine above to exit, in
 	// case of incomplete read.

cmd/disk-cache.go

@@ -14,7 +14,6 @@ import (
 
 	"github.com/djherbis/atime"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/wildcard"
 )
 
@@ -447,9 +446,9 @@ func checkAtimeSupport(dir string) (err error) {
 	return
 }
 func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
-	logger.StartupMessage(colorBlue("Cache migration initiated ...."))
+	logStartupMessage(colorBlue("Cache migration initiated ...."))
 
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 	errs := make([]error, len(c.cache))
 	for i, dc := range c.cache {
 		if dc == nil {
@@ -483,13 +482,12 @@ func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
 	c.migMutex.Lock()
 	defer c.migMutex.Unlock()
 	c.migrating = false
-	logger.StartupMessage(colorBlue("Cache migration completed successfully."))
+	logStartupMessage(colorBlue("Cache migration completed successfully."))
 }
 
 // PutObject - caches the uploaded object for single Put operations
 func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	putObjectFn := c.PutObjectFn
-	data := r.rawReader
 	dcache, err := c.getCacheToLoc(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
@@ -513,52 +511,24 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 		dcache.Delete(ctx, bucket, object)
 		return putObjectFn(ctx, bucket, object, r, opts)
 	}
-	// Initialize pipe to stream data to backend
-	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, size, "", "", data.ActualSize(), globalCLIContext.StrictS3Compat)
-	if err != nil {
-		return ObjectInfo{}, err
-	}
-	// Initialize pipe to stream data to cache
-	rPipe, wPipe := io.Pipe()
 
-	oinfoCh := make(chan ObjectInfo)
-	errCh := make(chan error)
-	go func() {
-		oinfo, perr := putObjectFn(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), opts)
-		if perr != nil {
-			pipeWriter.CloseWithError(perr)
-			wPipe.CloseWithError(perr)
-			close(oinfoCh)
-			errCh <- perr
-			return
-		}
-		close(errCh)
-		oinfoCh <- oinfo
-	}()
-	// get a namespace lock on cache until cache is filled.
-	cLock := c.nsMutex.NewNSLock(ctx, bucket, object)
-	if err := cLock.GetLock(globalObjectTimeout); err != nil {
-		return ObjectInfo{}, err
-	}
-	defer cLock.Unlock()
-	go func() {
-		if err = dcache.Put(ctx, bucket, object, rPipe, data.Size(), opts); err != nil {
-			wPipe.CloseWithError(err)
-			return
-		}
-	}()
+	objInfo, err = putObjectFn(ctx, bucket, object, r, opts)
 
-	mwriter := io.MultiWriter(pipeWriter, wPipe)
-	_, err = io.Copy(mwriter, data)
-	if err != nil {
-		err = <-errCh
-		return objInfo, err
+	if err == nil {
+		go func() {
+			// fill cache in the background
+			bReader, bErr := c.GetObjectNInfoFn(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})
+			if bErr != nil {
+				return
+			}
+			defer bReader.Close()
+			oi, err := c.stat(ctx, dcache, bucket, object)
+			// avoid cache overwrite if another background routine filled cache
+			if err != nil || oi.ETag != bReader.ObjInfo.ETag {
+				c.put(ctx, dcache, bucket, object, bReader, bReader.ObjInfo.Size, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)})
+			}
+		}()
 	}
-	pipeWriter.Close()
-	wPipe.Close()
-	objInfo = <-oinfoCh
-	dcache.updateETag(ctx, bucket, object, objInfo.ETag)
 
 	return objInfo, err
 

cmd/encryption-v1.go

@@ -32,7 +32,6 @@ import (
 	"github.com/minio/minio-go/v6/pkg/encrypt"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/ioutil"
 	sha256 "github.com/minio/sha256-simd"
 	"github.com/minio/sio"
 )
@@ -64,23 +63,6 @@ const (
 
 )
 
-const (
-	// SSESealAlgorithmDareSha256 specifies DARE as authenticated en/decryption scheme and SHA256 as cryptographic
-	// hash function. The key derivation of DARE-SHA256 is not optimal and does not include the object path.
-	// It is considered legacy and should not be used anymore.
-	SSESealAlgorithmDareSha256 = "DARE-SHA256"
-
-	// SSESealAlgorithmDareV2HmacSha256 specifies DAREv2 as authenticated en/decryption scheme and SHA256 as cryptographic
-	// hash function for the HMAC PRF.
-	SSESealAlgorithmDareV2HmacSha256 = "DAREv2-HMAC-SHA256"
-)
-
-// hasServerSideEncryptionHeader returns true if the given HTTP header
-// contains server-side-encryption.
-func hasServerSideEncryptionHeader(header http.Header) bool {
-	return crypto.S3.IsRequested(header) || crypto.SSEC.IsRequested(header)
-}
-
 // isEncryptedMultipart returns true if the current object is
 // uploaded by the user using multipart mechanism:
 // initiate new multipart, upload part, complete upload
@@ -597,236 +579,6 @@ func (d *DecryptBlocksReader) Read(p []byte) (int, error) {
 	return len(p), nil
 }
 
-// DecryptBlocksWriter - decrypts  multipart parts, while implementing
-// a io.Writer compatible interface.
-type DecryptBlocksWriter struct {
-	// Original writer where the plain data will be written
-	writer io.Writer
-	// Current decrypter for the current encrypted data block
-	decrypter io.WriteCloser
-	// Start sequence number
-	startSeqNum uint32
-	// Current part index
-	partIndex int
-	// Parts information
-	parts          []ObjectPartInfo
-	req            *http.Request
-	bucket, object string
-	metadata       map[string]string
-
-	partEncRelOffset int64
-
-	copySource bool
-	// Customer Key
-	customerKeyHeader string
-}
-
-func (w *DecryptBlocksWriter) buildDecrypter(partID int) error {
-	m := make(map[string]string)
-	for k, v := range w.metadata {
-		m[k] = v
-	}
-	// Initialize the first decrypter, new decrypters will be initialized in Write() operation as needed.
-	var key []byte
-	var err error
-	if w.copySource {
-		if crypto.SSEC.IsEncrypted(w.metadata) {
-			w.req.Header.Set(crypto.SSECopyKey, w.customerKeyHeader)
-			key, err = ParseSSECopyCustomerRequest(w.req.Header, w.metadata)
-		}
-	} else {
-		if crypto.SSEC.IsEncrypted(w.metadata) {
-			w.req.Header.Set(crypto.SSECKey, w.customerKeyHeader)
-			key, err = ParseSSECustomerRequest(w.req)
-		}
-	}
-	if err != nil {
-		return err
-	}
-
-	objectEncryptionKey, err := decryptObjectInfo(key, w.bucket, w.object, m)
-	if err != nil {
-		return err
-	}
-
-	var partIDbin [4]byte
-	binary.LittleEndian.PutUint32(partIDbin[:], uint32(partID)) // marshal part ID
-
-	mac := hmac.New(sha256.New, objectEncryptionKey) // derive part encryption key from part ID and object key
-	mac.Write(partIDbin[:])
-	partEncryptionKey := mac.Sum(nil)
-
-	// make sure to provide a NopCloser such that a Close
-	// on sio.decryptWriter doesn't close the underlying writer's
-	// close which perhaps can close the stream prematurely.
-	decrypter, err := newDecryptWriterWithObjectKey(ioutil.NopCloser(w.writer), partEncryptionKey, w.startSeqNum, m)
-	if err != nil {
-		return err
-	}
-
-	if w.decrypter != nil {
-		// Pro-actively close the writer such that any pending buffers
-		// are flushed already before we allocate a new decrypter.
-		err = w.decrypter.Close()
-		if err != nil {
-			return err
-		}
-	}
-
-	w.decrypter = decrypter
-	return nil
-}
-
-func (w *DecryptBlocksWriter) Write(p []byte) (int, error) {
-	var err error
-	var n1 int
-	if int64(len(p)) < w.parts[w.partIndex].Size-w.partEncRelOffset {
-		n1, err = w.decrypter.Write(p)
-		if err != nil {
-			return 0, err
-		}
-		w.partEncRelOffset += int64(n1)
-	} else {
-		n1, err = w.decrypter.Write(p[:w.parts[w.partIndex].Size-w.partEncRelOffset])
-		if err != nil {
-			return 0, err
-		}
-
-		// We should now proceed to next part, reset all values appropriately.
-		w.partEncRelOffset = 0
-		w.startSeqNum = 0
-
-		w.partIndex++
-
-		err = w.buildDecrypter(w.partIndex + 1)
-		if err != nil {
-			return 0, err
-		}
-
-		n1, err = w.decrypter.Write(p[n1:])
-		if err != nil {
-			return 0, err
-		}
-
-		w.partEncRelOffset += int64(n1)
-	}
-
-	return len(p), nil
-}
-
-// Close closes the LimitWriter. It behaves like io.Closer.
-func (w *DecryptBlocksWriter) Close() error {
-	if w.decrypter != nil {
-		err := w.decrypter.Close()
-		if err != nil {
-			return err
-		}
-	}
-
-	if closer, ok := w.writer.(io.Closer); ok {
-		return closer.Close()
-	}
-	return nil
-}
-
-// DecryptAllBlocksCopyRequest - setup a struct which can decrypt many concatenated encrypted data
-// parts information helps to know the boundaries of each encrypted data block, this function decrypts
-// all parts starting from part-1.
-func DecryptAllBlocksCopyRequest(client io.Writer, r *http.Request, bucket, object string, objInfo ObjectInfo) (io.WriteCloser, int64, error) {
-	w, _, size, err := DecryptBlocksRequest(client, r, bucket, object, 0, objInfo.Size, objInfo, true)
-	return w, size, err
-}
-
-// DecryptBlocksRequest - setup a struct which can decrypt many concatenated encrypted data
-// parts information helps to know the boundaries of each encrypted data block.
-func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object string, startOffset, length int64, objInfo ObjectInfo, copySource bool) (io.WriteCloser, int64, int64, error) {
-	var seqNumber uint32
-	var encStartOffset, encLength int64
-
-	if !isEncryptedMultipart(objInfo) {
-		seqNumber, encStartOffset, encLength = getEncryptedSinglePartOffsetLength(startOffset, length, objInfo)
-
-		var writer io.WriteCloser
-		var err error
-		if copySource {
-			writer, err = DecryptCopyRequest(client, r, bucket, object, objInfo.UserDefined)
-		} else {
-			writer, err = DecryptRequestWithSequenceNumber(client, r, bucket, object, seqNumber, objInfo.UserDefined)
-		}
-		if err != nil {
-			return nil, 0, 0, err
-		}
-		return writer, encStartOffset, encLength, nil
-	}
-
-	_, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
-
-	var partStartIndex int
-	var partStartOffset = startOffset
-	// Skip parts until final offset maps to a particular part offset.
-	for i, part := range objInfo.Parts {
-		decryptedSize, err := sio.DecryptedSize(uint64(part.Size))
-		if err != nil {
-			return nil, -1, -1, errObjectTampered
-		}
-
-		partStartIndex = i
-
-		// Offset is smaller than size we have reached the
-		// proper part offset, break out we start from
-		// this part index.
-		if partStartOffset < int64(decryptedSize) {
-			break
-		}
-
-		// Continue to look for next part.
-		partStartOffset -= int64(decryptedSize)
-	}
-
-	startSeqNum := partStartOffset / SSEDAREPackageBlockSize
-	partEncRelOffset := int64(startSeqNum) * (SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
-
-	w := &DecryptBlocksWriter{
-		writer:            client,
-		startSeqNum:       uint32(startSeqNum),
-		partEncRelOffset:  partEncRelOffset,
-		parts:             objInfo.Parts,
-		partIndex:         partStartIndex,
-		req:               r,
-		bucket:            bucket,
-		object:            object,
-		customerKeyHeader: r.Header.Get(crypto.SSECKey),
-		copySource:        copySource,
-	}
-
-	w.metadata = map[string]string{}
-	// Copy encryption metadata for internal use.
-	for k, v := range objInfo.UserDefined {
-		w.metadata[k] = v
-	}
-
-	// Purge all the encryption headers.
-	delete(objInfo.UserDefined, crypto.SSEIV)
-	delete(objInfo.UserDefined, crypto.SSESealAlgorithm)
-	delete(objInfo.UserDefined, crypto.SSECSealedKey)
-	delete(objInfo.UserDefined, crypto.SSEMultipart)
-
-	if crypto.S3.IsEncrypted(objInfo.UserDefined) {
-		delete(objInfo.UserDefined, crypto.S3SealedKey)
-		delete(objInfo.UserDefined, crypto.S3KMSKeyID)
-		delete(objInfo.UserDefined, crypto.S3KMSSealedKey)
-	}
-	if w.copySource {
-		w.customerKeyHeader = r.Header.Get(crypto.SSECopyKey)
-	}
-
-	if err := w.buildDecrypter(w.parts[w.partIndex].Number); err != nil {
-		return nil, 0, 0, err
-	}
-
-	return w, encStartOffset, encLength, nil
-}
-
 // getEncryptedMultipartsOffsetLength - fetch sequence number, encrypted start offset and encrypted length.
 func getEncryptedMultipartsOffsetLength(offset, length int64, obj ObjectInfo) (uint32, int64, int64) {
 	// Calculate encrypted offset of a multipart object

cmd/encryption-v1_test.go

@@ -28,86 +28,6 @@ import (
 	"github.com/minio/sio"
 )
 
-var hasServerSideEncryptionHeaderTests = []struct {
-	headers    map[string]string
-	sseRequest bool
-}{
-	{headers: map[string]string{crypto.SSECAlgorithm: "AES256", crypto.SSECKey: "key", crypto.SSECKeyMD5: "md5"}, sseRequest: true},                             // 0
-	{headers: map[string]string{crypto.SSECAlgorithm: "AES256"}, sseRequest: true},                                                                              // 1
-	{headers: map[string]string{crypto.SSECKey: "key"}, sseRequest: true},                                                                                       // 2
-	{headers: map[string]string{crypto.SSECKeyMD5: "md5"}, sseRequest: true},                                                                                    // 3
-	{headers: map[string]string{}, sseRequest: false},                                                                                                           // 4
-	{headers: map[string]string{crypto.SSECopyAlgorithm + " ": "AES256", " " + crypto.SSECopyKey: "key", crypto.SSECopyKeyMD5 + " ": "md5"}, sseRequest: false}, // 5
-	{headers: map[string]string{crypto.SSECopyAlgorithm: "", crypto.SSECopyKey: "", crypto.SSECopyKeyMD5: ""}, sseRequest: false},                               // 6
-	{headers: map[string]string{crypto.SSEHeader: ""}, sseRequest: true},                                                                                        // 7
-}
-
-func TestHasServerSideEncryptionHeader(t *testing.T) {
-	for i, test := range hasServerSideEncryptionHeaderTests {
-		headers := http.Header{}
-		for k, v := range test.headers {
-			headers.Set(k, v)
-		}
-		if hasServerSideEncryptionHeader(headers) != test.sseRequest {
-			t.Errorf("Test %d: Expected hasServerSideEncryptionHeader to return %v", i, test.sseRequest)
-		}
-	}
-}
-
-var hasSSECopyCustomerHeaderTests = []struct {
-	headers    map[string]string
-	sseRequest bool
-}{
-	{headers: map[string]string{crypto.SSECopyAlgorithm: "AES256", crypto.SSECopyKey: "key", crypto.SSECopyKeyMD5: "md5"}, sseRequest: true},                    // 0
-	{headers: map[string]string{crypto.SSECopyAlgorithm: "AES256"}, sseRequest: true},                                                                           // 1
-	{headers: map[string]string{crypto.SSECopyKey: "key"}, sseRequest: true},                                                                                    // 2
-	{headers: map[string]string{crypto.SSECopyKeyMD5: "md5"}, sseRequest: true},                                                                                 // 3
-	{headers: map[string]string{}, sseRequest: false},                                                                                                           // 4
-	{headers: map[string]string{crypto.SSECopyAlgorithm + " ": "AES256", " " + crypto.SSECopyKey: "key", crypto.SSECopyKeyMD5 + " ": "md5"}, sseRequest: false}, // 5
-	{headers: map[string]string{crypto.SSECopyAlgorithm: "", crypto.SSECopyKey: "", crypto.SSECopyKeyMD5: ""}, sseRequest: true},                                // 6
-	{headers: map[string]string{crypto.SSEHeader: ""}, sseRequest: false},                                                                                       // 7
-
-}
-
-func TestIsSSECopyCustomerRequest(t *testing.T) {
-	for i, test := range hasSSECopyCustomerHeaderTests {
-		headers := http.Header{}
-		for k, v := range test.headers {
-			headers.Set(k, v)
-		}
-		if crypto.SSECopy.IsRequested(headers) != test.sseRequest {
-			t.Errorf("Test %d: Expected crypto.SSECopy.IsRequested to return %v", i, test.sseRequest)
-		}
-	}
-}
-
-var hasSSECustomerHeaderTests = []struct {
-	headers    map[string]string
-	sseRequest bool
-}{
-	{headers: map[string]string{crypto.SSECAlgorithm: "AES256", crypto.SSECKey: "key", crypto.SSECKeyMD5: "md5"}, sseRequest: true},                    // 0
-	{headers: map[string]string{crypto.SSECAlgorithm: "AES256"}, sseRequest: true},                                                                     // 1
-	{headers: map[string]string{crypto.SSECKey: "key"}, sseRequest: true},                                                                              // 2
-	{headers: map[string]string{crypto.SSECKeyMD5: "md5"}, sseRequest: true},                                                                           // 3
-	{headers: map[string]string{}, sseRequest: false},                                                                                                  // 4
-	{headers: map[string]string{crypto.SSECAlgorithm + " ": "AES256", " " + crypto.SSECKey: "key", crypto.SSECKeyMD5 + " ": "md5"}, sseRequest: false}, // 5
-	{headers: map[string]string{crypto.SSECAlgorithm: "", crypto.SSECKey: "", crypto.SSECKeyMD5: ""}, sseRequest: true},                                // 6
-	{headers: map[string]string{crypto.SSEHeader: ""}, sseRequest: false},                                                                              // 7
-
-}
-
-func TestHasSSECustomerHeader(t *testing.T) {
-	for i, test := range hasSSECustomerHeaderTests {
-		headers := http.Header{}
-		for k, v := range test.headers {
-			headers.Set(k, v)
-		}
-		if crypto.SSEC.IsRequested(headers) != test.sseRequest {
-			t.Errorf("Test %d: Expected hasSSECustomerHeader to return %v", i, test.sseRequest)
-		}
-	}
-}
-
 var encryptRequestTests = []struct {
 	header   map[string]string
 	metadata map[string]string
@@ -173,7 +93,7 @@ var decryptRequestTests = []struct {
 			crypto.SSECKeyMD5:    "7PpPLAK26ONlVUGOWlusfg==",
 		},
 		metadata: map[string]string{
-			crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256,
+			crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm,
 			crypto.SSEIV:            "7nQqotA8xgrPx6QK7Ap3GCfjKitqJSrGP7xzgErSJlw=",
 			crypto.SSECSealedKey:    "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA==",
 		},
@@ -188,7 +108,7 @@ var decryptRequestTests = []struct {
 			crypto.SSECKeyMD5:    "7PpPLAK26ONlVUGOWlusfg==",
 		},
 		metadata: map[string]string{
-			crypto.SSESealAlgorithm: SSESealAlgorithmDareV2HmacSha256,
+			crypto.SSESealAlgorithm: crypto.SealAlgorithm,
 			crypto.SSEIV:            "qEqmsONcorqlcZXJxaw32H04eyXyXwUgjHzlhkaIYrU=",
 			crypto.SSECSealedKey:    "IAAfAIM14ugTGcM/dIrn4iQMrkl1sjKyeBQ8FBEvRebYj8vWvxG+0cJRpC6NXRU1wJN50JaUOATjO7kz0wZ2mA==",
 		},
@@ -218,7 +138,7 @@ var decryptRequestTests = []struct {
 			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
 		},
 		metadata: map[string]string{
-			crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256,
+			crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm,
 			crypto.SSEIV:            "RrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
 			crypto.SSECSealedKey:    "SY5E9AvI2tI7/nUrUAssIGE32Hcs4rR9z/CUuPqu5N4=",
 		},
@@ -233,7 +153,7 @@ var decryptRequestTests = []struct {
 			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
 		},
 		metadata: map[string]string{
-			crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256,
+			crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm,
 			crypto.SSEIV:            "XAm0dRrJsEsyPb1UuFNezv1bl9ehxuYsgUVC/MUctE2k=",
 			crypto.SSECSealedKey:    "SY5E9AvI2tI7/nUrUAssIGE32Hds4rR9z/CUuPqu5N4=",
 		},
@@ -248,7 +168,7 @@ var decryptRequestTests = []struct {
 			crypto.SSECKeyMD5:    "7PpPLAK26ONlVUGOWlusfg==",
 		},
 		metadata: map[string]string{
-			crypto.SSESealAlgorithm: SSESealAlgorithmDareV2HmacSha256,
+			crypto.SSESealAlgorithm: crypto.SealAlgorithm,
 			crypto.SSEIV:            "qEqmsONcorqlcZXJxaw32H04eyXyXwUgjHzlhkaIYrU=",
 			crypto.SSECSealedKey:    "IAAfAIM14ugTGcM/dIrn4iQMrkl1sjKyeBQ8FBEvRebYj8vWvxG+0cJRpC6NXRU1wJN50JaUOATjO7kz0wZ2mA==",
 		},
@@ -298,12 +218,12 @@ var decryptObjectInfoTests = []struct {
 		expErr:  nil,
 	},
 	{
-		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256}},
+		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
 		headers: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}},
 		expErr:  nil,
 	},
 	{
-		info:    ObjectInfo{Size: 0, UserDefined: map[string]string{crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256}},
+		info:    ObjectInfo{Size: 0, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
 		headers: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}},
 		expErr:  nil,
 	},
@@ -318,7 +238,7 @@ var decryptObjectInfoTests = []struct {
 		expErr:  errInvalidEncryptionParameters,
 	},
 	{
-		info:    ObjectInfo{Size: 31, UserDefined: map[string]string{crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256}},
+		info:    ObjectInfo{Size: 31, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
 		headers: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}},
 		expErr:  errObjectTampered,
 	},
@@ -408,7 +328,7 @@ func TestGetDecryptedRange(t *testing.T) {
 		}
 		udMap = func(isMulti bool) map[string]string {
 			m := map[string]string{
-				crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256,
+				crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm,
 				crypto.SSEMultipart:     "1",
 			}
 			if !isMulti {

cmd/endpoint.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,7 +20,6 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"net/http"
 	"net/url"
 	"os"
 	"path"
@@ -34,9 +33,6 @@ import (
 	"github.com/minio/cli"
 	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/cpu"
-	"github.com/minio/minio/pkg/disk"
-	"github.com/minio/minio/pkg/mem"
 	"github.com/minio/minio/pkg/mountinfo"
 )
 
@@ -287,89 +283,6 @@ func (endpoints EndpointList) UpdateIsLocal() error {
 
 }
 
-// localEndpointsMemUsage - returns ServerMemUsageInfo for only the
-// local endpoints from given list of endpoints
-func localEndpointsMemUsage(endpoints EndpointList, r *http.Request) ServerMemUsageInfo {
-	var memUsages []mem.Usage
-	var historicUsages []mem.Usage
-	scratchSpace := map[string]bool{}
-	for _, endpoint := range endpoints {
-		// Only proceed for local endpoints
-		if endpoint.IsLocal {
-			if _, ok := scratchSpace[endpoint.Host]; ok {
-				continue
-			}
-			memUsages = append(memUsages, mem.GetUsage())
-			historicUsages = append(historicUsages, mem.GetHistoricUsage())
-			scratchSpace[endpoint.Host] = true
-		}
-	}
-	addr := r.Host
-	if globalIsDistXL {
-		addr = GetLocalPeer(endpoints)
-	}
-	return ServerMemUsageInfo{
-		Addr:          addr,
-		Usage:         memUsages,
-		HistoricUsage: historicUsages,
-	}
-}
-
-// localEndpointsCPULoad - returns ServerCPULoadInfo for only the
-// local endpoints from given list of endpoints
-func localEndpointsCPULoad(endpoints EndpointList, r *http.Request) ServerCPULoadInfo {
-	var cpuLoads []cpu.Load
-	var historicLoads []cpu.Load
-	scratchSpace := map[string]bool{}
-	for _, endpoint := range endpoints {
-		// Only proceed for local endpoints
-		if endpoint.IsLocal {
-			if _, ok := scratchSpace[endpoint.Host]; ok {
-				continue
-			}
-			cpuLoads = append(cpuLoads, cpu.GetLoad())
-			historicLoads = append(historicLoads, cpu.GetHistoricLoad())
-			scratchSpace[endpoint.Host] = true
-		}
-	}
-	addr := r.Host
-	if globalIsDistXL {
-		addr = GetLocalPeer(endpoints)
-	}
-	return ServerCPULoadInfo{
-		Addr:         addr,
-		Load:         cpuLoads,
-		HistoricLoad: historicLoads,
-	}
-}
-
-// localEndpointsDrivePerf - returns ServerDrivesPerfInfo for only the
-// local endpoints from given list of endpoints
-func localEndpointsDrivePerf(endpoints EndpointList, r *http.Request) ServerDrivesPerfInfo {
-	var dps []disk.Performance
-	for _, endpoint := range endpoints {
-		// Only proceed for local endpoints
-		if endpoint.IsLocal {
-			if _, err := os.Stat(endpoint.Path); err != nil {
-				// Since this drive is not available, add relevant details and proceed
-				dps = append(dps, disk.Performance{Path: endpoint.Path, Error: err.Error()})
-				continue
-			}
-			dp := disk.GetPerformance(pathJoin(endpoint.Path, minioMetaTmpBucket, mustGetUUID()))
-			dp.Path = endpoint.Path
-			dps = append(dps, dp)
-		}
-	}
-	addr := r.Host
-	if globalIsDistXL {
-		addr = GetLocalPeer(endpoints)
-	}
-	return ServerDrivesPerfInfo{
-		Addr: addr,
-		Perf: dps,
-	}
-}
-
 // NewEndpointList - returns new endpoint list based on input args.
 func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
 	var endpointType EndpointType

cmd/format-meta.go

@@ -20,9 +20,6 @@ package cmd
 const (
 	// Format config file carries backend format specific details.
 	formatConfigFile = "format.json"
-
-	// Format config tmp file carries backend format.
-	formatConfigFileTmp = "format.json.tmp"
 )
 
 const (

cmd/format-xl.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -291,16 +292,6 @@ func formatXLMigrateV2ToV3(export string) error {
 	return ioutil.WriteFile(formatPath, b, 0644)
 }
 
-// Returns true, if one of the errors is non-nil and is Unformatted disk.
-func hasAnyErrorsUnformatted(errs []error) bool {
-	for _, err := range errs {
-		if err != nil && err == errUnformattedDisk {
-			return true
-		}
-	}
-	return false
-}
-
 // countErrs - count a specific error.
 func countErrs(errs []error, err error) int {
 	var i = 0
@@ -325,7 +316,7 @@ func quorumUnformattedDisks(errs []error) bool {
 // loadFormatXLAll - load all format config from all input disks in parallel.
 func loadFormatXLAll(storageDisks []StorageAPI) ([]*formatXLV3, []error) {
 	// Initialize sync waitgroup.
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 
 	// Initialize list of errors.
 	var sErrs = make([]error, len(storageDisks))
@@ -367,16 +358,18 @@ func saveFormatXL(disk StorageAPI, format interface{}) error {
 		return err
 	}
 
+	tmpFormatJSON := mustGetUUID() + ".json"
+
 	// Purge any existing temporary file, okay to ignore errors here.
-	defer disk.DeleteFile(minioMetaBucket, formatConfigFileTmp)
+	defer disk.DeleteFile(minioMetaBucket, tmpFormatJSON)
 
 	// Append file `format.json.tmp`.
-	if err = disk.AppendFile(minioMetaBucket, formatConfigFileTmp, formatBytes); err != nil {
+	if err = disk.WriteAll(minioMetaBucket, tmpFormatJSON, bytes.NewReader(formatBytes)); err != nil {
 		return err
 	}
 
-	// Rename file `format.json.tmp` --> `format.json`.
-	return disk.RenameFile(minioMetaBucket, formatConfigFileTmp, minioMetaBucket, formatConfigFile)
+	// Rename file `uuid.json` --> `format.json`.
+	return disk.RenameFile(minioMetaBucket, tmpFormatJSON, minioMetaBucket, formatConfigFile)
 }
 
 var ignoredHiddenDirectories = []string{
@@ -652,7 +645,7 @@ func formatXLV3Check(reference *formatXLV3, format *formatXLV3) error {
 func saveFormatXLAll(ctx context.Context, storageDisks []StorageAPI, formats []*formatXLV3) error {
 	var errs = make([]error, len(storageDisks))
 
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 
 	// Write `format.json` to all disks.
 	for index, disk := range storageDisks {
@@ -812,7 +805,7 @@ func initFormatXLMetaVolume(storageDisks []StorageAPI, formats []*formatXLV3) er
 	// This happens for the first time, but keep this here since this
 	// is the only place where it can be made expensive optimizing all
 	// other calls. Create minio meta volume, if it doesn't exist yet.
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 
 	// Initialize errs to collect errors inside go-routine.
 	var errs = make([]error, len(storageDisks))

cmd/fs-v1-multipart_test.go

@@ -95,7 +95,6 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 	defer os.RemoveAll(disk)
 	obj := initFSObjects(disk, t)
 
-	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 	objectName := "object"
 	data := []byte("12345")
@@ -105,7 +104,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -113,10 +112,13 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 	md5Hex := getMD5Hash(data)
 	sha256sum := ""
 
-	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	_, err = fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum), ObjectOptions{})
-	if !isSameType(err, BucketNotFound{}) {
-		t.Fatal("Unexpected error ", err)
+	newDisk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer os.RemoveAll(newDisk)
+	obj = initFSObjects(newDisk, t)
+	if _, err = obj.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum), ObjectOptions{}); err != nil {
+		if !isSameType(err, BucketNotFound{}) {
+			t.Fatal("Unexpected error ", err)
+		}
 	}
 }
 
@@ -127,7 +129,6 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 	defer os.RemoveAll(disk)
 	obj := initFSObjects(disk, t)
 
-	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 	objectName := "object"
 	data := []byte("12345")
@@ -136,7 +137,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -144,8 +145,10 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 	md5Hex := getMD5Hash(data)
 
 	parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
-	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	if _, err := fs.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
+	newDisk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer os.RemoveAll(newDisk)
+	obj = initFSObjects(newDisk, t)
+	if _, err := obj.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}
@@ -159,7 +162,6 @@ func TestCompleteMultipartUpload(t *testing.T) {
 	defer os.RemoveAll(disk)
 	obj := initFSObjects(disk, t)
 
-	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 	objectName := "object"
 	data := []byte("12345")
@@ -168,20 +170,19 @@ func TestCompleteMultipartUpload(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
 	md5Hex := getMD5Hash(data)
 
-	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
+	if _, err := obj.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
 	parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
-
-	if _, err := fs.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
+	if _, err := obj.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 }
@@ -193,7 +194,6 @@ func TestAbortMultipartUpload(t *testing.T) {
 	defer os.RemoveAll(disk)
 	obj := initFSObjects(disk, t)
 
-	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 	objectName := "object"
 	data := []byte("12345")
@@ -202,18 +202,18 @@ func TestAbortMultipartUpload(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
 	md5Hex := getMD5Hash(data)
 
-	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
+	if _, err := obj.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 	time.Sleep(time.Second) // Without Sleep on windows, the fs.AbortMultipartUpload() fails with "The process cannot access the file because it is being used by another process."
-	if err := fs.AbortMultipartUpload(context.Background(), bucketName, objectName, uploadID); err != nil {
+	if err := obj.AbortMultipartUpload(context.Background(), bucketName, objectName, uploadID); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 }
@@ -226,7 +226,6 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 
 	obj := initFSObjects(disk, t)
 
-	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 	objectName := "object"
 
@@ -234,13 +233,15 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	_, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
+	_, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
-	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	if _, err := fs.ListMultipartUploads(context.Background(), bucketName, objectName, "", "", "", 1000); err != nil {
+	newDisk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer os.RemoveAll(newDisk)
+	obj = initFSObjects(newDisk, t)
+	if _, err := obj.ListMultipartUploads(context.Background(), bucketName, objectName, "", "", "", 1000); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}

cmd/gateway-common.go

@@ -24,6 +24,7 @@ import (
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
+	xnet "github.com/minio/minio/pkg/net"
 
 	minio "github.com/minio/minio-go/v6"
 )
@@ -299,7 +300,7 @@ func ErrorRespToObjectError(err error, params ...string) error {
 		object = params[1]
 	}
 
-	if isNetworkOrHostDown(err) {
+	if xnet.IsNetworkOrHostDown(err) {
 		return BackendDown{}
 	}
 

cmd/gateway-main.go

@@ -304,7 +304,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 		// Print a warning message if gateway is not ready for production before the startup banner.
 		if !gw.Production() {
-			logger.StartupMessage(colorYellow("               *** Warning: Not Ready for Production ***"))
+			logStartupMessage(colorYellow("               *** Warning: Not Ready for Production ***"))
 		}
 
 		// Print gateway startup message.

cmd/gateway-startup-msg.go

@@ -20,8 +20,6 @@ import (
 	"context"
 	"fmt"
 	"strings"
-
-	"github.com/minio/minio/cmd/logger"
 )
 
 // Prints the formatted startup message.
@@ -57,15 +55,15 @@ func printGatewayCommonMsg(apiEndpoints []string) {
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")
 
 	// Colorize the message and print.
-	logger.StartupMessage(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
+	logStartupMessage(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
 	if isTerminal() && !globalCLIContext.Anonymous {
-		logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
-		logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
+		logStartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logStartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
 	}
 	printEventNotifiers()
 
 	if globalIsBrowserEnabled {
-		logger.StartupMessage(colorBlue("\nBrowser Access:"))
-		logger.StartupMessage(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr))
+		logStartupMessage(colorBlue("\nBrowser Access:"))
+		logStartupMessage(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr))
 	}
 }

cmd/gateway/b2/gateway-b2.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"hash"
 	"io"
-	"io/ioutil"
 	"net/http"
 
 	"strings"
@@ -460,13 +459,23 @@ func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object str
 	if err != nil {
 		return objInfo, err
 	}
-	f, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
+
+	f, _, err := bkt.ListFileNames(l.ctx, 1, object, "", "")
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return objInfo, b2ToObjectError(err, bucket, object)
 	}
-	f.Close()
-	fi, err := bkt.File(f.ID, object).GetFileInfo(l.ctx)
+
+	// B2's list will return the next item in the bucket if the object doesn't
+	// exist so we need to perform a name check too
+	if len(f) != 1 || (len(f) == 1 && f[0].Name != object) {
+		return objInfo, minio.ObjectNotFound{
+			Bucket: bucket,
+			Object: object,
+		}
+	}
+
+	fi, err := bkt.File(f[0].ID, object).GetFileInfo(l.ctx)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return objInfo, b2ToObjectError(err, bucket, object)
@@ -474,7 +483,7 @@ func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object str
 	return minio.ObjectInfo{
 		Bucket:      bucket,
 		Name:        object,
-		ETag:        minio.ToS3ETag(f.ID),
+		ETag:        minio.ToS3ETag(f[0].ID),
 		Size:        fi.Size,
 		ModTime:     fi.Timestamp,
 		ContentType: fi.ContentType,
@@ -595,14 +604,10 @@ func (l *b2Objects) DeleteObject(ctx context.Context, bucket string, object stri
 	if err != nil {
 		return err
 	}
-	reader, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return b2ToObjectError(err, bucket, object)
-	}
-	io.Copy(ioutil.Discard, reader)
-	reader.Close()
-	err = bkt.File(reader.ID, object).DeleteFileVersion(l.ctx)
+
+	// If we hide the file we'll conform to B2's versioning policy, it also
+	// saves an additional call to check if the file exists first
+	_, err = bkt.HideFile(l.ctx, object)
 	logger.LogIf(ctx, err)
 	return b2ToObjectError(err, bucket, object)
 }

cmd/gateway/hdfs/gateway-hdfs.go

@@ -18,6 +18,7 @@ package hdfs
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net"
@@ -280,7 +281,7 @@ func hdfsToObjectErr(ctx context.Context, err error, params ...string) error {
 			return minio.PrefixAccessDenied{Bucket: bucket, Object: object}
 		}
 		return minio.BucketAlreadyOwnedByYou{Bucket: bucket}
-	case isSysErrNotEmpty(err):
+	case errors.Is(err, syscall.ENOTEMPTY):
 		if object != "" {
 			return minio.PrefixAccessDenied{Bucket: bucket, Object: object}
 		}
@@ -387,20 +388,6 @@ func (n *hdfsObjects) ListObjects(ctx context.Context, bucket, prefix, marker, d
 	return minio.ListObjects(ctx, n, bucket, prefix, marker, delimiter, maxKeys, n.listPool, n.listDirFactory(), getObjectInfo, getObjectInfo)
 }
 
-// Check if the given error corresponds to ENOTEMPTY for unix
-// and ERROR_DIR_NOT_EMPTY for windows (directory not empty).
-func isSysErrNotEmpty(err error) bool {
-	if err == syscall.ENOTEMPTY {
-		return true
-	}
-	if pathErr, ok := err.(*os.PathError); ok {
-		if pathErr.Err == syscall.ENOTEMPTY {
-			return true
-		}
-	}
-	return false
-}
-
 // deleteObject deletes a file path if its empty. If it's successfully deleted,
 // it will recursively move up the tree, deleting empty parent directories
 // until it finds one with files in it. Returns nil for a non-empty directory.
@@ -411,16 +398,13 @@ func (n *hdfsObjects) deleteObject(basePath, deletePath string) error {
 
 	// Attempt to remove path.
 	if err := n.clnt.Remove(deletePath); err != nil {
-		switch {
-		case err == syscall.ENOTEMPTY:
-		case isSysErrNotEmpty(err):
+		if errors.Is(err, syscall.ENOTEMPTY) {
 			// Ignore errors if the directory is not empty. The server relies on
 			// this functionality, and sometimes uses recursion that should not
 			// error on parent directories.
 			return nil
-		default:
-			return err
 		}
+		return err
 	}
 
 	// Trailing slash is removed when found to ensure
@@ -523,12 +507,34 @@ func (n *hdfsObjects) GetObject(ctx context.Context, bucket, key string, startOf
 	return hdfsToObjectErr(ctx, err, bucket, key)
 }
 
+func (n *hdfsObjects) isObjectDir(ctx context.Context, bucket, object string) bool {
+	f, err := n.clnt.Open(minio.PathJoin(hdfsSeparator, bucket, object))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false
+		}
+		logger.LogIf(ctx, err)
+		return false
+	}
+	defer f.Close()
+	fis, err := f.Readdir(1)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return false
+	}
+	return len(fis) == 0
+}
+
 // GetObjectInfo reads object info and replies back ObjectInfo.
 func (n *hdfsObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	_, err = n.clnt.Stat(minio.PathJoin(hdfsSeparator, bucket))
 	if err != nil {
 		return objInfo, hdfsToObjectErr(ctx, err, bucket)
 	}
+	if strings.HasSuffix(object, hdfsSeparator) && !n.isObjectDir(ctx, bucket, object) {
+		return objInfo, hdfsToObjectErr(ctx, os.ErrNotExist, bucket, object)
+	}
+
 	fi, err := n.clnt.Stat(minio.PathJoin(hdfsSeparator, bucket, object))
 	if err != nil {
 		return objInfo, hdfsToObjectErr(ctx, err, bucket, object)
@@ -552,7 +558,7 @@ func (n *hdfsObjects) PutObject(ctx context.Context, bucket string, object strin
 	name := minio.PathJoin(hdfsSeparator, bucket, object)
 
 	// If its a directory create a prefix {
-	if strings.HasSuffix(object, hdfsSeparator) {
+	if strings.HasSuffix(object, hdfsSeparator) && r.Size() == 0 {
 		if err = n.clnt.MkdirAll(name, os.FileMode(0755)); err != nil {
 			n.deleteObject(minio.PathJoin(hdfsSeparator, bucket), name)
 			return objInfo, hdfsToObjectErr(ctx, err, bucket, object)

cmd/generic-handlers.go

@@ -221,7 +221,7 @@ func guessIsMetricsReq(req *http.Request) bool {
 		return false
 	}
 	aType := getRequestAuthType(req)
-	return aType == authTypeAnonymous &&
+	return (aType == authTypeAnonymous || aType == authTypeJWT) &&
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsPath
 }
 

cmd/generic-handlers_test.go

@@ -185,7 +185,7 @@ var containsReservedMetadataTests = []struct {
 		shouldFail: true,
 	},
 	{
-		header:     http.Header{crypto.SSESealAlgorithm: []string{SSESealAlgorithmDareSha256}},
+		header:     http.Header{crypto.SSESealAlgorithm: []string{crypto.InsecureSealAlgorithm}},
 		shouldFail: true,
 	},
 	{

cmd/http-tracer.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"net"
@@ -30,11 +29,10 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/handlers"
 	trace "github.com/minio/minio/pkg/trace"
 )
 
-var traceBodyPlaceHolder = []byte("<BODY>")
-
 // recordRequest - records the first recLen bytes
 // of a given io.Reader
 type recordRequest struct {
@@ -77,81 +75,7 @@ func (r *recordRequest) Data() []byte {
 		return r.buf.Bytes()
 	}
 	// ... otherwise we return <BODY> placeholder
-	return traceBodyPlaceHolder
-}
-
-// recordResponseWriter - records the first recLen bytes
-// of a given http.ResponseWriter
-type recordResponseWriter struct {
-	// Data source to record
-	http.ResponseWriter
-	// Response body should be logged
-	logBody bool
-	// Internal recording buffer
-	headers bytes.Buffer
-	body    bytes.Buffer
-	// The status code of the current HTTP request
-	statusCode int
-	// Indicate if headers are written in the log
-	headersLogged bool
-	// number of bytes written
-	bytesWritten int
-}
-
-// Write the headers into the given buffer
-func (r *recordResponseWriter) writeHeaders(w io.Writer, statusCode int, headers http.Header) {
-	n, _ := fmt.Fprintf(w, "%d %s\n", statusCode, http.StatusText(statusCode))
-	r.bytesWritten += n
-	for k, v := range headers {
-		n, _ := fmt.Fprintf(w, "%s: %s\n", k, v[0])
-		r.bytesWritten += n
-	}
-}
-
-// Record the headers.
-func (r *recordResponseWriter) WriteHeader(i int) {
-	r.statusCode = i
-	if !r.headersLogged {
-		r.writeHeaders(&r.headers, i, r.ResponseWriter.Header())
-		r.headersLogged = true
-	}
-	r.ResponseWriter.WriteHeader(i)
-}
-
-func (r *recordResponseWriter) Write(p []byte) (n int, err error) {
-	n, err = r.ResponseWriter.Write(p)
-	r.bytesWritten += n
-	if !r.headersLogged {
-		// We assume the response code to be '200 OK' when WriteHeader() is not called,
-		// that way following Golang HTTP response behavior.
-		r.writeHeaders(&r.headers, http.StatusOK, r.ResponseWriter.Header())
-		r.headersLogged = true
-	}
-	if r.statusCode >= http.StatusBadRequest || r.logBody {
-		// Always logging error responses.
-		r.body.Write(p)
-	}
-	return n, err
-}
-
-func (r *recordResponseWriter) Size() int {
-	return r.bytesWritten
-}
-
-// Calls the underlying Flush.
-func (r *recordResponseWriter) Flush() {
-	r.ResponseWriter.(http.Flusher).Flush()
-}
-
-// Return response body.
-func (r *recordResponseWriter) Body() []byte {
-	// If there was an error response or body logging is enabled
-	// then we return the body contents
-	if r.statusCode >= http.StatusBadRequest || r.logBody {
-		return r.body.Bytes()
-	}
-	// ... otherwise we return the <BODY> place holder
-	return traceBodyPlaceHolder
+	return logger.BodyPlaceHolder
 }
 
 // getOpName sanitizes the operation name for mc
@@ -175,7 +99,7 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 	name := getOpName(runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name())
 
 	// Setup a http request body recorder
-	reqHeaders := cloneHeader(r.Header)
+	reqHeaders := r.Header.Clone()
 	reqHeaders.Set("Content-Length", strconv.Itoa(int(r.ContentLength)))
 	reqHeaders.Set("Host", r.Host)
 	for _, enc := range r.TransferEncoding {
@@ -200,20 +124,19 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 		Method:   r.Method,
 		Path:     r.URL.Path,
 		RawQuery: r.URL.RawQuery,
-		Client:   r.RemoteAddr,
+		Client:   handlers.GetSourceIP(r),
 		Headers:  reqHeaders,
 		Body:     reqBodyRecorder.Data(),
 	}
-
-	// Setup a http response body recorder
-	respBodyRecorder := &recordResponseWriter{ResponseWriter: w, logBody: logBody}
-	f(logger.NewResponseWriter(respBodyRecorder), r)
+	rw, _ := w.(*logger.ResponseWriter)
+	rw.LogBody = logBody
+	f(rw, r)
 
 	rs := trace.ResponseInfo{
 		Time:       time.Now().UTC(),
-		Headers:    cloneHeader(respBodyRecorder.Header()),
-		StatusCode: respBodyRecorder.statusCode,
-		Body:       respBodyRecorder.Body(),
+		Headers:    rw.Header().Clone(),
+		StatusCode: rw.StatusCode,
+		Body:       rw.Body(),
 	}
 
 	if rs.StatusCode == 0 {
@@ -224,9 +147,10 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 	t.RespInfo = rs
 
 	t.CallStats = trace.CallStats{
-		Latency:     rs.Time.Sub(rq.Time),
-		InputBytes:  reqBodyRecorder.Size(),
-		OutputBytes: respBodyRecorder.Size(),
+		Latency:         rs.Time.Sub(rw.StartTime),
+		InputBytes:      reqBodyRecorder.Size(),
+		OutputBytes:     rw.Size(),
+		TimeToFirstByte: rw.TimeToFirstByte,
 	}
 	return t
 }

cmd/http/accountingconn.go

@@ -0,0 +1,57 @@
+/*
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package http
+
+import (
+	"net"
+)
+
+// AccountingConn - is a generic stream-oriented network connection supporting buffered reader and read/write timeout.
+type AccountingConn struct {
+	net.Conn
+	updateBytesReadFunc    func(int) // function to be called to update bytes read.
+	updateBytesWrittenFunc func(int) // function to be called to update bytes written.
+}
+
+// Read - reads data from the connection using wrapped buffered reader.
+func (c *AccountingConn) Read(b []byte) (n int, err error) {
+	n, err = c.Conn.Read(b)
+	if err == nil && c.updateBytesReadFunc != nil {
+		c.updateBytesReadFunc(n)
+	}
+
+	return n, err
+}
+
+// Write - writes data to the connection.
+func (c *AccountingConn) Write(b []byte) (n int, err error) {
+	n, err = c.Conn.Write(b)
+	if err == nil && c.updateBytesWrittenFunc != nil {
+		c.updateBytesWrittenFunc(n)
+	}
+
+	return n, err
+}
+
+// newAccountingConn - creates a new connection object wrapping net.Conn with deadlines.
+func newAccountingConn(c net.Conn, updateBytesReadFunc, updateBytesWrittenFunc func(int)) *AccountingConn {
+	return &AccountingConn{
+		Conn:                   c,
+		updateBytesReadFunc:    updateBytesReadFunc,
+		updateBytesWrittenFunc: updateBytesWrittenFunc,
+	}
+}

cmd/http/deadlineconn.go

@@ -1,77 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package http
-
-import (
-	"net"
-	"time"
-)
-
-// DeadlineConn - is a generic stream-oriented network connection supporting buffered reader and read/write timeout.
-type DeadlineConn struct {
-	net.Conn
-	readTimeout            time.Duration // sets the read timeout in the connection.
-	writeTimeout           time.Duration // sets the write timeout in the connection.
-	updateBytesReadFunc    func(int)     // function to be called to update bytes read.
-	updateBytesWrittenFunc func(int)     // function to be called to update bytes written.
-}
-
-// Sets read timeout
-func (c *DeadlineConn) setReadTimeout() {
-	if c.readTimeout != 0 {
-		c.SetReadDeadline(time.Now().UTC().Add(c.readTimeout))
-	}
-}
-
-func (c *DeadlineConn) setWriteTimeout() {
-	if c.writeTimeout != 0 {
-		c.SetWriteDeadline(time.Now().UTC().Add(c.writeTimeout))
-	}
-}
-
-// Read - reads data from the connection using wrapped buffered reader.
-func (c *DeadlineConn) Read(b []byte) (n int, err error) {
-	c.setReadTimeout()
-	n, err = c.Conn.Read(b)
-	if err == nil && c.updateBytesReadFunc != nil {
-		c.updateBytesReadFunc(n)
-	}
-
-	return n, err
-}
-
-// Write - writes data to the connection.
-func (c *DeadlineConn) Write(b []byte) (n int, err error) {
-	c.setWriteTimeout()
-	n, err = c.Conn.Write(b)
-	if err == nil && c.updateBytesWrittenFunc != nil {
-		c.updateBytesWrittenFunc(n)
-	}
-
-	return n, err
-}
-
-// newDeadlineConn - creates a new connection object wrapping net.Conn with deadlines.
-func newDeadlineConn(c net.Conn, readTimeout, writeTimeout time.Duration, updateBytesReadFunc, updateBytesWrittenFunc func(int)) *DeadlineConn {
-	return &DeadlineConn{
-		Conn:                   c,
-		readTimeout:            readTimeout,
-		writeTimeout:           writeTimeout,
-		updateBytesReadFunc:    updateBytesReadFunc,
-		updateBytesWrittenFunc: updateBytesWrittenFunc,
-	}
-}

cmd/http/deadlineconn_test.go

@@ -1,114 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package http
-
-import (
-	"bufio"
-	"io"
-	"net"
-	"sync"
-	"testing"
-	"time"
-)
-
-// Test deadlineconn handles read timeout properly by reading two messages beyond deadline.
-func TestBuffConnReadTimeout(t *testing.T) {
-	l, err := net.Listen("tcp", "localhost:0")
-	if err != nil {
-		t.Fatalf("unable to create listener. %v", err)
-	}
-	defer l.Close()
-	serverAddr := l.Addr().String()
-
-	tcpListener, ok := l.(*net.TCPListener)
-	if !ok {
-		t.Fatalf("failed to assert to net.TCPListener")
-	}
-
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-
-		tcpConn, terr := tcpListener.AcceptTCP()
-		if terr != nil {
-			t.Errorf("failed to accept new connection. %v", terr)
-			return
-		}
-		deadlineconn := newDeadlineConn(tcpConn, 1*time.Second, 1*time.Second, nil, nil)
-		defer deadlineconn.Close()
-
-		// Read a line
-		var b = make([]byte, 12)
-		_, terr = deadlineconn.Read(b)
-		if terr != nil {
-			t.Errorf("failed to read from client. %v", terr)
-			return
-		}
-		received := string(b)
-		if received != "message one\n" {
-			t.Errorf(`server: expected: "message one\n", got: %v`, received)
-			return
-		}
-
-		// Wait for more than read timeout to simulate processing.
-		time.Sleep(3 * time.Second)
-
-		_, terr = deadlineconn.Read(b)
-		if terr != nil {
-			t.Errorf("failed to read from client. %v", terr)
-			return
-		}
-		received = string(b)
-		if received != "message two\n" {
-			t.Errorf(`server: expected: "message two\n", got: %v`, received)
-			return
-		}
-
-		// Send a response.
-		_, terr = io.WriteString(deadlineconn, "messages received\n")
-		if terr != nil {
-			t.Errorf("failed to write to client. %v", terr)
-			return
-		}
-	}()
-
-	c, err := net.Dial("tcp", serverAddr)
-	if err != nil {
-		t.Fatalf("unable to connect to server. %v", err)
-	}
-	defer c.Close()
-
-	_, err = io.WriteString(c, "message one\n")
-	if err != nil {
-		t.Fatalf("failed to write to server. %v", err)
-	}
-	_, err = io.WriteString(c, "message two\n")
-	if err != nil {
-		t.Fatalf("failed to write to server. %v", err)
-	}
-
-	received, err := bufio.NewReader(c).ReadString('\n')
-	if err != nil {
-		t.Fatalf("failed to read from server. %v", err)
-	}
-	if received != "messages received\n" {
-		t.Fatalf(`client: expected: "messages received\n", got: %v`, received)
-	}
-
-	wg.Wait()
-}

cmd/http/listener.go

@@ -38,8 +38,6 @@ type httpListener struct {
 	acceptCh               chan acceptResult  // channel where all TCP listeners write accepted connection.
 	doneCh                 chan struct{}      // done channel for TCP listener goroutines.
 	tcpKeepAliveTimeout    time.Duration
-	readTimeout            time.Duration
-	writeTimeout           time.Duration
 	updateBytesReadFunc    func(int) // function to be called to update bytes read in Deadlineconn.
 	updateBytesWrittenFunc func(int) // function to be called to update bytes written in Deadlineconn.
 }
@@ -91,10 +89,9 @@ func (listener *httpListener) start() {
 		tcpConn.SetKeepAlive(true)
 		tcpConn.SetKeepAlivePeriod(listener.tcpKeepAliveTimeout)
 
-		deadlineConn := newDeadlineConn(tcpConn, listener.readTimeout,
-			listener.writeTimeout, listener.updateBytesReadFunc, listener.updateBytesWrittenFunc)
+		acctConn := newAccountingConn(tcpConn, listener.updateBytesReadFunc, listener.updateBytesWrittenFunc)
 
-		send(acceptResult{deadlineConn, nil}, doneCh)
+		send(acceptResult{acctConn, nil}, doneCh)
 	}
 
 	// Closure to handle TCPListener until done channel is closed.
@@ -176,8 +173,6 @@ func (listener *httpListener) Addrs() (addrs []net.Addr) {
 // * controls incoming connections only doing HTTP protocol
 func newHTTPListener(serverAddrs []string,
 	tcpKeepAliveTimeout time.Duration,
-	readTimeout time.Duration,
-	writeTimeout time.Duration,
 	updateBytesReadFunc func(int),
 	updateBytesWrittenFunc func(int)) (listener *httpListener, err error) {
 
@@ -214,8 +209,6 @@ func newHTTPListener(serverAddrs []string,
 	listener = &httpListener{
 		tcpListeners:           tcpListeners,
 		tcpKeepAliveTimeout:    tcpKeepAliveTimeout,
-		readTimeout:            readTimeout,
-		writeTimeout:           writeTimeout,
 		updateBytesReadFunc:    updateBytesReadFunc,
 		updateBytesWrittenFunc: updateBytesWrittenFunc,
 	}

cmd/http/listener_test.go

@@ -154,8 +154,6 @@ func TestNewHTTPListener(t *testing.T) {
 		listener, err := newHTTPListener(
 			testCase.serverAddrs,
 			testCase.tcpKeepAliveTimeout,
-			testCase.readTimeout,
-			testCase.writeTimeout,
 			testCase.updateBytesReadFunc,
 			testCase.updateBytesWrittenFunc,
 		)
@@ -192,8 +190,6 @@ func TestHTTPListenerStartClose(t *testing.T) {
 		listener, err := newHTTPListener(
 			testCase.serverAddrs,
 			time.Duration(0),
-			time.Duration(0),
-			time.Duration(0),
 			nil, nil,
 		)
 		if err != nil {
@@ -235,8 +231,6 @@ func TestHTTPListenerAddr(t *testing.T) {
 		listener, err := newHTTPListener(
 			testCase.serverAddrs,
 			time.Duration(0),
-			time.Duration(0),
-			time.Duration(0),
 			nil, nil,
 		)
 		if err != nil {
@@ -275,8 +269,6 @@ func TestHTTPListenerAddrs(t *testing.T) {
 		listener, err := newHTTPListener(
 			testCase.serverAddrs,
 			time.Duration(0),
-			time.Duration(0),
-			time.Duration(0),
 			nil, nil,
 		)
 		if err != nil {

cmd/http/server.go

@@ -20,7 +20,6 @@ import (
 	"crypto/tls"
 	"errors"
 	"net/http"
-	"os"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -31,13 +30,6 @@ import (
 	"github.com/minio/minio/pkg/certs"
 )
 
-func init() {
-	// Opt-in to TLS 1.3. See: https://golang.org/pkg/crypto/tls
-	// In future Go versions TLS 1.3 probably gets enabled by default.
-	// So, we can remove this line as soon as this is the case.
-	os.Setenv("GODEBUG", os.Getenv("GODEBUG")+",tls13=1")
-}
-
 const (
 	serverShutdownPoll = 500 * time.Millisecond
 
@@ -47,12 +39,6 @@ const (
 	// DefaultTCPKeepAliveTimeout - default TCP keep alive timeout for accepted connection.
 	DefaultTCPKeepAliveTimeout = 30 * time.Second
 
-	// DefaultReadTimeout - default timout to read data from accepted connection.
-	DefaultReadTimeout = 5 * time.Minute
-
-	// DefaultWriteTimeout - default timout to write data to accepted connection.
-	DefaultWriteTimeout = 5 * time.Minute
-
 	// DefaultMaxHeaderBytes - default maximum HTTP header size in bytes.
 	DefaultMaxHeaderBytes = 1 * humanize.MiByte
 )
@@ -61,8 +47,6 @@ const (
 type Server struct {
 	http.Server
 	Addrs                  []string      // addresses on which the server listens for new connection.
-	ReadTimeout            time.Duration // timeout used for net.Conn.Read() deadlines.
-	WriteTimeout           time.Duration // timeout used for net.Conn.Write() deadlines.
 	ShutdownTimeout        time.Duration // timeout used for graceful server shutdown.
 	TCPKeepAliveTimeout    time.Duration // timeout used for underneath TCP connection.
 	UpdateBytesReadFunc    func(int)     // function to be called to update bytes read in bufConn.
@@ -85,8 +69,6 @@ func (srv *Server) Start() (err error) {
 	if srv.TLSConfig != nil {
 		tlsConfig = srv.TLSConfig.Clone()
 	}
-	readTimeout := srv.ReadTimeout
-	writeTimeout := srv.WriteTimeout
 	handler := srv.Handler // if srv.Handler holds non-synced state -> possible data race
 
 	addrs := set.CreateStringSet(srv.Addrs...).ToSlice() // copy and remove duplicates
@@ -97,8 +79,6 @@ func (srv *Server) Start() (err error) {
 	listener, err = newHTTPListener(
 		addrs,
 		tcpKeepAliveTimeout,
-		readTimeout,
-		writeTimeout,
 		srv.UpdateBytesReadFunc,
 		srv.UpdateBytesWrittenFunc,
 	)
@@ -221,8 +201,6 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
 	}
 	httpServer.Handler = handler
 	httpServer.TLSConfig = tlsConfig
-	httpServer.ReadTimeout = DefaultReadTimeout
-	httpServer.WriteTimeout = DefaultWriteTimeout
 	httpServer.MaxHeaderBytes = DefaultMaxHeaderBytes
 
 	return httpServer

cmd/http/server_test.go

@@ -77,14 +77,6 @@ func TestNewServer(t *testing.T) {
 			t.Fatalf("Case %v: server.TCPKeepAliveTimeout: expected: %v, got: %v", (i + 1), DefaultTCPKeepAliveTimeout, server.TCPKeepAliveTimeout)
 		}
 
-		if server.ReadTimeout != DefaultReadTimeout {
-			t.Fatalf("Case %v: server.ReadTimeout: expected: %v, got: %v", (i + 1), DefaultReadTimeout, server.ReadTimeout)
-		}
-
-		if server.WriteTimeout != DefaultWriteTimeout {
-			t.Fatalf("Case %v: server.WriteTimeout: expected: %v, got: %v", (i + 1), DefaultWriteTimeout, server.WriteTimeout)
-		}
-
 		if server.MaxHeaderBytes != DefaultMaxHeaderBytes {
 			t.Fatalf("Case %v: server.MaxHeaderBytes: expected: %v, got: %v", (i + 1), DefaultMaxHeaderBytes, server.MaxHeaderBytes)
 		}

cmd/iam-etcd-store.go

@@ -415,23 +415,35 @@ func (ies *IAMEtcdStore) loadAll(sys *IAMSys, objectAPI ObjectLayer) error {
 	iamUserPolicyMap := make(map[string]MappedPolicy)
 	iamGroupPolicyMap := make(map[string]MappedPolicy)
 
+	isMinIOUsersSys := false
+	sys.RLock()
+	if sys.usersSysType == MinIOUsersSysType {
+		isMinIOUsersSys = true
+	}
+	sys.RUnlock()
+
 	if err := ies.loadPolicyDocs(iamPolicyDocsMap); err != nil {
 		return err
 	}
-	if err := ies.loadUsers(false, iamUsersMap); err != nil {
-		return err
-	}
-	// load STS temp users into the same map
+
+	// load STS temp users
 	if err := ies.loadUsers(true, iamUsersMap); err != nil {
 		return err
 	}
-	if err := ies.loadGroups(iamGroupsMap); err != nil {
-		return err
+
+	if isMinIOUsersSys {
+		// load long term users
+		if err := ies.loadUsers(false, iamUsersMap); err != nil {
+			return err
+		}
+		if err := ies.loadGroups(iamGroupsMap); err != nil {
+			return err
+		}
+		if err := ies.loadMappedPolicies(false, false, iamUserPolicyMap); err != nil {
+			return err
+		}
 	}
 
-	if err := ies.loadMappedPolicies(false, false, iamUserPolicyMap); err != nil {
-		return err
-	}
 	// load STS policy mappings into the same map
 	if err := ies.loadMappedPolicies(true, false, iamUserPolicyMap); err != nil {
 		return err
@@ -491,28 +503,35 @@ func (ies *IAMEtcdStore) deleteGroupInfo(name string) error {
 
 func (ies *IAMEtcdStore) watch(sys *IAMSys) {
 	watchEtcd := func() {
-		// Refresh IAMSys with etcd watch.
 		for {
+		outerLoop:
+			// Refresh IAMSys with etcd watch.
 			watchCh := ies.client.Watch(context.Background(),
 				iamConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
-			select {
-			case <-GlobalServiceDoneCh:
-				return
-			case watchResp, ok := <-watchCh:
-				if !ok {
-					time.Sleep(1 * time.Second)
-					continue
-				}
-				if err := watchResp.Err(); err != nil {
-					logger.LogIf(context.Background(), err)
-					// log and retry.
-					time.Sleep(1 * time.Second)
-					continue
-				}
-				for _, event := range watchResp.Events {
-					sys.Lock()
-					ies.reloadFromEvent(sys, event)
-					sys.Unlock()
+			for {
+				select {
+				case <-GlobalServiceDoneCh:
+					return
+				case watchResp, ok := <-watchCh:
+					if !ok {
+						time.Sleep(1 * time.Second)
+						// Upon an error on watch channel
+						// re-init the watch channel.
+						goto outerLoop
+					}
+					if err := watchResp.Err(); err != nil {
+						logger.LogIf(context.Background(), err)
+						// log and retry.
+						time.Sleep(1 * time.Second)
+						// Upon an error on watch channel
+						// re-init the watch channel.
+						goto outerLoop
+					}
+					for _, event := range watchResp.Events {
+						sys.Lock()
+						ies.reloadFromEvent(sys, event)
+						sys.Unlock()
+					}
 				}
 			}
 		}

cmd/iam-object-store.go

@@ -429,24 +429,33 @@ func (iamOS *IAMObjectStore) loadAll(sys *IAMSys, objectAPI ObjectLayer) error {
 	iamUserPolicyMap := make(map[string]MappedPolicy)
 	iamGroupPolicyMap := make(map[string]MappedPolicy)
 
+	isMinIOUsersSys := false
+	sys.RLock()
+	if sys.usersSysType == MinIOUsersSysType {
+		isMinIOUsersSys = true
+	}
+	sys.RUnlock()
+
 	if err := iamOS.loadPolicyDocs(iamPolicyDocsMap); err != nil {
 		return err
 	}
-	if err := iamOS.loadUsers(false, iamUsersMap); err != nil {
-		return err
-	}
-	// load STS temp users into the same map
+	// load STS temp users
 	if err := iamOS.loadUsers(true, iamUsersMap); err != nil {
 		return err
 	}
-	if err := iamOS.loadGroups(iamGroupsMap); err != nil {
-		return err
-	}
+	if isMinIOUsersSys {
+		if err := iamOS.loadUsers(false, iamUsersMap); err != nil {
+			return err
+		}
+		if err := iamOS.loadGroups(iamGroupsMap); err != nil {
+			return err
+		}
 
-	if err := iamOS.loadMappedPolicies(false, false, iamUserPolicyMap); err != nil {
-		return err
+		if err := iamOS.loadMappedPolicies(false, false, iamUserPolicyMap); err != nil {
+			return err
+		}
 	}
-	// load STS policy mappings into the same map
+	// load STS policy mappings
 	if err := iamOS.loadMappedPolicies(true, false, iamUserPolicyMap); err != nil {
 		return err
 	}

cmd/iam.go

@@ -30,6 +30,20 @@ import (
 	"github.com/minio/minio/pkg/madmin"
 )
 
+// UsersSysType - defines the type of users and groups system that is
+// active on the server.
+type UsersSysType string
+
+// Types of users configured in the server.
+const (
+	// This mode uses the internal users system in MinIO.
+	MinIOUsersSysType UsersSysType = "MinIOUsersSys"
+
+	// This mode uses users and groups from a configured LDAP
+	// server.
+	LDAPUsersSysType UsersSysType = "LDAPUsersSys"
+)
+
 const (
 	// IAM configuration directory.
 	iamConfigPrefix = minioConfigPrefix + "/iam"
@@ -145,6 +159,9 @@ func newMappedPolicy(policy string) MappedPolicy {
 // IAMSys - config system.
 type IAMSys struct {
 	sync.RWMutex
+
+	usersSysType UsersSysType
+
 	// map of policy names to policy definitions
 	iamPolicyDocsMap map[string]iampolicy.Policy
 	// map of usernames to credentials
@@ -463,6 +480,13 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 		return errServerNotInitialized
 	}
 
+	sys.Lock()
+	defer sys.Unlock()
+
+	if sys.usersSysType != MinIOUsersSysType {
+		return errIAMActionNotAllowed
+	}
+
 	// It is ok to ignore deletion error on the mapped policy
 	sys.store.deleteMappedPolicy(accessKey, false, false)
 	err := sys.store.deleteUserIdentity(accessKey, false)
@@ -472,9 +496,6 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 		err = nil
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
-
 	delete(sys.iamUsersMap, accessKey)
 	delete(sys.iamUserPolicyMap, accessKey)
 
@@ -533,6 +554,10 @@ func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
 	sys.RLock()
 	defer sys.RUnlock()
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return nil, errIAMActionNotAllowed
+	}
+
 	for k, v := range sys.iamUsersMap {
 		users[k] = madmin.UserInfo{
 			PolicyName: sys.iamUserPolicyMap[k].Policy,
@@ -553,6 +578,12 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 	sys.RLock()
 	defer sys.RUnlock()
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return madmin.UserInfo{
+			PolicyName: sys.iamUserPolicyMap[name].Policy,
+		}, nil
+	}
+
 	creds, found := sys.iamUsersMap[name]
 	if !found {
 		return u, errNoSuchUser
@@ -580,6 +611,10 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 	sys.Lock()
 	defer sys.Unlock()
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return errIAMActionNotAllowed
+	}
+
 	cred, ok := sys.iamUsersMap[accessKey]
 	if !ok {
 		return errNoSuchUser
@@ -614,6 +649,10 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 	sys.Lock()
 	defer sys.Unlock()
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return errIAMActionNotAllowed
+	}
+
 	if err := sys.store.saveUserIdentity(accessKey, false, u); err != nil {
 		return err
 	}
@@ -636,6 +675,10 @@ func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
 	sys.Lock()
 	defer sys.Unlock()
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return errIAMActionNotAllowed
+	}
+
 	cred, ok := sys.iamUsersMap[accessKey]
 	if !ok {
 		return errNoSuchUser
@@ -675,6 +718,10 @@ func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
 	sys.Lock()
 	defer sys.Unlock()
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return errIAMActionNotAllowed
+	}
+
 	// Validate that all members exist.
 	for _, member := range members {
 		_, ok := sys.iamUsersMap[member]
@@ -729,6 +776,10 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 	sys.Lock()
 	defer sys.Unlock()
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return errIAMActionNotAllowed
+	}
+
 	// Validate that all members exist.
 	for _, member := range members {
 		_, ok := sys.iamUsersMap[member]
@@ -802,6 +853,10 @@ func (sys *IAMSys) SetGroupStatus(group string, enabled bool) error {
 	sys.Lock()
 	defer sys.Unlock()
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return errIAMActionNotAllowed
+	}
+
 	if group == "" {
 		return errInvalidArgument
 	}
@@ -830,6 +885,7 @@ func (sys *IAMSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err e
 	if err != nil {
 		return gd, err
 	}
+
 	// A group may be mapped to at most one policy.
 	policy := ""
 	if len(ps) > 0 {
@@ -839,6 +895,13 @@ func (sys *IAMSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err e
 	sys.RLock()
 	defer sys.RUnlock()
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return madmin.GroupDesc{
+			Name:   group,
+			Policy: policy,
+		}, nil
+	}
+
 	gi, ok := sys.iamGroupsMap[group]
 	if !ok {
 		return gd, errNoSuchGroup
@@ -852,15 +915,19 @@ func (sys *IAMSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err e
 	}, nil
 }
 
-// ListGroups - lists groups
-func (sys *IAMSys) ListGroups() (r []string) {
+// ListGroups - lists groups.
+func (sys *IAMSys) ListGroups() (r []string, err error) {
 	sys.RLock()
 	defer sys.RUnlock()
 
+	if sys.usersSysType != MinIOUsersSysType {
+		return nil, errIAMActionNotAllowed
+	}
+
 	for k := range sys.iamGroupsMap {
 		r = append(r, k)
 	}
-	return r
+	return r, nil
 }
 
 // PolicyDBSet - sets a policy for a user or group in the
@@ -889,13 +956,16 @@ func (sys *IAMSys) policyDBSet(objectAPI ObjectLayer, name, policy string, isSTS
 	if _, ok := sys.iamPolicyDocsMap[policy]; !ok {
 		return errNoSuchPolicy
 	}
-	if !isGroup {
-		if _, ok := sys.iamUsersMap[name]; !ok {
-			return errNoSuchUser
-		}
-	} else {
-		if _, ok := sys.iamGroupsMap[name]; !ok {
-			return errNoSuchGroup
+
+	if sys.usersSysType == MinIOUsersSysType {
+		if !isGroup {
+			if _, ok := sys.iamUsersMap[name]; !ok {
+				return errNoSuchUser
+			}
+		} else {
+			if _, ok := sys.iamGroupsMap[name]; !ok {
+				return errNoSuchGroup
+			}
 		}
 	}
 
@@ -980,6 +1050,63 @@ func (sys *IAMSys) policyDBGet(name string, isGroup bool) ([]string, error) {
 // which implements claims validation and verification other than
 // applying policies.
 func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args) bool {
+	// If it is an LDAP request, check that user and group
+	// policies allow the request.
+	if userIface, ok := args.Claims[ldapUser]; ok {
+		var user string
+		if u, ok := userIface.(string); ok {
+			user = u
+		} else {
+			return false
+		}
+
+		var groups []string
+		groupsVal := args.Claims[ldapGroups]
+		if g, ok := groupsVal.([]interface{}); ok {
+			for _, eachG := range g {
+				if eachGStr, ok := eachG.(string); ok {
+					groups = append(groups, eachGStr)
+				}
+			}
+		} else {
+			return false
+		}
+
+		sys.RLock()
+		defer sys.RUnlock()
+
+		// We look up the policy mapping directly to bypass
+		// users exists, group exists validations that do not
+		// apply here.
+		var policies []iampolicy.Policy
+		if policy, ok := sys.iamUserPolicyMap[user]; ok {
+			p, found := sys.iamPolicyDocsMap[policy.Policy]
+			if found {
+				policies = append(policies, p)
+			}
+		}
+		for _, group := range groups {
+			policy, ok := sys.iamGroupPolicyMap[group]
+			if !ok {
+				continue
+			}
+			p, found := sys.iamPolicyDocsMap[policy.Policy]
+			if found {
+				policies = append(policies, p)
+			}
+		}
+		if len(policies) == 0 {
+			return false
+		}
+		combinedPolicy := policies[0]
+		for i := 1; i < len(policies); i++ {
+			combinedPolicy.Statements =
+				append(combinedPolicy.Statements,
+					policies[i].Statements...)
+		}
+		return combinedPolicy.IsAllowed(args)
+	}
+
 	pname, ok := args.Claims[iampolicy.PolicyName]
 	if !ok {
 		// When claims are set, it should have a "policy" field.
@@ -1155,7 +1282,20 @@ func (sys *IAMSys) removeGroupFromMembershipsMap(group string) {
 
 // NewIAMSys - creates new config system object.
 func NewIAMSys() *IAMSys {
+	// Check global server configuration to determine the type of
+	// users system configured.
+
+	// The default users system
+	var utype UsersSysType
+	switch {
+	case globalServerConfig.LDAPServerConfig.ServerAddr != "":
+		utype = LDAPUsersSysType
+	default:
+		utype = MinIOUsersSysType
+	}
+
 	return &IAMSys{
+		usersSysType:            utype,
 		iamUsersMap:             make(map[string]auth.Credentials),
 		iamPolicyDocsMap:        make(map[string]iampolicy.Policy),
 		iamUserPolicyMap:        make(map[string]MappedPolicy),

cmd/ldap-ops.go

@@ -0,0 +1,178 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"regexp"
+	"time"
+
+	ldap "gopkg.in/ldap.v3"
+)
+
+const (
+	defaultLDAPExpiry = time.Hour * 1
+)
+
+// ldapServerConfig contains server connectivity information.
+type ldapServerConfig struct {
+	IsEnabled bool `json:"enabled"`
+
+	// E.g. "ldap.minio.io:636"
+	ServerAddr string `json:"serverAddr"`
+
+	// STS credentials expiry duration
+	STSExpiryDuration string        `json:"stsExpiryDuration"`
+	stsExpiryDuration time.Duration // contains converted value
+
+	// Skips TLS verification (for testing, not
+	// recommended in production).
+	SkipTLSVerify bool `json:"skipTLSverify"`
+
+	// Format string for usernames
+	UsernameFormat string `json:"usernameFormat"`
+
+	GroupSearchBaseDN  string `json:"groupSearchBaseDN"`
+	GroupSearchFilter  string `json:"groupSearchFilter"`
+	GroupNameAttribute string `json:"groupNameAttribute"`
+}
+
+func (l *ldapServerConfig) Connect() (ldapConn *ldap.Conn, err error) {
+	if l == nil {
+		// Happens when LDAP is not configured.
+		return
+	}
+	if l.SkipTLSVerify {
+		ldapConn, err = ldap.DialTLS("tcp", l.ServerAddr, &tls.Config{InsecureSkipVerify: true})
+	} else {
+		ldapConn, err = ldap.DialTLS("tcp", l.ServerAddr, &tls.Config{})
+	}
+	return
+}
+
+// newLDAPConfigFromEnv loads configuration from the environment
+func newLDAPConfigFromEnv() (l ldapServerConfig, err error) {
+	if ldapServer, ok := os.LookupEnv("MINIO_IDENTITY_LDAP_SERVER_ADDR"); ok {
+		l.IsEnabled = true
+		l.ServerAddr = ldapServer
+
+		if v := os.Getenv("MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY"); v == "true" {
+			l.SkipTLSVerify = true
+		}
+
+		if v := os.Getenv("MINIO_IDENTITY_LDAP_STS_EXPIRY"); v != "" {
+			expDur, err := time.ParseDuration(v)
+			if err != nil {
+				return l, errors.New("LDAP expiry time err:" + err.Error())
+			}
+			if expDur <= 0 {
+				return l, errors.New("LDAP expiry time has to be positive")
+			}
+			l.STSExpiryDuration = v
+			l.stsExpiryDuration = expDur
+		} else {
+			l.stsExpiryDuration = defaultLDAPExpiry
+		}
+
+		if v := os.Getenv("MINIO_IDENTITY_LDAP_USERNAME_FORMAT"); v != "" {
+			subs := newSubstituter("username", "test")
+			if _, err := subs.substitute(v); err != nil {
+				return l, errors.New("Only username may be substituted in the username format")
+			}
+			l.UsernameFormat = v
+		}
+
+		grpSearchFilter := os.Getenv("MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER")
+		grpSearchNameAttr := os.Getenv("MINIO_IDENTITY_LDAP_GROUP_NAME_ATTRIBUTE")
+		grpSearchBaseDN := os.Getenv("MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN")
+
+		// Either all group params must be set or none must be set.
+		allNotSet := grpSearchFilter == "" && grpSearchNameAttr == "" && grpSearchBaseDN == ""
+		allSet := grpSearchFilter != "" && grpSearchNameAttr != "" && grpSearchBaseDN != ""
+		if !allNotSet && !allSet {
+			return l, errors.New("All group related parameters must be set")
+		}
+
+		if allSet {
+			subs := newSubstituter("username", "test", "usernamedn", "test2")
+			if _, err := subs.substitute(grpSearchFilter); err != nil {
+				return l, errors.New("Only username and usernamedn may be substituted in the group search filter string")
+			}
+			l.GroupSearchFilter = grpSearchFilter
+
+			l.GroupNameAttribute = grpSearchNameAttr
+
+			subs = newSubstituter("username", "test", "usernamedn", "test2")
+			if _, err := subs.substitute(grpSearchBaseDN); err != nil {
+				return l, errors.New("Only username and usernamedn may be substituted in the base DN string")
+			}
+			l.GroupSearchBaseDN = grpSearchBaseDN
+		}
+	}
+	return
+}
+
+// substituter - This type is to allow restricted runtime
+// substitutions of variables in LDAP configuration items during
+// runtime.
+type substituter struct {
+	vals map[string]string
+}
+
+// newSubstituter - sets up the substituter for usage, for e.g.:
+//
+//  subber := newSubstituter("username", "john")
+func newSubstituter(v ...string) substituter {
+	if len(v)%2 != 0 {
+		log.Fatal("Need an even number of arguments")
+	}
+	vals := make(map[string]string)
+	for i := 0; i < len(v); i += 2 {
+		vals[v[i]] = v[i+1]
+	}
+	return substituter{vals: vals}
+}
+
+// substitute - performs substitution on the given string `t`. Returns
+// an error if there are any variables in the input that do not have
+// values in the substituter. E.g.:
+//
+// subber.substitute("uid=${username},cn=users,dc=example,dc=com")
+//
+// returns "uid=john,cn=users,dc=example,dc=com"
+//
+// whereas:
+//
+// subber.substitute("uid=${usernamedn}")
+//
+// returns an error.
+func (s *substituter) substitute(t string) (string, error) {
+	for k, v := range s.vals {
+		re := regexp.MustCompile(fmt.Sprintf(`\$\{%s\}`, k))
+		t = re.ReplaceAllLiteralString(t, v)
+	}
+	// Check if all requested substitutions have been made.
+	re := regexp.MustCompile(`\$\{.*\}`)
+	if re.MatchString(t) {
+		return "", errors.New("unsupported substitution requested")
+	}
+	return t, nil
+}

cmd/logger/audit.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2018, 2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,9 @@
 package logger
 
 import (
+	"bytes"
+	"fmt"
+	"io"
 	"net/http"
 	"time"
 
@@ -27,9 +30,18 @@ import (
 // ResponseWriter - is a wrapper to trap the http response status code.
 type ResponseWriter struct {
 	http.ResponseWriter
-	statusCode      int
-	startTime       time.Time
-	timeToFirstByte time.Duration
+	StatusCode int
+	// Response body should be logged
+	LogBody         bool
+	TimeToFirstByte time.Duration
+	StartTime       time.Time
+	// number of bytes written
+	bytesWritten int
+	// Internal recording buffer
+	headers bytes.Buffer
+	body    bytes.Buffer
+	// Indicate if headers are written in the log
+	headersLogged bool
 }
 
 // NewResponseWriter - returns a wrapped response writer to trap
@@ -37,25 +49,64 @@ type ResponseWriter struct {
 func NewResponseWriter(w http.ResponseWriter) *ResponseWriter {
 	return &ResponseWriter{
 		ResponseWriter: w,
-		statusCode:     http.StatusOK,
-		startTime:      time.Now().UTC(),
+		StatusCode:     http.StatusOK,
+		StartTime:      time.Now().UTC(),
 	}
 }
 
 func (lrw *ResponseWriter) Write(p []byte) (int, error) {
 	n, err := lrw.ResponseWriter.Write(p)
+	lrw.bytesWritten += n
+	if lrw.TimeToFirstByte == 0 {
+		lrw.TimeToFirstByte = time.Now().UTC().Sub(lrw.StartTime)
+	}
+	if !lrw.headersLogged {
+		// We assume the response code to be '200 OK' when WriteHeader() is not called,
+		// that way following Golang HTTP response behavior.
+		lrw.writeHeaders(&lrw.headers, http.StatusOK, lrw.Header())
+		lrw.headersLogged = true
+	}
+	if lrw.StatusCode >= http.StatusBadRequest || lrw.LogBody {
+		// Always logging error responses.
+		lrw.body.Write(p)
+	}
 	if err != nil {
 		return n, err
 	}
-	if lrw.timeToFirstByte == 0 {
-		lrw.timeToFirstByte = time.Now().UTC().Sub(lrw.startTime)
-	}
 	return n, err
 }
 
+// Write the headers into the given buffer
+func (lrw *ResponseWriter) writeHeaders(w io.Writer, statusCode int, headers http.Header) {
+	n, _ := fmt.Fprintf(w, "%d %s\n", statusCode, http.StatusText(statusCode))
+	lrw.bytesWritten += n
+	for k, v := range headers {
+		n, _ := fmt.Fprintf(w, "%s: %s\n", k, v[0])
+		lrw.bytesWritten += n
+	}
+}
+
+// BodyPlaceHolder returns a dummy body placeholder
+var BodyPlaceHolder = []byte("<BODY>")
+
+// Body - Return response body.
+func (lrw *ResponseWriter) Body() []byte {
+	// If there was an error response or body logging is enabled
+	// then we return the body contents
+	if lrw.StatusCode >= http.StatusBadRequest || lrw.LogBody {
+		return lrw.body.Bytes()
+	}
+	// ... otherwise we return the <BODY> place holder
+	return BodyPlaceHolder
+}
+
 // WriteHeader - writes http status code
 func (lrw *ResponseWriter) WriteHeader(code int) {
-	lrw.statusCode = code
+	lrw.StatusCode = code
+	if !lrw.headersLogged {
+		lrw.writeHeaders(&lrw.headers, code, lrw.ResponseWriter.Header())
+		lrw.headersLogged = true
+	}
 	lrw.ResponseWriter.WriteHeader(code)
 }
 
@@ -64,6 +115,11 @@ func (lrw *ResponseWriter) Flush() {
 	lrw.ResponseWriter.(http.Flusher).Flush()
 }
 
+// Size - reutrns the number of bytes written
+func (lrw *ResponseWriter) Size() int {
+	return lrw.bytesWritten
+}
+
 // AuditTargets is the list of enabled audit loggers
 var AuditTargets = []Target{}
 
@@ -80,9 +136,9 @@ func AuditLog(w http.ResponseWriter, r *http.Request, api string, reqClaims map[
 	var timeToFirstByte time.Duration
 	lrw, ok := w.(*ResponseWriter)
 	if ok {
-		statusCode = lrw.statusCode
-		timeToResponse = time.Now().UTC().Sub(lrw.startTime)
-		timeToFirstByte = lrw.timeToFirstByte
+		statusCode = lrw.StatusCode
+		timeToResponse = time.Now().UTC().Sub(lrw.StartTime)
+		timeToFirstByte = lrw.TimeToFirstByte
 	}
 
 	vars := mux.Vars(r)

cmd/metrics-router.go

@@ -28,5 +28,5 @@ const (
 func registerMetricsRouter(router *mux.Router) {
 	// metrics router
 	metricsRouter := router.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
-	metricsRouter.Handle(prometheusMetricsPath, metricsHandler())
+	metricsRouter.Handle(prometheusMetricsPath, AuthMiddleware(metricsHandler()))
 }

cmd/metrics.go

@@ -199,6 +199,7 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 }
 
 func metricsHandler() http.Handler {
+
 	registry := prometheus.NewRegistry()
 
 	err := registry.Register(minioVersionInfo)
@@ -222,4 +223,17 @@ func metricsHandler() http.Handler {
 				ErrorHandling: promhttp.ContinueOnError,
 			}),
 	)
+
+}
+
+// AuthMiddleware checks if the bearer token is valid and authorized.
+func AuthMiddleware(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		claims, _, authErr := webRequestAuthenticate(r)
+		if authErr != nil || !claims.VerifyIssuer("prometheus", true) {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		h.ServeHTTP(w, r)
+	})
 }

cmd/naughty-disk_test.go

@@ -196,9 +196,9 @@ func (d *naughtyDisk) ReadAll(volume string, path string) (buf []byte, err error
 	return d.disk.ReadAll(volume, path)
 }
 
-func (d *naughtyDisk) VerifyFile(volume, path string, empty bool, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
+func (d *naughtyDisk) VerifyFile(volume, path string, size int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
 	if err := d.calcError(); err != nil {
 		return err
 	}
-	return d.disk.VerifyFile(volume, path, empty, algo, sum, shardSize)
+	return d.disk.VerifyFile(volume, path, size, algo, sum, shardSize)
 }

cmd/notification.go

@@ -977,8 +977,8 @@ func (sys *NotificationSys) CollectNetPerfInfo(size int64) map[string][]ServerNe
 }
 
 // DrivePerfInfo - Drive speed (read and write) information
-func (sys *NotificationSys) DrivePerfInfo() []ServerDrivesPerfInfo {
-	reply := make([]ServerDrivesPerfInfo, len(sys.peerClients))
+func (sys *NotificationSys) DrivePerfInfo(size int64) []madmin.ServerDrivesPerfInfo {
+	reply := make([]madmin.ServerDrivesPerfInfo, len(sys.peerClients))
 	var wg sync.WaitGroup
 	for i, client := range sys.peerClients {
 		if client == nil {
@@ -987,7 +987,7 @@ func (sys *NotificationSys) DrivePerfInfo() []ServerDrivesPerfInfo {
 		wg.Add(1)
 		go func(client *peerRESTClient, idx int) {
 			defer wg.Done()
-			di, err := client.DrivePerfInfo()
+			di, err := client.DrivePerfInfo(size)
 			if err != nil {
 				reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", client.host.String())
 				ctx := logger.SetReqInfo(context.Background(), reqInfo)

cmd/object-api-utils.go

@@ -354,7 +354,7 @@ func (o ObjectInfo) GetActualSize() int64 {
 // Using compression and encryption together enables room for side channel attacks.
 // Eliminate non-compressible objects by extensions/content-types.
 func isCompressible(header http.Header, object string) bool {
-	if hasServerSideEncryptionHeader(header) || excludeForCompression(header, object) {
+	if crypto.IsRequested(header) || excludeForCompression(header, object) {
 		return false
 	}
 	return true

cmd/object-handlers.go

@@ -92,7 +92,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if !api.EncryptionEnabled() && hasServerSideEncryptionHeader(r.Header) {
+	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -251,7 +251,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if !api.EncryptionEnabled() && hasServerSideEncryptionHeader(r.Header) {
+	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -422,7 +422,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrBadRequest))
 		return
 	}
-	if !api.EncryptionEnabled() && hasServerSideEncryptionHeader(r.Header) {
+	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -646,7 +646,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) // SSE-KMS is not supported
 		return
 	}
-	if !api.EncryptionEnabled() && (hasServerSideEncryptionHeader(r.Header) || crypto.SSECopy.IsRequested(r.Header)) {
+	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -707,6 +707,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidMetadataDirective), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	// This request header needs to be set prior to setting ObjectOptions
 	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
 		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
@@ -733,11 +734,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
-
-	// Deny if WORM is enabled. If operation is key rotation of SSE-S3 encrypted object
-	// allow the operation
-	if globalWORMEnabled && !(cpSrcDstSame && crypto.S3.IsRequested(r.Header)) {
-		if _, err = objectAPI.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts); err == nil {
+	if globalWORMEnabled { // Deny if WORM is enabled.
+		if _, err := objectAPI.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts); err == nil {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
 			return
 		}
@@ -775,11 +773,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	// Deny if WORM is enabled, and it is not a SSE-S3 -> SSE-S3 key rotation or if metadata replacement is requested.
-	if globalWORMEnabled && cpSrcDstSame && (!crypto.S3.IsEncrypted(srcInfo.UserDefined) || isMetadataReplace(r.Header)) {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
-		return
-	}
 	// We have to copy metadata only if source and destination are same.
 	// this changes for encryption which can be observed below.
 	if cpSrcDstSame {
@@ -866,13 +859,11 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		// - the object is encrypted using SSE-S3 and the SSE-S3 header is present
 		// than execute a key rotation.
 		var keyRotation bool
-		if cpSrcDstSame && ((sseCopyC && sseC) || (sseS3 && sseCopyS3)) {
-			if sseCopyC && sseC {
-				oldKey, err = ParseSSECopyCustomerRequest(r.Header, srcInfo.UserDefined)
-				if err != nil {
-					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-					return
-				}
+		if cpSrcDstSame && (sseCopyC && sseC) {
+			oldKey, err = ParseSSECopyCustomerRequest(r.Header, srcInfo.UserDefined)
+			if err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+				return
 			}
 
 			for k, v := range srcInfo.UserDefined {
@@ -1051,7 +1042,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) // SSE-KMS is not supported
 		return
 	}
-	if !api.EncryptionEnabled() && hasServerSideEncryptionHeader(r.Header) {
+	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -1232,7 +1223,11 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 
 	var objectEncryptionKey []byte
 	if objectAPI.IsEncryptionSupported() {
-		if hasServerSideEncryptionHeader(r.Header) && !hasSuffix(object, SlashSeparator) { // handle SSE requests
+		if crypto.IsRequested(r.Header) && !hasSuffix(object, SlashSeparator) { // handle SSE requests
+			if crypto.SSECopy.IsRequested(r.Header) {
+				writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL, guessIsBrowserReq(r))
+				return
+			}
 			reader, objectEncryptionKey, err = EncryptRequest(hashReader, r, bucket, object, metadata)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -1264,7 +1259,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		if !strings.HasSuffix(objInfo.ETag, "-1") {
 			etag = objInfo.ETag + "-1"
 		}
-	} else if hasServerSideEncryptionHeader(r.Header) {
+	} else if crypto.IsRequested(r.Header) {
 		etag = getDecryptedETag(r.Header, objInfo, false)
 	}
 	w.Header()[xhttp.ETag] = []string{"\"" + etag + "\""}
@@ -1318,7 +1313,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) // SSE-KMS is not supported
 		return
 	}
-	if !api.EncryptionEnabled() && hasServerSideEncryptionHeader(r.Header) {
+	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -1365,7 +1360,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 	var encMetadata = map[string]string{}
 
 	if objectAPI.IsEncryptionSupported() {
-		if hasServerSideEncryptionHeader(r.Header) {
+		if crypto.IsRequested(r.Header) {
 			if err = setEncryptionMetadata(r, bucket, object, encMetadata); err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
@@ -1432,7 +1427,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) // SSE-KMS is not supported
 		return
 	}
-	if !api.EncryptionEnabled() && (hasServerSideEncryptionHeader(r.Header) || crypto.SSECopy.IsRequested(r.Header)) {
+	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -1747,7 +1742,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) // SSE-KMS is not supported
 		return
 	}
-	if !api.EncryptionEnabled() && hasServerSideEncryptionHeader(r.Header) {
+	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}

cmd/peer-rest-client.go

@@ -173,8 +173,10 @@ func (client *peerRESTClient) CPULoadInfo() (info ServerCPULoadInfo, err error)
 }
 
 // DrivePerfInfo - fetch Drive performance information for a remote node.
-func (client *peerRESTClient) DrivePerfInfo() (info ServerDrivesPerfInfo, err error) {
-	respBody, err := client.call(peerRESTMethodDrivePerfInfo, nil, nil, -1)
+func (client *peerRESTClient) DrivePerfInfo(size int64) (info madmin.ServerDrivesPerfInfo, err error) {
+	params := make(url.Values)
+	params.Set(peerRESTDrivePerfSize, strconv.FormatInt(size, 10))
+	respBody, err := client.call(peerRESTMethodDrivePerfInfo, params, nil, -1)
 	if err != nil {
 		return
 	}

cmd/peer-rest-common.go

@@ -56,6 +56,7 @@ const (
 
 const (
 	peerRESTNetPerfSize   = "netperfsize"
+	peerRESTDrivePerfSize = "driveperfsize"
 	peerRESTBucket        = "bucket"
 	peerRESTUser          = "user"
 	peerRESTGroup         = "group"

cmd/peer-rest-server.go

@@ -32,7 +32,6 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/lifecycle"
-	"github.com/minio/minio/pkg/madmin"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/policy"
 	trace "github.com/minio/minio/pkg/trace"
@@ -418,7 +417,7 @@ func (s *peerRESTServer) CPULoadInfoHandler(w http.ResponseWriter, r *http.Reque
 	}
 
 	ctx := newContext(r, w, "CPULoadInfo")
-	info := localEndpointsCPULoad(globalEndpoints, r)
+	info := getLocalCPULoad(globalEndpoints, r)
 
 	defer w.(http.Flusher).Flush()
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
@@ -431,8 +430,23 @@ func (s *peerRESTServer) DrivePerfInfoHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
+	params := mux.Vars(r)
+
+	sizeStr, found := params[peerRESTDrivePerfSize]
+	if !found {
+		s.writeErrorResponse(w, errors.New("size is missing"))
+		return
+	}
+
+	size, err := strconv.ParseInt(sizeStr, 10, 64)
+	if err != nil || size < 0 {
+		s.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
 	ctx := newContext(r, w, "DrivePerfInfo")
-	info := localEndpointsDrivePerf(globalEndpoints, r)
+
+	info := getLocalDrivesPerf(globalEndpoints, size, r)
 
 	defer w.(http.Flusher).Flush()
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
@@ -445,7 +459,7 @@ func (s *peerRESTServer) MemUsageInfoHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 	ctx := newContext(r, w, "MemUsageInfo")
-	info := localEndpointsMemUsage(globalEndpoints, r)
+	info := getLocalMemUsage(globalEndpoints, r)
 
 	defer w.(http.Flusher).Flush()
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
@@ -925,8 +939,7 @@ func (s *peerRESTServer) ConsoleLogHandler(w http.ResponseWriter, r *http.Reques
 	for {
 		select {
 		case entry := <-ch:
-			log := entry.(madmin.LogInfo)
-			if err := enc.Encode(log); err != nil {
+			if err := enc.Encode(entry); err != nil {
 				return
 			}
 			w.(http.Flusher).Flush()
@@ -960,7 +973,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodServerInfo).HandlerFunc(httpTraceHdrs(server.ServerInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodCPULoadInfo).HandlerFunc(httpTraceHdrs(server.CPULoadInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodMemUsageInfo).HandlerFunc(httpTraceHdrs(server.MemUsageInfoHandler))
-	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodDrivePerfInfo).HandlerFunc(httpTraceHdrs(server.DrivePerfInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodDrivePerfInfo).HandlerFunc(httpTraceHdrs(server.DrivePerfInfoHandler)).Queries(restQueries(peerRESTDrivePerfSize)...)
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodDeleteBucket).HandlerFunc(httpTraceHdrs(server.DeleteBucketHandler)).Queries(restQueries(peerRESTBucket)...)
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodSignalService).HandlerFunc(httpTraceHdrs(server.SignalServiceHandler)).Queries(restQueries(peerRESTSignal)...)
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodServerUpdate).HandlerFunc(httpTraceHdrs(server.ServerUpdateHandler)).Queries(restQueries(peerRESTUpdateURL, peerRESTSha256Hex, peerRESTLatestRelease)...)

cmd/posix-errors.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"errors"
 	"os"
 	"runtime"
 	"syscall"
@@ -24,86 +25,54 @@ import (
 
 // Function not implemented error
 func isSysErrNoSys(err error) bool {
-	if err == syscall.ENOSYS {
-		return true
-	}
-	pathErr, ok := err.(*os.PathError)
-	return ok && pathErr.Err == syscall.ENOSYS
-
+	return errors.Is(err, syscall.ENOSYS)
 }
 
 // Not supported error
 func isSysErrOpNotSupported(err error) bool {
-	if err == syscall.EOPNOTSUPP {
-		return true
-	}
-	pathErr, ok := err.(*os.PathError)
-	return ok && pathErr.Err == syscall.EOPNOTSUPP
-
+	return errors.Is(err, syscall.EOPNOTSUPP)
 }
 
 // No space left on device error
 func isSysErrNoSpace(err error) bool {
-	if err == syscall.ENOSPC {
-		return true
-	}
-	pathErr, ok := err.(*os.PathError)
-	return ok && pathErr.Err == syscall.ENOSPC
+	return errors.Is(err, syscall.ENOSPC)
 }
 
 // Input/output error
 func isSysErrIO(err error) bool {
-	if err == syscall.EIO {
-		return true
-	}
-	pathErr, ok := err.(*os.PathError)
-	return ok && pathErr.Err == syscall.EIO
+	return errors.Is(err, syscall.EIO)
 }
 
 // Check if the given error corresponds to EISDIR (is a directory).
 func isSysErrIsDir(err error) bool {
-	if err == syscall.EISDIR {
-		return true
-	}
-	pathErr, ok := err.(*os.PathError)
-	return ok && pathErr.Err == syscall.EISDIR
-
+	return errors.Is(err, syscall.EISDIR)
 }
 
 // Check if the given error corresponds to ENOTDIR (is not a directory).
 func isSysErrNotDir(err error) bool {
-	if err == syscall.ENOTDIR {
-		return true
-	}
-	pathErr, ok := err.(*os.PathError)
-	return ok && pathErr.Err == syscall.ENOTDIR
+	return errors.Is(err, syscall.ENOTDIR)
 }
 
 // Check if the given error corresponds to the ENAMETOOLONG (name too long).
 func isSysErrTooLong(err error) bool {
-	if err == syscall.ENAMETOOLONG {
-		return true
-	}
-	pathErr, ok := err.(*os.PathError)
-	return ok && pathErr.Err == syscall.ENAMETOOLONG
+	return errors.Is(err, syscall.ENAMETOOLONG)
 }
 
 // Check if the given error corresponds to ENOTEMPTY for unix
 // and ERROR_DIR_NOT_EMPTY for windows (directory not empty).
 func isSysErrNotEmpty(err error) bool {
-	if err == syscall.ENOTEMPTY {
+	if errors.Is(err, syscall.ENOTEMPTY) {
 		return true
 	}
-	if pathErr, ok := err.(*os.PathError); ok {
+	var pathErr *os.PathError
+	if errors.As(err, &pathErr) {
 		if runtime.GOOS == globalWindowsOSName {
-			if errno, _ok := pathErr.Err.(syscall.Errno); _ok && errno == 0x91 {
+			var errno syscall.Errno
+			if errors.As(pathErr.Err, &errno) {
 				// ERROR_DIR_NOT_EMPTY
-				return true
+				return errno == 0x91
 			}
 		}
-		if pathErr.Err == syscall.ENOTEMPTY {
-			return true
-		}
 	}
 	return false
 }
@@ -113,10 +82,12 @@ func isSysErrPathNotFound(err error) bool {
 	if runtime.GOOS != globalWindowsOSName {
 		return false
 	}
-	if pathErr, ok := err.(*os.PathError); ok {
-		if errno, _ok := pathErr.Err.(syscall.Errno); _ok && errno == 0x03 {
+	var pathErr *os.PathError
+	if errors.As(err, &pathErr) {
+		var errno syscall.Errno
+		if errors.As(pathErr.Err, &errno) {
 			// ERROR_PATH_NOT_FOUND
-			return true
+			return errno == 0x03
 		}
 	}
 	return false
@@ -128,20 +99,22 @@ func isSysErrHandleInvalid(err error) bool {
 		return false
 	}
 	// Check if err contains ERROR_INVALID_HANDLE errno
-	errno, ok := err.(syscall.Errno)
-	return ok && errno == 0x6
+	var pathErr *os.PathError
+	if errors.As(err, &pathErr) {
+		var errno syscall.Errno
+		if errors.As(pathErr.Err, &errno) {
+			// ERROR_PATH_NOT_FOUND
+			return errno == 0x6
+		}
+	}
+	return false
 }
 
 func isSysErrCrossDevice(err error) bool {
-	e, ok := err.(*os.LinkError)
-	return ok && e.Err == syscall.EXDEV
+	return errors.Is(err, syscall.EXDEV)
 }
 
 // Check if given error corresponds to too many open files
 func isSysErrTooManyFiles(err error) bool {
-	if err == syscall.ENFILE || err == syscall.EMFILE {
-		return true
-	}
-	pathErr, ok := err.(*os.PathError)
-	return ok && (pathErr.Err == syscall.ENFILE || pathErr.Err == syscall.EMFILE)
+	return errors.Is(err, syscall.ENFILE) || errors.Is(err, syscall.EMFILE)
 }

cmd/posix-list-dir_unix.go

@@ -58,7 +58,6 @@ func parseDirEnt(buf []byte) (consumed int, name string, typ os.FileMode, err er
 		// to handle such files, MinIO is only interested in
 		// files and directories.
 		typ = unexpectedFileMode
-		return
 	}
 
 	nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0]))

cmd/posix.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"encoding/hex"
+	"errors"
 	"io"
 	"io/ioutil"
 	"os"
@@ -328,6 +329,10 @@ func (s *posix) DiskInfo() (info DiskInfo, err error) {
 		return info, errFaultyDisk
 	}
 
+	if err := s.checkDiskFound(); err != nil {
+		return info, err
+	}
+
 	di, err := getDiskInfo(s.diskPath)
 	if err != nil {
 		return info, err
@@ -845,17 +850,11 @@ func (s *posix) ReadAll(volume, path string) (buf []byte, err error) {
 			return nil, errFileNotFound
 		} else if os.IsPermission(err) {
 			return nil, errFileAccessDenied
-		} else if pathErr, ok := err.(*os.PathError); ok {
-			switch pathErr.Err {
-			case syscall.ENOTDIR, syscall.EISDIR:
-				return nil, errFileNotFound
-			default:
-				if isSysErrHandleInvalid(pathErr.Err) {
-					// This case is special and needs to be handled for windows.
-					return nil, errFileNotFound
-				}
-			}
-			return nil, pathErr
+		} else if errors.Is(err, syscall.ENOTDIR) || errors.Is(err, syscall.EISDIR) {
+			return nil, errFileNotFound
+		} else if isSysErrHandleInvalid(err) {
+			// This case is special and needs to be handled for windows.
+			return nil, errFileNotFound
 		} else if isSysErrIO(err) {
 			return nil, errFaultyDisk
 		}
@@ -1546,7 +1545,7 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e
 	return nil
 }
 
-func (s *posix) VerifyFile(volume, path string, empty bool, algo BitrotAlgorithm, sum []byte, shardSize int64) (err error) {
+func (s *posix) VerifyFile(volume, path string, fileSize int64, algo BitrotAlgorithm, sum []byte, shardSize int64) (err error) {
 	defer func() {
 		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
@@ -1622,7 +1621,9 @@ func (s *posix) VerifyFile(volume, path string, empty bool, algo BitrotAlgorithm
 		return err
 	}
 
-	if empty && fi.Size() != 0 || !empty && fi.Size() == 0 {
+	// Calculate the size of the bitrot file and compare
+	// it with the actual file size.
+	if fi.Size() != bitrotShardFileSize(fileSize, shardSize, algo) {
 		return errFileUnexpectedSize
 	}
 

cmd/posix_test.go

@@ -1797,7 +1797,7 @@ func TestPosixVerifyFile(t *testing.T) {
 	if err := posixStorage.WriteAll(volName, fileName, bytes.NewBuffer(data)); err != nil {
 		t.Fatal(err)
 	}
-	if err := posixStorage.VerifyFile(volName, fileName, false, algo, hashBytes, 0); err != nil {
+	if err := posixStorage.VerifyFile(volName, fileName, size, algo, hashBytes, 0); err != nil {
 		t.Fatal(err)
 	}
 
@@ -1805,7 +1805,14 @@ func TestPosixVerifyFile(t *testing.T) {
 	if err := posixStorage.AppendFile(volName, fileName, []byte("a")); err != nil {
 		t.Fatal(err)
 	}
-	if err := posixStorage.VerifyFile(volName, fileName, false, algo, hashBytes, 0); err == nil {
+
+	// Check if VerifyFile reports the incorrect file length (the correct length is `size+1`)
+	if err := posixStorage.VerifyFile(volName, fileName, size, algo, hashBytes, 0); err == nil {
+		t.Fatal("expected to fail bitrot check")
+	}
+
+	// Check if bitrot fails
+	if err := posixStorage.VerifyFile(volName, fileName, size+1, algo, hashBytes, 0); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
 
@@ -1833,7 +1840,7 @@ func TestPosixVerifyFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	w.Close()
-	if err := posixStorage.VerifyFile(volName, fileName, false, algo, nil, shardSize); err != nil {
+	if err := posixStorage.VerifyFile(volName, fileName, size, algo, nil, shardSize); err != nil {
 		t.Fatal(err)
 	}
 
@@ -1847,7 +1854,10 @@ func TestPosixVerifyFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	f.Close()
-	if err := posixStorage.VerifyFile(volName, fileName, false, algo, nil, shardSize); err == nil {
+	if err := posixStorage.VerifyFile(volName, fileName, size, algo, nil, shardSize); err == nil {
+		t.Fatal("expected to fail bitrot check")
+	}
+	if err := posixStorage.VerifyFile(volName, fileName, size+1, algo, nil, shardSize); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
 }

cmd/post-policy_test.go

@@ -314,7 +314,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
 		{
 			objectName:         "test",
 			data:               []byte("Hello, World"),
-			expectedRespStatus: http.StatusBadRequest,
+			expectedRespStatus: http.StatusForbidden,
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
 			dates:              []interface{}{curTimePlus5Min.Format(expirationDateFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)},

cmd/postpolicyform.go

@@ -101,8 +101,9 @@ type contentLengthRange struct {
 type PostPolicyForm struct {
 	Expiration time.Time // Expiration date and time of the POST policy.
 	Conditions struct {  // Conditional policy structure.
-		Policies map[string]struct {
+		Policies []struct {
 			Operator string
+			Key      string
 			Value    string
 		}
 		ContentLengthRange contentLengthRange
@@ -130,10 +131,6 @@ func parsePostPolicyForm(policy string) (ppf PostPolicyForm, e error) {
 	if err != nil {
 		return ppf, err
 	}
-	parsedPolicy.Conditions.Policies = make(map[string]struct {
-		Operator string
-		Value    string
-	})
 
 	// Parse conditions.
 	for _, val := range rawPolicy.Conditions {
@@ -146,13 +143,13 @@ func parsePostPolicyForm(policy string) (ppf PostPolicyForm, e error) {
 				}
 				// {"acl": "public-read" } is an alternate way to indicate - [ "eq", "$acl", "public-read" ]
 				// In this case we will just collapse this into "eq" for all use cases.
-				parsedPolicy.Conditions.Policies["$"+strings.ToLower(k)] = struct {
+				parsedPolicy.Conditions.Policies = append(parsedPolicy.Conditions.Policies, struct {
 					Operator string
+					Key      string
 					Value    string
 				}{
-					Operator: policyCondEqual,
-					Value:    toString(v),
-				}
+					policyCondEqual, "$" + strings.ToLower(k), toString(v),
+				})
 			}
 		case []interface{}: // Handle array types.
 			if len(condt) != 3 { // Return error if we have insufficient elements.
@@ -167,13 +164,16 @@ func parsePostPolicyForm(policy string) (ppf PostPolicyForm, e error) {
 					}
 				}
 				operator, matchType, value := toLowerString(condt[0]), toLowerString(condt[1]), toString(condt[2])
-				parsedPolicy.Conditions.Policies[matchType] = struct {
+				if !strings.HasPrefix(matchType, "$") {
+					return parsedPolicy, fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]", operator, matchType, value)
+				}
+				parsedPolicy.Conditions.Policies = append(parsedPolicy.Conditions.Policies, struct {
 					Operator string
+					Key      string
 					Value    string
 				}{
-					Operator: operator,
-					Value:    value,
-				}
+					operator, matchType, value,
+				})
 			case policyCondContentLength:
 				min, err := toInteger(condt[1])
 				if err != nil {
@@ -224,10 +224,10 @@ func checkPostPolicy(formValues http.Header, postPolicyForm PostPolicyForm) erro
 	}
 	// map to store the metadata
 	metaMap := make(map[string]string)
-	for cond, v := range postPolicyForm.Conditions.Policies {
-		if strings.HasPrefix(cond, "$x-amz-meta-") {
-			formCanonicalName := http.CanonicalHeaderKey(strings.TrimPrefix(cond, "$"))
-			metaMap[formCanonicalName] = v.Value
+	for _, policy := range postPolicyForm.Conditions.Policies {
+		if strings.HasPrefix(policy.Key, "$x-amz-meta-") {
+			formCanonicalName := http.CanonicalHeaderKey(strings.TrimPrefix(policy.Key, "$"))
+			metaMap[formCanonicalName] = policy.Value
 		}
 	}
 	// Check if any extra metadata field is passed as input
@@ -243,30 +243,30 @@ func checkPostPolicy(formValues http.Header, postPolicyForm PostPolicyForm) erro
 	condPassed := true
 
 	// Iterate over policy conditions and check them against received form fields
-	for cond, v := range postPolicyForm.Conditions.Policies {
+	for _, policy := range postPolicyForm.Conditions.Policies {
 		// Form fields names are in canonical format, convert conditions names
 		// to canonical for simplification purpose, so `$key` will become `Key`
-		formCanonicalName := http.CanonicalHeaderKey(strings.TrimPrefix(cond, "$"))
+		formCanonicalName := http.CanonicalHeaderKey(strings.TrimPrefix(policy.Key, "$"))
 		// Operator for the current policy condition
-		op := v.Operator
+		op := policy.Operator
 		// If the current policy condition is known
-		if startsWithSupported, condFound := startsWithConds[cond]; condFound {
+		if startsWithSupported, condFound := startsWithConds[policy.Key]; condFound {
 			// Check if the current condition supports starts-with operator
 			if op == policyCondStartsWith && !startsWithSupported {
 				return fmt.Errorf("Invalid according to Policy: Policy Condition failed")
 			}
 			// Check if current policy condition is satisfied
-			condPassed = checkPolicyCond(op, formValues.Get(formCanonicalName), v.Value)
+			condPassed = checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value)
 			if !condPassed {
 				return fmt.Errorf("Invalid according to Policy: Policy Condition failed")
 			}
 		} else {
 			// This covers all conditions X-Amz-Meta-* and X-Amz-*
-			if strings.HasPrefix(cond, "$x-amz-meta-") || strings.HasPrefix(cond, "$x-amz-") {
+			if strings.HasPrefix(policy.Key, "$x-amz-meta-") || strings.HasPrefix(policy.Key, "$x-amz-") {
 				// Check if policy condition is satisfied
-				condPassed = checkPolicyCond(op, formValues.Get(formCanonicalName), v.Value)
+				condPassed = checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value)
 				if !condPassed {
-					return fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]", op, cond, v.Value)
+					return fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]", op, policy.Key, policy.Value)
 				}
 			}
 		}

cmd/server-main.go

@@ -120,14 +120,7 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}miniostorage
      {{.Prompt}} {{.HelpName}} http://node{1...32}.example.com/mnt/export/{1...32}
 
-  6. Start minio server with edge caching enabled.
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
-     {{.Prompt}} {{.HelpName}} /home/shared
-
-  7. Start minio server with KMS enabled.
+  6. Start minio server with KMS enabled.
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_APPROLE_ID{{.AssignmentOperator}}9b56cc08-8258-45d5-24a3-679876769126
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_APPROLE_SECRET{{.AssignmentOperator}}4e30c52f-13e4-a6f5-0763-d50e8cb4321f
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SSE_VAULT_ENDPOINT{{.AssignmentOperator}}https://vault-endpoint-ip:8200
@@ -339,13 +332,6 @@ func serverMain(ctx *cli.Context) {
 	// Load logger subsystem
 	loadLoggers()
 
-	var cacheConfig = globalServerConfig.GetCacheConfig()
-	if len(cacheConfig.Drives) > 0 {
-		// initialize the new disk cache objects.
-		globalCacheObjectAPI, err = newServerCacheObjects(context.Background(), cacheConfig)
-		logger.FatalIf(err, "Unable to initialize disk caching")
-	}
-
 	// Create new IAM system.
 	globalIAMSys = NewIAMSys()
 	if err = globalIAMSys.Init(newObject); err != nil {
@@ -360,6 +346,9 @@ func serverMain(ctx *cli.Context) {
 		logger.Fatal(err, "Unable to initialize policy system")
 	}
 
+	if globalIsDiskCacheEnabled {
+		logger.StartupMessage(colorRed(colorBold("Disk caching is allowed only for gateway deployments")))
+	}
 	// Create new lifecycle system.
 	globalLifecycleSys = NewLifecycleSys()
 

cmd/server-startup-msg.go

@@ -25,7 +25,6 @@ import (
 	"strings"
 
 	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/minio/cmd/logger"
 	xnet "github.com/minio/minio/pkg/net"
 )
 
@@ -122,19 +121,19 @@ func printServerCommonMsg(apiEndpoints []string) {
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")
 
 	// Colorize the message and print.
-	logger.StartupMessage(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
+	logStartupMessage(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
 	if isTerminal() && !globalCLIContext.Anonymous {
-		logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
-		logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
+		logStartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logStartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
 		if region != "" {
-			logger.StartupMessage(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region)))
+			logStartupMessage(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region)))
 		}
 	}
 	printEventNotifiers()
 
 	if globalIsBrowserEnabled {
-		logger.StartupMessage(colorBlue("\nBrowser Access:"))
-		logger.StartupMessage(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr))
+		logStartupMessage(colorBlue("\nBrowser Access:"))
+		logStartupMessage(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr))
 	}
 }
 
@@ -150,7 +149,7 @@ func printEventNotifiers() {
 		arnMsg += colorBold(fmt.Sprintf(getFormatStr(len(arn), 1), arn))
 	}
 
-	logger.StartupMessage(arnMsg)
+	logStartupMessage(arnMsg)
 }
 
 // Prints startup message for command line access. Prints link to our documentation
@@ -161,25 +160,25 @@ func printCLIAccessMsg(endPoint string, alias string) {
 
 	// Configure 'mc', following block prints platform specific information for minio client.
 	if isTerminal() {
-		logger.StartupMessage(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide)
+		logStartupMessage(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide)
 		if runtime.GOOS == globalWindowsOSName {
 			mcMessage := fmt.Sprintf("$ mc.exe config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey)
-			logger.StartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+			logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
 		} else {
 			mcMessage := fmt.Sprintf("$ mc config host add %s %s %s %s", alias, endPoint, cred.AccessKey, cred.SecretKey)
-			logger.StartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+			logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
 		}
 	}
 }
 
 // Prints startup message for Object API acces, prints link to our SDK documentation.
 func printObjectAPIMsg() {
-	logger.StartupMessage(colorBlue("\nObject API (Amazon S3 compatible):"))
-	logger.StartupMessage(colorBlue("   Go: ") + fmt.Sprintf(getFormatStr(len(goQuickStartGuide), 8), goQuickStartGuide))
-	logger.StartupMessage(colorBlue("   Java: ") + fmt.Sprintf(getFormatStr(len(javaQuickStartGuide), 6), javaQuickStartGuide))
-	logger.StartupMessage(colorBlue("   Python: ") + fmt.Sprintf(getFormatStr(len(pyQuickStartGuide), 4), pyQuickStartGuide))
-	logger.StartupMessage(colorBlue("   JavaScript: ") + jsQuickStartGuide)
-	logger.StartupMessage(colorBlue("   .NET: ") + fmt.Sprintf(getFormatStr(len(dotnetQuickStartGuide), 6), dotnetQuickStartGuide))
+	logStartupMessage(colorBlue("\nObject API (Amazon S3 compatible):"))
+	logStartupMessage(colorBlue("   Go: ") + fmt.Sprintf(getFormatStr(len(goQuickStartGuide), 8), goQuickStartGuide))
+	logStartupMessage(colorBlue("   Java: ") + fmt.Sprintf(getFormatStr(len(javaQuickStartGuide), 6), javaQuickStartGuide))
+	logStartupMessage(colorBlue("   Python: ") + fmt.Sprintf(getFormatStr(len(pyQuickStartGuide), 4), pyQuickStartGuide))
+	logStartupMessage(colorBlue("   JavaScript: ") + jsQuickStartGuide)
+	logStartupMessage(colorBlue("   .NET: ") + fmt.Sprintf(getFormatStr(len(dotnetQuickStartGuide), 6), dotnetQuickStartGuide))
 }
 
 // Get formatted disk/storage info message.
@@ -195,7 +194,7 @@ func getStorageInfoMsg(storageInfo StorageInfo) string {
 // Prints startup message of storage capacity and erasure information.
 func printStorageInfo(storageInfo StorageInfo) {
 	if msg := getStorageInfoMsg(storageInfo); msg != "" {
-		logger.StartupMessage(msg)
+		logStartupMessage(msg)
 	}
 }
 
@@ -203,7 +202,7 @@ func printCacheStorageInfo(storageInfo CacheStorageInfo) {
 	msg := fmt.Sprintf("%s %s Free, %s Total", colorBlue("Cache Capacity:"),
 		humanize.IBytes(uint64(storageInfo.Free)),
 		humanize.IBytes(uint64(storageInfo.Total)))
-	logger.StartupMessage(msg)
+	logStartupMessage(msg)
 }
 
 // Prints certificate expiry date warning
@@ -226,5 +225,5 @@ func getCertificateChainMsg(certs []*x509.Certificate) string {
 
 // Prints the certificate expiry message.
 func printCertificateMsg(certs []*x509.Certificate) {
-	logger.StartupMessage(getCertificateChainMsg(certs))
+	logStartupMessage(getCertificateChainMsg(certs))
 }

cmd/storage-interface.go

@@ -52,7 +52,7 @@ type StorageAPI interface {
 	StatFile(volume string, path string) (file FileInfo, err error)
 	DeleteFile(volume string, path string) (err error)
 	DeleteFileBulk(volume string, paths []string) (errs []error, err error)
-	VerifyFile(volume, path string, empty bool, algo BitrotAlgorithm, sum []byte, shardSize int64) error
+	VerifyFile(volume, path string, size int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error
 
 	// Write all data, syncs the data to disk.
 	WriteAll(volume string, path string, reader io.Reader) (err error)

cmd/storage-rest-client.go

@@ -45,7 +45,7 @@ func isNetworkError(err error) bool {
 		return true
 	}
 	if nerr, ok := err.(*rest.NetworkError); ok {
-		return isNetworkOrHostDown(nerr.Err)
+		return xnet.IsNetworkOrHostDown(nerr.Err)
 	}
 	return false
 }
@@ -439,16 +439,15 @@ func (client *storageRESTClient) getInstanceID() (err error) {
 	return nil
 }
 
-func (client *storageRESTClient) VerifyFile(volume, path string, empty bool, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
+func (client *storageRESTClient) VerifyFile(volume, path string, size int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTFilePath, path)
 	values.Set(storageRESTBitrotAlgo, algo.String())
-	values.Set(storageRESTEmpty, strconv.FormatBool(empty))
-	values.Set(storageRESTLength, strconv.Itoa(int(shardSize)))
-	if len(sum) != 0 {
-		values.Set(storageRESTBitrotHash, hex.EncodeToString(sum))
-	}
+	values.Set(storageRESTLength, strconv.FormatInt(size, 10))
+	values.Set(storageRESTShardSize, strconv.Itoa(int(shardSize)))
+	values.Set(storageRESTBitrotHash, hex.EncodeToString(sum))
+
 	respBody, err := client.call(storageRESTMethodVerifyFile, values, nil, -1)
 	defer http.DrainBody(respBody)
 	if err != nil {

cmd/storage-rest-common.go

@@ -16,7 +16,7 @@
 
 package cmd
 
-const storageRESTVersion = "v8"
+const storageRESTVersion = "v9"
 const storageRESTPath = minioReservedBucketPath + "/storage/" + storageRESTVersion + SlashSeparator
 
 const (
@@ -52,7 +52,7 @@ const (
 	storageRESTDstPath    = "destination-path"
 	storageRESTOffset     = "offset"
 	storageRESTLength     = "length"
-	storageRESTEmpty      = "empty"
+	storageRESTShardSize  = "shard-size"
 	storageRESTCount      = "count"
 	storageRESTMarkerPath = "marker"
 	storageRESTLeafFile   = "leaf-file"

cmd/storage-rest-server.go

@@ -520,12 +520,12 @@ func (s *storageRESTServer) VerifyFile(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	volume := vars[storageRESTVolume]
 	filePath := vars[storageRESTFilePath]
-	empty, err := strconv.ParseBool(vars[storageRESTEmpty])
+	size, err := strconv.ParseInt(vars[storageRESTLength], 10, 0)
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
-	shardSize, err := strconv.Atoi(vars[storageRESTLength])
+	shardSize, err := strconv.Atoi(vars[storageRESTShardSize])
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
@@ -547,7 +547,7 @@ func (s *storageRESTServer) VerifyFile(w http.ResponseWriter, r *http.Request) {
 	algo := BitrotAlgorithmFromString(algoStr)
 	w.Header().Set(xhttp.ContentType, "text/event-stream")
 	doneCh := sendWhiteSpaceVerifyFile(w)
-	err = s.storage.VerifyFile(volume, filePath, empty, algo, hash, int64(shardSize))
+	err = s.storage.VerifyFile(volume, filePath, size, algo, hash, int64(shardSize))
 	<-doneCh
 	gob.NewEncoder(w).Encode(VerifyFileResp{err})
 }
@@ -600,7 +600,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpoints EndpointList) {
 		subrouter.Methods(http.MethodPost).Path(SlashSeparator + storageRESTMethodRenameFile).HandlerFunc(httpTraceHdrs(server.RenameFileHandler)).
 			Queries(restQueries(storageRESTSrcVolume, storageRESTSrcPath, storageRESTDstVolume, storageRESTDstPath)...)
 		subrouter.Methods(http.MethodPost).Path(SlashSeparator + storageRESTMethodVerifyFile).HandlerFunc(httpTraceHdrs(server.VerifyFile)).
-			Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTBitrotAlgo, storageRESTLength, storageRESTEmpty)...)
+			Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTBitrotAlgo, storageRESTBitrotHash, storageRESTLength, storageRESTShardSize)...)
 		subrouter.Methods(http.MethodPost).Path(SlashSeparator + storageRESTMethodGetInstanceID).HandlerFunc(httpTraceAll(server.GetInstanceID))
 	}
 

cmd/sts-datatypes.go

@@ -174,3 +174,19 @@ type ClientGrantsResult struct {
 	// provider as the token's sub (Subject) claim.
 	SubjectFromToken string `xml:",omitempty"`
 }
+
+// AssumeRoleWithLDAPResponse contains the result of successful
+// AssumeRoleWithLDAPIdentity request
+type AssumeRoleWithLDAPResponse struct {
+	XMLName          xml.Name           `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithLDAPIdentityResponse" json:"-"`
+	Result           LDAPIdentityResult `xml:"AssumeRoleWithLDAPIdentityResult"`
+	ResponseMetadata struct {
+		RequestID string `xml:"RequestId,omitempty"`
+	} `xml:"ResponseMetadata,omitempty"`
+}
+
+// LDAPIdentityResult - contains credentials for a successful
+// AssumeRoleWithLDAPIdentity request.
+type LDAPIdentityResult struct {
+	Credentials auth.Credentials `xml:",omitempty"`
+}

cmd/sts-handlers.go

@@ -30,6 +30,7 @@ import (
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/iam/validator"
 	"github.com/minio/minio/pkg/wildcard"
+	ldap "gopkg.in/ldap.v3"
 )
 
 const (
@@ -39,9 +40,14 @@ const (
 	// STS API action constants
 	clientGrants = "AssumeRoleWithClientGrants"
 	webIdentity  = "AssumeRoleWithWebIdentity"
+	ldapIdentity = "AssumeRoleWithLDAPIdentity"
 	assumeRole   = "AssumeRole"
 
 	stsRequestBodyLimit = 10 * (1 << 20) // 10 MiB
+
+	// LDAP claim keys
+	ldapUser   = "ldapUser"
+	ldapGroups = "ldapGroups"
 )
 
 // stsAPIHandlers implements and provides http handlers for AWS STS API.
@@ -82,6 +88,12 @@ func registerSTSRouter(router *mux.Router) {
 		Queries("Version", stsAPIVersion).
 		Queries("WebIdentityToken", "{Token:.*}")
 
+	// AssumeRoleWithLDAPIdentity
+	stsRouter.Methods("POST").HandlerFunc(httpTraceAll(sts.AssumeRoleWithLDAPIdentity)).
+		Queries("Action", ldapIdentity).
+		Queries("Version", stsAPIVersion).
+		Queries("LDAPUsername", "{LDAPUsername:.*}").
+		Queries("LDAPPassword", "{LDAPPassword:.*}")
 }
 
 func checkAssumeRoleAuth(ctx context.Context, r *http.Request) (user auth.Credentials, stsErr STSErrorCode) {
@@ -327,6 +339,10 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
 		}
 	}
 
+	if len(sessionPolicyStr) > 0 {
+		m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicyStr))
+	}
+
 	secret := globalServerConfig.GetCredential().SecretKey
 	cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
 	if err != nil {
@@ -349,10 +365,6 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
 		subFromToken, _ = v.(string)
 	}
 
-	if len(sessionPolicyStr) > 0 {
-		m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicyStr))
-	}
-
 	// Set the newly generated credentials.
 	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred, policyName); err != nil {
 		logger.LogIf(ctx, err)
@@ -411,3 +423,165 @@ func (sts *stsAPIHandlers) AssumeRoleWithWebIdentity(w http.ResponseWriter, r *h
 func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *http.Request) {
 	sts.AssumeRoleWithJWT(w, r)
 }
+
+// AssumeRoleWithLDAPIdentity - implements user auth against LDAP server
+func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AssumeRoleWithLDAPIdentity")
+
+	// Parse the incoming form data.
+	if err := r.ParseForm(); err != nil {
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+		return
+	}
+
+	if r.Form.Get("Version") != stsAPIVersion {
+		logger.LogIf(ctx, fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get("Version"), stsAPIVersion))
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSMissingParameter))
+		return
+	}
+
+	action := r.Form.Get("Action")
+	switch action {
+	case ldapIdentity:
+	default:
+		logger.LogIf(ctx, fmt.Errorf("Unsupported action %s", action))
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+		return
+	}
+
+	ctx = newContext(r, w, action)
+	defer logger.AuditLog(w, r, action, nil)
+
+	ldapUsername := r.Form.Get("LDAPUsername")
+	ldapPassword := r.Form.Get("LDAPPassword")
+
+	if ldapUsername == "" || ldapPassword == "" {
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSMissingParameter))
+		return
+	}
+
+	sessionPolicyStr := r.Form.Get("Policy")
+	// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
+	// The plain text that you use for both inline and managed session
+	// policies shouldn't exceed 2048 characters.
+	if len(sessionPolicyStr) > 2048 {
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+		return
+	}
+
+	if len(sessionPolicyStr) > 0 {
+		sessionPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
+		if err != nil {
+			writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+			return
+		}
+
+		// Version in policy must not be empty
+		if sessionPolicy.Version == "" {
+			writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+			return
+		}
+	}
+
+	ldapConn, err := globalServerConfig.LDAPServerConfig.Connect()
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("LDAP server connection failure: %v", err))
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+		return
+	}
+	if ldapConn == nil {
+		logger.LogIf(ctx, fmt.Errorf("LDAP server not configured: %v", err))
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+		return
+	}
+
+	usernameSubs := newSubstituter("username", ldapUsername)
+	// We ignore error below as we already validated the username
+	// format string at startup.
+	usernameDN, _ := usernameSubs.substitute(globalServerConfig.LDAPServerConfig.UsernameFormat)
+	// Bind with user credentials to validate the password
+	err = ldapConn.Bind(usernameDN, ldapPassword)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("LDAP authentication failure: %v", err))
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+		return
+	}
+
+	groups := []string{}
+	if globalServerConfig.LDAPServerConfig.GroupSearchFilter != "" {
+		// Verified user credentials. Now we find the groups they are
+		// a member of.
+		searchSubs := newSubstituter(
+			"username", ldapUsername,
+			"usernamedn", usernameDN,
+		)
+		// We ignore error below as we already validated the search string
+		// at startup.
+		groupSearchFilter, _ := searchSubs.substitute(globalServerConfig.LDAPServerConfig.GroupSearchFilter)
+		baseDN, _ := searchSubs.substitute(globalServerConfig.LDAPServerConfig.GroupSearchBaseDN)
+		searchRequest := ldap.NewSearchRequest(
+			baseDN,
+			ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
+			groupSearchFilter,
+			[]string{globalServerConfig.LDAPServerConfig.GroupNameAttribute},
+			nil,
+		)
+
+		sr, err := ldapConn.Search(searchRequest)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("LDAP search failure: %v", err))
+			writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+			return
+		}
+		for _, entry := range sr.Entries {
+			// We only queried one attribute, so we only look up
+			// the first one.
+			groups = append(groups, entry.Attributes[0].Values...)
+		}
+	}
+	expiryDur := globalServerConfig.LDAPServerConfig.stsExpiryDuration
+	m := map[string]interface{}{
+		"exp":        UTCNow().Add(expiryDur).Unix(),
+		"ldapUser":   ldapUsername,
+		"ldapGroups": groups,
+	}
+
+	if len(sessionPolicyStr) > 0 {
+		m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicyStr))
+	}
+
+	secret := globalServerConfig.GetCredential().SecretKey
+	cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInternalError))
+		return
+	}
+
+	policyName := ""
+	// Set the newly generated credentials.
+	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred, policyName); err != nil {
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInternalError))
+		return
+	}
+
+	// Notify all other MinIO peers to reload temp users
+	for _, nerr := range globalNotificationSys.LoadUser(cred.AccessKey, true) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+
+	ldapIdentityResponse := &AssumeRoleWithLDAPResponse{
+		Result: LDAPIdentityResult{
+			Credentials: cred,
+		},
+	}
+	ldapIdentityResponse.ResponseMetadata.RequestID = w.Header().Get(xhttp.AmzRequestID)
+	encodedSuccessResponse := encodeResponse(ldapIdentityResponse)
+
+	writeSuccessResponseXML(w, encodedSuccessResponse)
+}

cmd/test-utils_test.go

@@ -1932,18 +1932,18 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
 		t.Fatalf("Initialization of API handler tests failed: <ERROR> %s", err)
 	}
 
-	globalIAMSys = NewIAMSys()
-	globalIAMSys.Init(objLayer)
-
-	globalPolicySys = NewPolicySys()
-	globalPolicySys.Init(objLayer)
-
 	// initialize the server and obtain the credentials and root.
 	// credentials are necessary to sign the HTTP request.
 	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
 		t.Fatalf("Unable to initialize server config. %s", err)
 	}
 
+	globalIAMSys = NewIAMSys()
+	globalIAMSys.Init(objLayer)
+
+	globalPolicySys = NewPolicySys()
+	globalPolicySys.Init(objLayer)
+
 	credentials := globalServerConfig.GetCredential()
 
 	// Executing the object layer tests for single node setup.

cmd/typed-errors.go

@@ -90,5 +90,8 @@ var errGroupNotEmpty = errors.New("Specified group is not empty - cannot remove
 // error returned in IAM subsystem when policy doesn't exist.
 var errNoSuchPolicy = errors.New("Specified canned policy does not exist")
 
+// error returned in IAM subsystem when an external users systems is configured.
+var errIAMActionNotAllowed = errors.New("Specified IAM action is not allowed under the current configuration")
+
 // error returned when access is denied.
 var errAccessDenied = errors.New("Do not have enough permissions to access this resource")

cmd/ui-errors-utils.go

@@ -17,9 +17,9 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
 	"io"
-	"net"
 	"os"
 	"syscall"
 )
@@ -80,27 +80,15 @@ func errorToUIErr(err error) uiErr {
 	}
 
 	// Show a generic message for known golang errors
-	switch e := err.(type) {
-	case *net.OpError:
-		if e.Op == "listen" {
-			if oe, ok := e.Err.(*os.SyscallError); ok {
-				if oe.Err == syscall.EADDRINUSE {
-					return uiErrPortAlreadyInUse(e).Msg("Specified port '" + e.Addr.String() + "' is already in use")
-				} else if oe.Err == syscall.EACCES {
-					return uiErrPortAccess(e).Msg("Insufficient permissions to use specified port '" + e.Addr.String() + "'")
-				}
-			}
-		}
-	case *os.PathError:
-		if os.IsPermission(e) {
-			return uiErrNoPermissionsToAccessDirFiles(e).Msg("Insufficient permissions to access path, `" + e.Path + "`")
-		}
-	}
-
-	switch err {
-	case io.ErrUnexpectedEOF:
+	if errors.Is(err, syscall.EADDRINUSE) {
+		return uiErrPortAlreadyInUse(err).Msg("Specified port is already in use")
+	} else if errors.Is(err, syscall.EACCES) {
+		return uiErrPortAccess(err).Msg("Insufficient permissions to use specified port")
+	} else if os.IsPermission(err) {
+		return uiErrNoPermissionsToAccessDirFiles(err).Msg("Insufficient permissions to access path")
+	} else if errors.Is(err, io.ErrUnexpectedEOF) {
 		return uiErrUnexpectedDataContent(err)
-	default:
+	} else {
 		// Failed to identify what type of error this, return a simple UI error
 		return uiErr{msg: err.Error()}
 	}

cmd/update.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"crypto"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -34,6 +35,7 @@ import (
 	"github.com/inconshreveable/go-update"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	xnet "github.com/minio/minio/pkg/net"
 	_ "github.com/minio/sha256-simd" // Needed for sha256 hash verifier.
 )
 
@@ -282,7 +284,7 @@ func downloadReleaseURL(releaseChecksumURL string, timeout time.Duration, mode s
 	client := &http.Client{Transport: getUpdateTransport(timeout)}
 	resp, err := client.Do(req)
 	if err != nil {
-		if isNetworkOrHostDown(err) {
+		if xnet.IsNetworkOrHostDown(err) {
 			return content, AdminError{
 				Code:       AdminUpdateURLNotReachable,
 				Message:    err.Error(),
@@ -499,7 +501,7 @@ func doUpdate(updateURL, sha256Hex, mode string) (err error) {
 
 	resp, err := clnt.Do(req)
 	if err != nil {
-		if isNetworkOrHostDown(err) {
+		if xnet.IsNetworkOrHostDown(err) {
 			return AdminError{
 				Code:       AdminUpdateURLNotReachable,
 				Message:    err.Error(),
@@ -521,10 +523,19 @@ func doUpdate(updateURL, sha256Hex, mode string) (err error) {
 			Checksum: sha256Sum,
 		},
 	); err != nil {
-		if os.IsPermission(err) {
+		if rerr := update.RollbackError(err); rerr != nil {
 			return AdminError{
 				Code:       AdminUpdateApplyFailure,
-				Message:    err.Error(),
+				Message:    fmt.Sprintf("Failed to rollback from bad update: %v", rerr),
+				StatusCode: http.StatusInternalServerError,
+			}
+		}
+		var pathErr *os.PathError
+		if errors.As(err, &pathErr) {
+			return AdminError{
+				Code: AdminUpdateApplyFailure,
+				Message: fmt.Sprintf("Unable to update the binary at %s: %v",
+					filepath.Dir(pathErr.Path), pathErr.Err),
 				StatusCode: http.StatusForbidden,
 			}
 		}

cmd/utils.go

@@ -29,7 +29,6 @@ import (
 	"io/ioutil"
 	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -53,25 +52,13 @@ func IsErrIgnored(err error, ignoredErrs ...error) bool {
 // IsErr returns whether given error is exact error.
 func IsErr(err error, errs ...error) bool {
 	for _, exactErr := range errs {
-		if err == exactErr {
+		if errors.Is(err, exactErr) {
 			return true
 		}
 	}
 	return false
 }
 
-// make a copy of http.Header
-func cloneHeader(h http.Header) http.Header {
-	h2 := make(http.Header, len(h))
-	for k, vv := range h {
-		vv2 := make([]string, len(vv))
-		copy(vv2, vv)
-		h2[k] = vv2
-
-	}
-	return h2
-}
-
 func request2BucketObjectName(r *http.Request) (bucketName, objectName string) {
 	path, err := getResource(r.URL.Path, r.Host, globalDomainNames)
 	if err != nil {
@@ -289,7 +276,7 @@ var globalProfiler minioProfiler
 
 // dump the request into a string in JSON format.
 func dumpRequest(r *http.Request) string {
-	header := cloneHeader(r.Header)
+	header := r.Header.Clone()
 	header.Set("Host", r.Host)
 	// Replace all '%' to '%%' so that printer format parser
 	// to ignore URL encoded values.
@@ -438,43 +425,6 @@ func newContext(r *http.Request, w http.ResponseWriter, api string) context.Cont
 	return logger.SetReqInfo(r.Context(), reqInfo)
 }
 
-// isNetworkOrHostDown - if there was a network error or if the host is down.
-func isNetworkOrHostDown(err error) bool {
-	if err == nil {
-		return false
-	}
-	// We need to figure if the error either a timeout
-	// or a non-temporary error.
-	e, ok := err.(net.Error)
-	if ok {
-		urlErr, ok := e.(*url.Error)
-		if ok {
-			switch urlErr.Err.(type) {
-			case *net.DNSError, *net.OpError, net.UnknownNetworkError:
-				return true
-			}
-		}
-		if e.Timeout() {
-			return true
-		}
-	}
-	ok = false
-	// Fallback to other mechanisms.
-	if strings.Contains(err.Error(), "Connection closed by foreign host") {
-		ok = true
-	} else if strings.Contains(err.Error(), "TLS handshake timeout") {
-		// If error is - tlsHandshakeTimeoutError.
-		ok = true
-	} else if strings.Contains(err.Error(), "i/o timeout") {
-		// If error is - tcp timeoutError.
-		ok = true
-	} else if strings.Contains(err.Error(), "connection timed out") {
-		// If err is a net.Dial timeout.
-		ok = true
-	}
-	return ok
-}
-
 // Used for registering with rest handlers (have a look at registerStorageRESTHandlers for usage example)
 // If it is passed ["aaaa", "bbbb"], it returns ["aaaa", "{aaaa:.*}", "bbbb", "{bbbb:.*}"]
 func restQueries(keys ...string) []string {

cmd/utils_test.go

@@ -30,30 +30,6 @@ import (
 	"testing"
 )
 
-// Tests http.Header clone.
-func TestCloneHeader(t *testing.T) {
-	headers := []http.Header{
-		{
-			"Content-Type":   {"text/html; charset=UTF-8"},
-			"Content-Length": {"0"},
-		},
-		{
-			"Content-Length": {"0", "1", "2"},
-		},
-		{
-			"Expires":          {"-1"},
-			"Content-Length":   {"0"},
-			"Content-Encoding": {"gzip"},
-		},
-	}
-	for i, header := range headers {
-		clonedHeader := cloneHeader(header)
-		if !reflect.DeepEqual(header, clonedHeader) {
-			t.Errorf("Test %d failed", i+1)
-		}
-	}
-}
-
 // Tests maximum object size.
 func TestMaxObjectSize(t *testing.T) {
 	sizes := []struct {

cmd/web-handlers.go

@@ -1022,7 +1022,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if objectAPI.IsEncryptionSupported() {
-		if hasServerSideEncryptionHeader(r.Header) && !hasSuffix(object, SlashSeparator) { // handle SSE requests
+		if crypto.IsRequested(r.Header) && !hasSuffix(object, SlashSeparator) { // handle SSE requests
 			rawReader := hashReader
 			var objectEncryptionKey []byte
 			reader, objectEncryptionKey, err = EncryptRequest(hashReader, r, bucket, object, metadata)

cmd/xl-sets.go

@@ -821,13 +821,16 @@ func (f *FileInfoCh) Push(fi FileInfo) {
 	f.Valid = true
 }
 
-// Calculate least entry across multiple FileInfo channels, additionally
-// returns a boolean to indicate if the caller needs to call again.
-func leastEntry(entriesCh []FileInfoCh, readQuorum int) (FileInfo, bool) {
-	var entriesValid = make([]bool, len(entriesCh))
-	var entries = make([]FileInfo, len(entriesCh))
-	for i := range entriesCh {
-		entries[i], entriesValid[i] = entriesCh[i].Pop()
+// Calculate least entry across multiple FileInfo channels,
+// returns the least common entry and the total number of times
+// we found this entry. Additionally also returns a boolean
+// to indicate if the caller needs to call this function
+// again to list the next entry. It is callers responsibility
+// if the caller wishes to list N entries to call leastEntry
+// N times until this boolean is 'false'.
+func leastEntry(entryChs []FileInfoCh, entries []FileInfo, entriesValid []bool) (FileInfo, int, bool) {
+	for i := range entryChs {
+		entries[i], entriesValid[i] = entryChs[i].Pop()
 	}
 
 	var isTruncated = false
@@ -856,9 +859,9 @@ func leastEntry(entriesCh []FileInfoCh, readQuorum int) (FileInfo, bool) {
 	}
 
 	// We haven't been able to find any least entry,
-	// this would mean that we don't have valid.
+	// this would mean that we don't have valid entry.
 	if !found {
-		return lentry, isTruncated
+		return lentry, 0, isTruncated
 	}
 
 	leastEntryCount := 0
@@ -876,50 +879,76 @@ func leastEntry(entriesCh []FileInfoCh, readQuorum int) (FileInfo, bool) {
 
 		// Push all entries which are lexically higher
 		// and will be returned later in Pop()
-		entriesCh[i].Push(entries[i])
+		entryChs[i].Push(entries[i])
 	}
 
-	if readQuorum < 0 {
-		return lentry, isTruncated
-	}
-
-	quorum := lentry.Quorum
-	if quorum == 0 {
-		quorum = readQuorum
-	}
-
-	if leastEntryCount >= quorum {
-		return lentry, isTruncated
-	}
-
-	return leastEntry(entriesCh, readQuorum)
+	return lentry, leastEntryCount, isTruncated
 }
 
 // mergeEntriesCh - merges FileInfo channel to entries upto maxKeys.
-func mergeEntriesCh(entriesCh []FileInfoCh, maxKeys int, readQuorum int) (entries FilesInfo) {
+func mergeEntriesCh(entryChs []FileInfoCh, maxKeys int, totalDrives int, heal bool) (entries FilesInfo) {
 	var i = 0
+	entriesInfos := make([]FileInfo, len(entryChs))
+	entriesValid := make([]bool, len(entryChs))
 	for {
-		fi, valid := leastEntry(entriesCh, readQuorum)
+		fi, quorumCount, valid := leastEntry(entryChs, entriesInfos, entriesValid)
 		if !valid {
+			// We have reached EOF across all entryChs, break the loop.
 			break
 		}
-		if i == maxKeys {
-			entries.IsTruncated = true
-			// Re-insert the last entry so it can be
-			// listed in the next listing iteration.
-			for j := range entriesCh {
-				if !entriesCh[j].Valid {
-					entriesCh[j].Push(fi)
-				}
+
+		rquorum := fi.Quorum
+		// Quorum is zero for all directories.
+		if rquorum == 0 {
+			// Choose N/2 quoroum for directory entries.
+			rquorum = totalDrives / 2
+		}
+
+		if heal {
+			// When healing is enabled, we should
+			// list only objects which need healing.
+			if quorumCount == totalDrives {
+				// Skip good entries.
+				continue
+			}
+		} else {
+			// Regular listing, we skip entries not in quorum.
+			if quorumCount < rquorum {
+				// Skip entries which do not have quorum.
+				continue
 			}
-			break
 		}
 		entries.Files = append(entries.Files, fi)
 		i++
+		if i == maxKeys {
+			entries.IsTruncated = isTruncated(entryChs, entriesInfos, entriesValid)
+			break
+		}
 	}
 	return entries
 }
 
+func isTruncated(entryChs []FileInfoCh, entries []FileInfo, entriesValid []bool) bool {
+	for i := range entryChs {
+		entries[i], entriesValid[i] = entryChs[i].Pop()
+	}
+
+	var isTruncated = false
+	for _, valid := range entriesValid {
+		if !valid {
+			continue
+		}
+		isTruncated = true
+		break
+	}
+	for i := range entryChs {
+		if entriesValid[i] {
+			entryChs[i].Push(entries[i])
+		}
+	}
+	return isTruncated
+}
+
 // Starts a walk channel across all disks and returns a slice.
 func (s *xlSets) startMergeWalks(ctx context.Context, bucket, prefix, marker string, recursive bool, endWalkCh chan struct{}) []FileInfoCh {
 	var entryChs []FileInfoCh
@@ -948,20 +977,30 @@ func (s *xlSets) listObjectsNonSlash(ctx context.Context, bucket, prefix, marker
 	recursive := true
 	entryChs := s.startMergeWalks(context.Background(), bucket, prefix, "", recursive, endWalkCh)
 
-	readQuorum := s.drivesPerSet / 2
 	var objInfos []ObjectInfo
 	var eof bool
 	var prevPrefix string
 
+	entriesValid := make([]bool, len(entryChs))
+	entries := make([]FileInfo, len(entryChs))
 	for {
 		if len(objInfos) == maxKeys {
 			break
 		}
-		result, ok := leastEntry(entryChs, readQuorum)
+		result, quorumCount, ok := leastEntry(entryChs, entries, entriesValid)
 		if !ok {
 			eof = true
 			break
 		}
+		rquorum := result.Quorum
+		// Quorum is zero for all directories.
+		if rquorum == 0 {
+			// Choose N/2 quorum for directory entries.
+			rquorum = s.drivesPerSet / 2
+		}
+		if quorumCount < rquorum {
+			continue
+		}
 
 		var objInfo ObjectInfo
 
@@ -1087,12 +1126,7 @@ func (s *xlSets) listObjects(ctx context.Context, bucket, prefix, marker, delimi
 		entryChs = s.startMergeWalks(context.Background(), bucket, prefix, marker, recursive, endWalkCh)
 	}
 
-	readQuorum := s.drivesPerSet / 2
-	if heal {
-		readQuorum = -1
-	}
-
-	entries := mergeEntriesCh(entryChs, maxKeys, readQuorum)
+	entries := mergeEntriesCh(entryChs, maxKeys, s.drivesPerSet, heal)
 	if len(entries.Files) == 0 {
 		return loi, nil
 	}
@@ -1458,7 +1492,7 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
 		}
 	}
 
-	if !hasAnyErrorsUnformatted(sErrs) {
+	if countErrs(sErrs, errUnformattedDisk) == 0 {
 		// No unformatted disks found disks are either offline
 		// or online, no healing is required.
 		return res, errNoHealRequired

cmd/xl-v1-bucket.go

@@ -43,7 +43,7 @@ func (xl xlObjects) MakeBucketWithLocation(ctx context.Context, bucket, location
 	}
 
 	// Initialize sync waitgroup.
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 
 	// Initialize list of errors.
 	var dErrs = make([]error, len(xl.getDisks()))
@@ -82,7 +82,7 @@ func (xl xlObjects) MakeBucketWithLocation(ctx context.Context, bucket, location
 
 func (xl xlObjects) undoDeleteBucket(bucket string) {
 	// Initialize sync waitgroup.
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 	// Undo previous make bucket entry on all underlying storage disks.
 	for index, disk := range xl.getDisks() {
 		if disk == nil {
@@ -103,7 +103,7 @@ func (xl xlObjects) undoDeleteBucket(bucket string) {
 // undo make bucket operation upon quorum failure.
 func undoMakeBucket(storageDisks []StorageAPI, bucket string) {
 	// Initialize sync waitgroup.
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 	// Undo previous make bucket entry on all underlying storage disks.
 	for index, disk := range storageDisks {
 		if disk == nil {
@@ -245,7 +245,7 @@ func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string) error {
 	defer bucketLock.Unlock()
 
 	// Collect if all disks report volume not found.
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 	var dErrs = make([]error, len(xl.getDisks()))
 
 	// Remove a volume entry on all underlying storage disks.

cmd/xl-v1-healing-common.go

@@ -183,7 +183,7 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			// it needs healing too.
 			for _, part := range partsMetadata[i].Parts {
 				checksumInfo := erasureInfo.GetChecksumInfo(part.Name)
-				err = onlineDisk.VerifyFile(bucket, pathJoin(object, part.Name), part.Size == 0, checksumInfo.Algorithm, checksumInfo.Hash, erasure.ShardSize())
+				err = onlineDisk.VerifyFile(bucket, pathJoin(object, part.Name), erasure.ShardFileSize(part.Size), checksumInfo.Algorithm, checksumInfo.Hash, erasure.ShardSize())
 				if err != nil {
 					isCorrupt := strings.HasPrefix(err.Error(), "Bitrot verification mismatch - expected ")
 					if !isCorrupt && err != errFileNotFound && err != errVolumeNotFound && err != errFileUnexpectedSize {

cmd/xl-v1-healing.go

@@ -57,7 +57,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, bucket string, w
 	dryRun bool) (res madmin.HealResultItem, err error) {
 
 	// Initialize sync waitgroup.
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 
 	// Initialize list of errors.
 	var dErrs = make([]error, len(storageDisks))

cmd/xl-v1-metadata.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
 	"net/http"
@@ -28,6 +27,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/minio/sha256-simd"
+
 	"github.com/minio/minio/cmd/logger"
 )
 
@@ -451,7 +452,7 @@ func renameXLMetadata(ctx context.Context, disks []StorageAPI, srcBucket, srcEnt
 
 // writeUniqueXLMetadata - writes unique `xl.json` content for each disk in order.
 func writeUniqueXLMetadata(ctx context.Context, disks []StorageAPI, bucket, prefix string, xlMetas []xlMetaV1, quorum int) ([]StorageAPI, error) {
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 	var mErrs = make([]error, len(disks))
 
 	// Start writing `xl.json` to all disks in parallel.
@@ -461,19 +462,17 @@ func writeUniqueXLMetadata(ctx context.Context, disks []StorageAPI, bucket, pref
 			continue
 		}
 		wg.Add(1)
+
+		// Pick one xlMeta for a disk at index.
+		xlMetas[index].Erasure.Index = index + 1
+
 		// Write `xl.json` in a routine.
-		go func(index int, disk StorageAPI) {
+		go func(index int, disk StorageAPI, xlMeta xlMetaV1) {
 			defer wg.Done()
 
-			// Pick one xlMeta for a disk at index.
-			xlMetas[index].Erasure.Index = index + 1
-
 			// Write unique `xl.json` for a disk at index.
-			err := writeXLMetadata(ctx, disk, bucket, prefix, xlMetas[index])
-			if err != nil {
-				mErrs[index] = err
-			}
-		}(index, disk)
+			mErrs[index] = writeXLMetadata(ctx, disk, bucket, prefix, xlMeta)
+		}(index, disk, xlMetas[index])
 	}
 
 	// Wait for all the routines.
@@ -482,37 +481,3 @@ func writeUniqueXLMetadata(ctx context.Context, disks []StorageAPI, bucket, pref
 	err := reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, quorum)
 	return evalDisks(disks, mErrs), err
 }
-
-// writeSameXLMetadata - write `xl.json` on all disks in order.
-func writeSameXLMetadata(ctx context.Context, disks []StorageAPI, bucket, prefix string, xlMeta xlMetaV1, writeQuorum int) ([]StorageAPI, error) {
-	var wg = &sync.WaitGroup{}
-	var mErrs = make([]error, len(disks))
-
-	// Start writing `xl.json` to all disks in parallel.
-	for index, disk := range disks {
-		if disk == nil {
-			mErrs[index] = errDiskNotFound
-			continue
-		}
-		wg.Add(1)
-		// Write `xl.json` in a routine.
-		go func(index int, disk StorageAPI, metadata xlMetaV1) {
-			defer wg.Done()
-
-			// Save the disk order index.
-			metadata.Erasure.Index = index + 1
-
-			// Write xl metadata.
-			err := writeXLMetadata(ctx, disk, bucket, prefix, metadata)
-			if err != nil {
-				mErrs[index] = err
-			}
-		}(index, disk, xlMeta)
-	}
-
-	// Wait for all the routines.
-	wg.Wait()
-
-	err := reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, writeQuorum)
-	return evalDisks(disks, mErrs), err
-}

cmd/xl-v1-multipart.go

@@ -56,7 +56,7 @@ func (xl xlObjects) checkUploadIDExists(ctx context.Context, bucket, object, upl
 // Removes part given by partName belonging to a mulitpart upload from minioMetaBucket
 func (xl xlObjects) removeObjectPart(bucket, object, uploadID, partName string) {
 	curpartPath := path.Join(bucket, object, uploadID, partName)
-	wg := sync.WaitGroup{}
+	var wg sync.WaitGroup
 	for i, disk := range xl.getDisks() {
 		if disk == nil {
 			continue
@@ -103,7 +103,7 @@ func (xl xlObjects) statPart(ctx context.Context, bucket, object, uploadID, part
 
 // commitXLMetadata - commit `xl.json` from source prefix to destination prefix in the given slice of disks.
 func commitXLMetadata(ctx context.Context, disks []StorageAPI, srcBucket, srcPrefix, dstBucket, dstPrefix string, quorum int) ([]StorageAPI, error) {
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 	var mErrs = make([]error, len(disks))
 
 	srcJSONFile := path.Join(srcPrefix, xlMetaJSONFile)
@@ -123,13 +123,7 @@ func commitXLMetadata(ctx context.Context, disks []StorageAPI, srcBucket, srcPre
 			defer disk.DeleteFile(srcBucket, srcPrefix)
 
 			// Renames `xl.json` from source prefix to destination prefix.
-			rErr := disk.RenameFile(srcBucket, srcJSONFile, dstBucket, dstJSONFile)
-			if rErr != nil {
-				logger.LogIf(ctx, rErr)
-				mErrs[index] = rErr
-				return
-			}
-			mErrs[index] = nil
+			mErrs[index] = disk.RenameFile(srcBucket, srcJSONFile, dstBucket, dstJSONFile)
 		}(index, disk)
 	}
 	// Wait for all the routines.
@@ -218,16 +212,23 @@ func (xl xlObjects) newMultipartUpload(ctx context.Context, bucket string, objec
 	// success.
 	defer xl.deleteObject(ctx, minioMetaTmpBucket, tempUploadIDPath, writeQuorum, false)
 
+	onlineDisks := xl.getDisks()
+	var partsMetadata = make([]xlMetaV1, len(onlineDisks))
+	for i := range onlineDisks {
+		partsMetadata[i] = xlMeta
+	}
+
+	var err error
 	// Write updated `xl.json` to all disks.
-	disks, err := writeSameXLMetadata(ctx, xl.getDisks(), minioMetaTmpBucket, tempUploadIDPath, xlMeta, writeQuorum)
+	onlineDisks, err = writeUniqueXLMetadata(ctx, onlineDisks, minioMetaTmpBucket, tempUploadIDPath, partsMetadata, writeQuorum)
 	if err != nil {
 		return "", toObjectErr(err, minioMetaTmpBucket, tempUploadIDPath)
 	}
 
 	// Attempt to rename temp upload object to actual upload path object
-	_, rErr := rename(ctx, disks, minioMetaTmpBucket, tempUploadIDPath, minioMetaMultipartBucket, uploadIDPath, true, writeQuorum, nil)
-	if rErr != nil {
-		return "", toObjectErr(rErr, minioMetaMultipartBucket, uploadIDPath)
+	_, err = rename(ctx, onlineDisks, minioMetaTmpBucket, tempUploadIDPath, minioMetaMultipartBucket, uploadIDPath, true, writeQuorum, nil)
+	if err != nil {
+		return "", toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
 	}
 
 	// Return success.
@@ -456,7 +457,8 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	defer xl.deleteObject(ctx, minioMetaTmpBucket, tempXLMetaPath, writeQuorum, false)
 
 	// Writes a unique `xl.json` each disk carrying new checksum related information.
-	if onlineDisks, err = writeUniqueXLMetadata(ctx, onlineDisks, minioMetaTmpBucket, tempXLMetaPath, partsMetadata, writeQuorum); err != nil {
+	onlineDisks, err = writeUniqueXLMetadata(ctx, onlineDisks, minioMetaTmpBucket, tempXLMetaPath, partsMetadata, writeQuorum)
+	if err != nil {
 		return pi, toObjectErr(err, minioMetaTmpBucket, tempXLMetaPath)
 	}
 
@@ -518,7 +520,7 @@ func (xl xlObjects) ListObjectParts(ctx context.Context, bucket, object, uploadI
 
 	reducedErr := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
 	if reducedErr == errXLWriteQuorum {
-		return result, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
+		return result, toObjectErr(reducedErr, minioMetaMultipartBucket, uploadIDPath)
 	}
 
 	_, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)

cmd/xl-v1-object.go

@@ -33,7 +33,7 @@ var objectOpIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied)
 
 // putObjectDir hints the bottom layer to create a new directory.
 func (xl xlObjects) putObjectDir(ctx context.Context, bucket, object string, writeQuorum int) error {
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 
 	errs := make([]error, len(xl.getDisks()))
 	// Prepare object creation in all disks
@@ -335,7 +335,7 @@ func (xl xlObjects) getObject(ctx context.Context, bucket, object string, startO
 
 // getObjectInfoDir - This getObjectInfo is specific to object directory lookup.
 func (xl xlObjects) getObjectInfoDir(ctx context.Context, bucket, object string) (oi ObjectInfo, err error) {
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 
 	errs := make([]error, len(xl.getDisks()))
 	// Prepare object creation in a all disks
@@ -423,7 +423,7 @@ func (xl xlObjects) getObjectInfo(ctx context.Context, bucket, object string) (o
 }
 
 func undoRename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, isDir bool, errs []error) {
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 	// Undo rename object on disks where RenameFile succeeded.
 
 	// If srcEntry/dstEntry are objects then add a trailing slash to copy
@@ -453,7 +453,7 @@ func undoRename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry str
 // the respective underlying storage layer representations.
 func rename(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, isDir bool, writeQuorum int, ignoredErr []error) ([]StorageAPI, error) {
 	// Initialize sync waitgroup.
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 
 	// Initialize list of errors.
 	var errs = make([]error, len(disks))
@@ -737,7 +737,7 @@ func (xl xlObjects) deleteObject(ctx context.Context, bucket, object string, wri
 	}
 
 	// Initialize sync waitgroup.
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 
 	// Initialize list of errors.
 	var dErrs = make([]error, len(disks))

cmd/xl-v1-utils.go

@@ -174,14 +174,8 @@ func readXLMeta(ctx context.Context, disk StorageAPI, bucket string, object stri
 	if len(xlMetaBuf) == 0 {
 		return xlMetaV1{}, errFileNotFound
 	}
-	xlMeta, err = xlMetaV1UnmarshalJSON(ctx, xlMetaBuf)
-	if err != nil {
-		logger.GetReqInfo(ctx).AppendTags("disk", disk.String())
-		logger.LogIf(ctx, err)
-		return xlMetaV1{}, err
-	}
-	// Return structured `xl.json`.
-	return xlMeta, nil
+	logger.GetReqInfo(ctx).AppendTags("disk", disk.String())
+	return xlMetaV1UnmarshalJSON(ctx, xlMetaBuf)
 }
 
 // Reads all `xl.json` metadata as a xlMetaV1 slice.
@@ -189,7 +183,7 @@ func readXLMeta(ctx context.Context, disk StorageAPI, bucket string, object stri
 func readAllXLMetadata(ctx context.Context, disks []StorageAPI, bucket, object string) ([]xlMetaV1, []error) {
 	errs := make([]error, len(disks))
 	metadataArray := make([]xlMetaV1, len(disks))
-	var wg = &sync.WaitGroup{}
+	var wg sync.WaitGroup
 	// Read `xl.json` parallelly across disks.
 	for index, disk := range disks {
 		if disk == nil {
@@ -200,12 +194,7 @@ func readAllXLMetadata(ctx context.Context, disks []StorageAPI, bucket, object s
 		// Read `xl.json` in routine.
 		go func(index int, disk StorageAPI) {
 			defer wg.Done()
-			var err error
-			metadataArray[index], err = readXLMeta(ctx, disk, bucket, object)
-			if err != nil {
-				errs[index] = err
-				return
-			}
+			metadataArray[index], errs[index] = readXLMeta(ctx, disk, bucket, object)
 		}(index, disk)
 	}
 

docs/bucket/notifications/README.md

@@ -1047,7 +1047,59 @@ mc cp myphoto.jpg myminio/images
 
 ```
 kafkacat -b localhost:9092 -t bucketevents
-{"EventType":"s3:ObjectCreated:Put","Key":"images/myphoto.jpg","Records":[{"eventVersion":"2.0","eventSource":"aws:s3","awsRegion":"","eventTime":"2017-01-31T10:01:51Z","eventName":"s3:ObjectCreated:Put","userIdentity":{"principalId":"88QR09S7IOT4X1IBAQ9B"},"requestParameters":{"sourceIPAddress":"192.173.5.2:57904"},"responseElements":{"x-amz-request-id":"149ED2FD25589220","x-minio-origin-endpoint":"http://192.173.5.2:9000"},"s3":{"s3SchemaVersion":"1.0","configurationId":"Config","bucket":{"name":"images","ownerIdentity":{"principalId":"88QR09S7IOT4X1IBAQ9B"},"arn":"arn:aws:s3:::images"},"object":{"key":"myphoto.jpg","size":541596,"eTag":"04451d05b4faf4d62f3d538156115e2a","sequencer":"149ED2FD25589220"}}}],"level":"info","msg":"","time":"2017-01-31T15:31:51+05:30"}
+{
+    "EventName": "s3:ObjectCreated:Put",
+    "Key": "images/myphoto.jpg",
+    "Records": [
+        {
+            "eventVersion": "2.0",
+            "eventSource": "minio:s3",
+            "awsRegion": "",
+            "eventTime": "2019-09-10T17:41:54Z",
+            "eventName": "s3:ObjectCreated:Put",
+            "userIdentity": {
+                "principalId": "AKIAIOSFODNN7EXAMPLE"
+            },
+            "requestParameters": {
+                "accessKey": "AKIAIOSFODNN7EXAMPLE",
+                "region": "",
+                "sourceIPAddress": "192.168.56.192"
+            },
+            "responseElements": {
+                "x-amz-request-id": "15C3249451E12784",
+                "x-minio-deployment-id": "751a8ba6-acb2-42f6-a297-4cdf1cf1fa4f",
+                "x-minio-origin-endpoint": "http://192.168.97.83:9000"
+            },
+            "s3": {
+                "s3SchemaVersion": "1.0",
+                "configurationId": "Config",
+                "bucket": {
+                    "name": "images",
+                    "ownerIdentity": {
+                        "principalId": "AKIAIOSFODNN7EXAMPLE"
+                    },
+                    "arn": "arn:aws:s3:::images"
+                },
+                "object": {
+                    "key": "myphoto.jpg",
+                    "size": 6474,
+                    "eTag": "430f89010c77aa34fc8760696da62d08-1",
+                    "contentType": "image/jpeg",
+                    "userMetadata": {
+                        "content-type": "image/jpeg"
+                    },
+                    "versionId": "1",
+                    "sequencer": "15C32494527B46C5"
+                }
+            },
+            "source": {
+                "host": "192.168.56.192",
+                "port": "",
+                "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0"
+            }
+        }
+    ]
+}
 ```
 
 <a name="webhooks"></a>