Update federation target to etcd/clientv3 (#6119)

With CoreDNS now supporting etcdv3 as the DNS backend, we
can update our federation target to etcdv3. Users will now be
able to use etcdv3 server as the federation backbone.

Minio will update bucket data to etcdv3 and CoreDNS can pick
that data up and serve it as bucket style DNS path.

.travis.yml

@@ -8,27 +8,35 @@ dist: trusty
 
 language: go
 
-os:
-- linux
-
-env:
-- ARCH=x86_64
+matrix:
+  include:
+    - os: linux
+      env:
+        - ARCH=x86_64
+      go: 1.10.1
+      script:
+        - make
+        - diff -au <(gofmt -s -d cmd) <(printf "")
+        - diff -au <(gofmt -s -d pkg) <(printf "")
+        - make test GOFLAGS="-timeout 15m -race -v"
+        - make coverage
+        - node --version
+        - cd browser && yarn && yarn test && cd ..
+    - os: linux-ppc64le
+      env:
+        - ARCH=ppc64le
+      go: 1.10.1
+      script:
+        - make
+        - diff -au <(gofmt -s -d cmd) <(printf "")
+        - diff -au <(gofmt -s -d pkg) <(printf "")
+        - make test GOFLAGS="-timeout 15m -v"
+        - make coverage
+        - node --version
+        - cd browser && yarn && yarn test && cd ..
 
 before_install:
 - nvm install stable
 
-script:
-## Run all the tests
-- make
-- diff -au <(gofmt -s -d cmd) <(printf "")
-- diff -au <(gofmt -s -d pkg) <(printf "")
-- make test GOFLAGS="-timeout 15m -race -v"
-- make coverage
-- node --version
-- cd browser && yarn && yarn test && cd ..
-
 after_success:
-- bash <(curl -s https://codecov.io/bash)
-
-go:
-- '1.10.1'
+- bash <(curl -s https://codecov.io/bash)

Makefile

@@ -50,9 +50,11 @@ deadcode:
 	@${GOPATH}/bin/deadcode -test $(shell go list ./...) || true
 
 spelling:
-	@${GOPATH}/bin/misspell -error `find cmd/`
-	@${GOPATH}/bin/misspell -error `find pkg/`
-	@${GOPATH}/bin/misspell -error `find docs/`
+	@${GOPATH}/bin/misspell -locale US -error `find cmd/`
+	@${GOPATH}/bin/misspell -locale US -error `find pkg/`
+	@${GOPATH}/bin/misspell -locale US -error `find docs/`
+	@${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
+	@${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
 
 # Builds minio, runs the verifiers then runs the tests.
 check: test
@@ -89,7 +91,7 @@ pkg-list:
 # Builds minio and installs it to $GOPATH/bin.
 install: build
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
-	@cp $(PWD)/minio $(GOPATH)/bin/minio
+	@mkdir -p $(GOPATH)/bin && cp $(PWD)/minio $(GOPATH)/bin/minio
 	@echo "Installation successful. To learn more, try \"minio --help\"."
 
 clean:

browser/app/js/browser/Login.js

@@ -25,14 +25,35 @@ import web from "../web"
 import { Redirect } from "react-router-dom"
 
 export class Login extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      accessKey: "",
+      secretKey: ""
+    }
+  }
+
+  // Handle field changes
+  accessKeyChange(e) {
+    this.setState({
+      accessKey: e.target.value
+    })
+  }
+
+  secretKeyChange(e) {
+    this.setState({
+      secretKey: e.target.value
+    })
+  }
+
   handleSubmit(event) {
     event.preventDefault()
     const { showAlert, history } = this.props
     let message = ""
-    if (!document.getElementById("accessKey").value) {
+    if (this.state.accessKey === "") {
       message = "Access Key cannot be empty"
     }
-    if (!document.getElementById("secretKey").value) {
+    if (this.state.secretKey === "") {
       message = "Secret Key cannot be empty"
     }
     if (message) {
@@ -41,8 +62,8 @@ export class Login extends React.Component {
     }
     web
       .Login({
-        username: document.getElementById("accessKey").value,
-        password: document.getElementById("secretKey").value
+        username: this.state.accessKey,
+        password: this.state.secretKey
       })
       .then(res => {
         history.push("/")
@@ -77,6 +98,8 @@ export class Login extends React.Component {
         <div className="l-wrap">
           <form onSubmit={this.handleSubmit.bind(this)}>
             <InputGroup
+              value={this.state.accessKey}
+              onChange={this.accessKeyChange.bind(this)}
               className="ig-dark"
               label="Access Key"
               id="accessKey"
@@ -87,6 +110,8 @@ export class Login extends React.Component {
               autoComplete="username"
             />
             <InputGroup
+              value={this.state.secretKey}
+              onChange={this.secretKeyChange.bind(this)}
               className="ig-dark"
               label="Secret Key"
               id="secretKey"

browser/app/js/browser/__tests__/Login.test.js

@@ -60,20 +60,25 @@ describe("Login", () => {
         alert={{ show: false, type: "danger"}}
         showAlert={showAlertMock}
         clearAlert={clearAlertMock}
-      />,
-      { attachTo: document.body }
+      />
     )
     // case where both keys are empty - displays the second warning
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Secret Key cannot be empty")
 
     // case where access key is empty
-    document.getElementById("secretKey").value = "secretKey"
+    wrapper.setState({
+      accessKey: "",
+      secretKey: "secretKey"
+    })
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Access Key cannot be empty")
 
     // case where secret key is empty
-    document.getElementById("accessKey").value = "accessKey"
+    wrapper.setState({
+      accessKey: "accessKey",
+      secretKey: ""
+    })
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Secret Key cannot be empty")
   })
@@ -85,11 +90,12 @@ describe("Login", () => {
         alert={{ show: false, type: "danger"}}
         showAlert={showAlertMock}
         clearAlert={clearAlertMock}
-      />,
-      { attachTo: document.body }
+      />
     )
-    document.getElementById("accessKey").value = "accessKey"
-    document.getElementById("secretKey").value = "secretKey"
+    wrapper.setState({
+      accessKey: "accessKey",
+      secretKey: "secretKey"
+    })
     wrapper.find("form").simulate("submit")
     expect(web.Login).toHaveBeenCalledWith({
       "username": "accessKey", 

buildscripts/checkdeps.sh

@@ -34,7 +34,7 @@ _init() {
 
 ## FIXME:
 ## In OSX, 'readlink -f' option does not exist, hence
-## we have our own readlink -f behaviour here.
+## we have our own readlink -f behavior here.
 ## Once OSX has the option, below function is good enough.
 ## 
 ## readlink() {
@@ -89,11 +89,11 @@ check_minimum_version() {
 
 assert_is_supported_arch() {
     case "${ARCH}" in
-        x86_64 | amd64 | aarch64 | arm* )
+        x86_64 | amd64 | aarch64 | ppc64le | arm* )
             return
             ;;
         *)
-            echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, arm*]"
+            echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*]"
             exit 1
     esac
 }

buildscripts/gen-ldflags.go

@@ -33,6 +33,7 @@ func genLDFlags(version string) string {
 	ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
 	ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]
 	ldflagsStr += " -X github.com/minio/minio/cmd.GOPATH=" + os.Getenv("GOPATH")
+	ldflagsStr += " -X github.com/minio/minio/cmd.GOROOT=" + os.Getenv("GOROOT")
 	return ldflagsStr
 }
 

cmd/admin-handlers.go

@@ -474,7 +474,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -818,7 +818,7 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 
 	// Update local credentials in memory.
 	globalServerConfig.SetCredential(creds)
-	if err = globalServerConfig.Save(); err != nil {
+	if err = globalServerConfig.Save(getConfigFile()); err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
 		return
 	}

cmd/admin-handlers_test.go

@@ -176,10 +176,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	// Init global heal state
 	initAllHealState(globalIsXL)
 
-	globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
-	if err != nil {
-		return nil, err
-	}
+	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
 
 	// Create new policy system.
 	globalPolicySys = NewPolicySys()

cmd/admin-heal-ops.go

@@ -57,7 +57,7 @@ const (
 var (
 	errHealIdleTimeout      = fmt.Errorf("healing results were not consumed for too long")
 	errHealPushStopNDiscard = fmt.Errorf("heal push stopped due to heal stop signal")
-	errHealStopSignalled    = fmt.Errorf("heal stop signalled")
+	errHealStopSignalled    = fmt.Errorf("heal stop signaled")
 
 	errFnHealFromAPIErr = func(err error) error {
 		errCode := toAPIErrorCode(err)
@@ -301,7 +301,7 @@ type healSequence struct {
 	// current accumulated status of the heal sequence
 	currentStatus healSequenceStatus
 
-	// channel signalled by background routine when traversal has
+	// channel signaled by background routine when traversal has
 	// completed
 	traverseAndHealDoneCh chan error
 
@@ -441,7 +441,7 @@ func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
 	h.currentStatus.updateLock.Unlock()
 
 	// This is a "safe" point for the heal sequence to quit if
-	// signalled externally.
+	// signaled externally.
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}

cmd/admin-rpc-client.go

@@ -179,9 +179,9 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
 
 	for _, hostStr := range GetRemotePeers(endpoints) {
 		host, err := xnet.ParseHost(hostStr)
-		logger.CriticalIf(context.Background(), err)
+		logger.FatalIf(err, "Unable to parse Admin RPC Host", context.Background())
 		rpcClient, err := NewAdminRPCClient(host)
-		logger.CriticalIf(context.Background(), err)
+		logger.FatalIf(err, "Unable to initialize Admin RPC Client", context.Background())
 		adminPeerList = append(adminPeerList, adminPeer{
 			addr:      hostStr,
 			cmdRunner: rpcClient,

cmd/admin-rpc-client.go.orig

@@ -1,650 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014, 2015, 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"net"
-	"os"
-	"path"
-	"path/filepath"
-	"sort"
-	"sync"
-	"time"
-
-	"github.com/minio/minio-go/pkg/set"
-	"github.com/minio/minio/cmd/logger"
-)
-
-const (
-	// Admin service names
-	signalServiceRPC  = "Admin.SignalService"
-	reInitFormatRPC   = "Admin.ReInitFormat"
-	listLocksRPC      = "Admin.ListLocks"
-	serverInfoDataRPC = "Admin.ServerInfoData"
-	getConfigRPC      = "Admin.GetConfig"
-	writeTmpConfigRPC = "Admin.WriteTmpConfig"
-	commitConfigRPC   = "Admin.CommitConfig"
-)
-
-// localAdminClient - represents admin operation to be executed locally.
-type localAdminClient struct {
-}
-
-// remoteAdminClient - represents admin operation to be executed
-// remotely, via RPC.
-type remoteAdminClient struct {
-	*AuthRPCClient
-}
-
-// adminCmdRunner - abstracts local and remote execution of admin
-// commands like service stop and service restart.
-type adminCmdRunner interface {
-	SignalService(s serviceSignal) error
-	ReInitFormat(dryRun bool) error
-	ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
-	ServerInfoData() (ServerInfoData, error)
-	GetConfig() ([]byte, error)
-	WriteTmpConfig(tmpFileName string, configBytes []byte) error
-	CommitConfig(tmpFileName string) error
-}
-
-var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
-
-// SignalService - sends a restart or stop signal to the local server
-func (lc localAdminClient) SignalService(s serviceSignal) error {
-	switch s {
-	case serviceRestart, serviceStop:
-		globalServiceSignalCh <- s
-	default:
-		return errUnsupportedSignal
-	}
-	return nil
-}
-
-// ReInitFormat - re-initialize disk format.
-func (lc localAdminClient) ReInitFormat(dryRun bool) error {
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-	return objectAPI.ReloadFormat(context.Background(), dryRun)
-}
-
-// ListLocks - Fetches lock information from local lock instrumentation.
-func (lc localAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	// check if objectLayer is initialized, if not return.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return nil, errServerNotInitialized
-	}
-	return objectAPI.ListLocks(context.Background(), bucket, prefix, duration)
-}
-
-func (rc remoteAdminClient) SignalService(s serviceSignal) (err error) {
-	switch s {
-	case serviceRestart, serviceStop:
-		reply := AuthRPCReply{}
-		err = rc.Call(signalServiceRPC, &SignalServiceArgs{Sig: s},
-			&reply)
-	default:
-		err = errUnsupportedSignal
-	}
-	return err
-}
-
-// ReInitFormat - re-initialize disk format, remotely.
-func (rc remoteAdminClient) ReInitFormat(dryRun bool) error {
-	reply := AuthRPCReply{}
-	return rc.Call(reInitFormatRPC, &ReInitFormatArgs{
-		DryRun: dryRun,
-	}, &reply)
-}
-
-// ListLocks - Sends list locks command to remote server via RPC.
-func (rc remoteAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	listArgs := ListLocksQuery{
-		Bucket:   bucket,
-		Prefix:   prefix,
-		Duration: duration,
-	}
-	var reply ListLocksReply
-	if err := rc.Call(listLocksRPC, &listArgs, &reply); err != nil {
-		return nil, err
-	}
-	return reply.VolLocks, nil
-}
-
-// ServerInfoData - Returns the server info of this server.
-func (lc localAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
-	if globalBootTime.IsZero() {
-		return sid, errServerNotInitialized
-	}
-
-	// Build storage info
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		return sid, errServerNotInitialized
-	}
-	storage := objLayer.StorageInfo(context.Background())
-
-	return ServerInfoData{
-		StorageInfo: storage,
-		ConnStats:   globalConnStats.toServerConnStats(),
-		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
-		Properties: ServerProperties{
-			Uptime:   UTCNow().Sub(globalBootTime),
-			Version:  Version,
-			CommitID: CommitID,
-			SQSARN:   globalNotificationSys.GetARNList(),
-			Region:   globalServerConfig.GetRegion(),
-		},
-	}, nil
-}
-
-// ServerInfo - returns the server info of the server to which the RPC call is made.
-func (rc remoteAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
-	args := AuthRPCArgs{}
-	reply := ServerInfoDataReply{}
-	err := rc.Call(serverInfoDataRPC, &args, &reply)
-	if err != nil {
-		return sid, err
-	}
-
-	return reply.ServerInfoData, nil
-}
-
-// GetConfig - returns config.json of the local server.
-func (lc localAdminClient) GetConfig() ([]byte, error) {
-	if globalServerConfig == nil {
-		return nil, fmt.Errorf("config not present")
-	}
-
-	return json.Marshal(globalServerConfig)
-}
-
-// GetConfig - returns config.json of the remote server.
-func (rc remoteAdminClient) GetConfig() ([]byte, error) {
-	args := AuthRPCArgs{}
-	reply := ConfigReply{}
-	if err := rc.Call(getConfigRPC, &args, &reply); err != nil {
-		return nil, err
-	}
-	return reply.Config, nil
-}
-
-// WriteTmpConfig - writes config file content to a temporary file on
-// the local server.
-func (lc localAdminClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
-	return writeTmpConfigCommon(tmpFileName, configBytes)
-}
-
-// WriteTmpConfig - writes config file content to a temporary file on
-// a remote node.
-func (rc remoteAdminClient) WriteTmpConfig(tmpFileName string, configBytes []byte) error {
-	wArgs := WriteConfigArgs{
-		TmpFileName: tmpFileName,
-		Buf:         configBytes,
-	}
-
-	err := rc.Call(writeTmpConfigRPC, &wArgs, &WriteConfigReply{})
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return err
-	}
-
-	return nil
-}
-
-// CommitConfig - Move the new config in tmpFileName onto config.json
-// on a local node.
-func (lc localAdminClient) CommitConfig(tmpFileName string) error {
-	configFile := getConfigFile()
-	tmpConfigFile := filepath.Join(getConfigDir(), tmpFileName)
-
-	err := os.Rename(tmpConfigFile, configFile)
-	reqInfo := (&logger.ReqInfo{}).AppendTags("tmpConfigFile", tmpConfigFile)
-	reqInfo.AppendTags("configFile", configFile)
-	ctx := logger.SetReqInfo(context.Background(), reqInfo)
-	logger.LogIf(ctx, err)
-	return err
-}
-
-// CommitConfig - Move the new config in tmpFileName onto config.json
-// on a remote node.
-func (rc remoteAdminClient) CommitConfig(tmpFileName string) error {
-	cArgs := CommitConfigArgs{
-		FileName: tmpFileName,
-	}
-	cReply := CommitConfigReply{}
-	err := rc.Call(commitConfigRPC, &cArgs, &cReply)
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return err
-	}
-
-	return nil
-}
-
-// adminPeer - represents an entity that implements admin API RPCs.
-type adminPeer struct {
-	addr      string
-	cmdRunner adminCmdRunner
-	isLocal   bool
-}
-
-// type alias for a collection of adminPeer.
-type adminPeers []adminPeer
-
-// makeAdminPeers - helper function to construct a collection of adminPeer.
-func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
-	thisPeer := globalMinioAddr
-	if globalMinioHost == "" {
-		// When host is not explicitly provided simply
-		// use the first IPv4.
-		thisPeer = net.JoinHostPort(sortIPs(localIP4.ToSlice())[0], globalMinioPort)
-	}
-	adminPeerList = append(adminPeerList, adminPeer{
-		thisPeer,
-		localAdminClient{},
-		true,
-	})
-
-	hostSet := set.CreateStringSet(globalMinioAddr)
-	cred := globalServerConfig.GetCredential()
-	serviceEndpoint := path.Join(minioReservedBucketPath, adminPath)
-	for _, host := range GetRemotePeers(endpoints) {
-		if hostSet.Contains(host) {
-			continue
-		}
-		hostSet.Add(host)
-		adminPeerList = append(adminPeerList, adminPeer{
-			addr: host,
-			cmdRunner: &remoteAdminClient{newAuthRPCClient(authConfig{
-				accessKey:       cred.AccessKey,
-				secretKey:       cred.SecretKey,
-				serverAddr:      host,
-				serviceEndpoint: serviceEndpoint,
-				secureConn:      globalIsSSL,
-				serviceName:     "Admin",
-			})},
-		})
-	}
-
-	return adminPeerList
-}
-
-// peersReInitFormat - reinitialize remote object layers to new format.
-func peersReInitFormat(peers adminPeers, dryRun bool) error {
-	errs := make([]error, len(peers))
-
-	// Send ReInitFormat RPC call to all nodes.
-	// for local adminPeer this is a no-op.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			if !peer.isLocal {
-				errs[idx] = peer.cmdRunner.ReInitFormat(dryRun)
-			}
-		}(i, peer)
-	}
-	wg.Wait()
-	return nil
-}
-
-// Initialize global adminPeer collection.
-func initGlobalAdminPeers(endpoints EndpointList) {
-	globalAdminPeers = makeAdminPeers(endpoints)
-}
-
-// invokeServiceCmd - Invoke Restart/Stop command.
-func invokeServiceCmd(cp adminPeer, cmd serviceSignal) (err error) {
-	switch cmd {
-	case serviceRestart, serviceStop:
-		err = cp.cmdRunner.SignalService(cmd)
-	}
-	return err
-}
-
-// sendServiceCmd - Invoke Restart command on remote peers
-// adminPeer followed by on the local peer.
-func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
-	// Send service command like stop or restart to all remote nodes and finally run on local node.
-	errs := make([]error, len(cps))
-	var wg sync.WaitGroup
-	remotePeers := cps[1:]
-	for i := range remotePeers {
-		wg.Add(1)
-		go func(idx int) {
-			defer wg.Done()
-			// we use idx+1 because remotePeers slice is 1 position shifted w.r.t cps
-			errs[idx+1] = invokeServiceCmd(remotePeers[idx], cmd)
-		}(i)
-	}
-	wg.Wait()
-	errs[0] = invokeServiceCmd(cps[0], cmd)
-}
-
-// listPeerLocksInfo - fetch list of locks held on the given bucket,
-// matching prefix held longer than duration from all peer servers.
-func listPeerLocksInfo(peers adminPeers, bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
-	// Used to aggregate volume lock information from all nodes.
-	allLocks := make([][]VolumeLockInfo, len(peers))
-	errs := make([]error, len(peers))
-	var wg sync.WaitGroup
-	localPeer := peers[0]
-	remotePeers := peers[1:]
-	for i, remotePeer := range remotePeers {
-		wg.Add(1)
-		go func(idx int, remotePeer adminPeer) {
-			defer wg.Done()
-			// `remotePeers` is right-shifted by one position relative to `peers`
-			allLocks[idx], errs[idx] = remotePeer.cmdRunner.ListLocks(bucket, prefix, duration)
-		}(i+1, remotePeer)
-	}
-	wg.Wait()
-	allLocks[0], errs[0] = localPeer.cmdRunner.ListLocks(bucket, prefix, duration)
-
-	// Summarizing errors received for ListLocks RPC across all
-	// nodes.  N B the possible unavailability of quorum in errors
-	// applies only to distributed setup.
-	errCount, err := reduceErrs(errs, []error{})
-	if err != nil {
-		if errCount >= (len(peers)/2 + 1) {
-			return nil, err
-		}
-		return nil, InsufficientReadQuorum{}
-	}
-
-	// Group lock information across nodes by (bucket, object)
-	// pair. For readability only.
-	paramLockMap := make(map[nsParam][]VolumeLockInfo)
-	for _, nodeLocks := range allLocks {
-		for _, lockInfo := range nodeLocks {
-			param := nsParam{
-				volume: lockInfo.Bucket,
-				path:   lockInfo.Object,
-			}
-			paramLockMap[param] = append(paramLockMap[param], lockInfo)
-		}
-	}
-	groupedLockInfos := []VolumeLockInfo{}
-	for _, volLocks := range paramLockMap {
-		groupedLockInfos = append(groupedLockInfos, volLocks...)
-	}
-	return groupedLockInfos, nil
-}
-
-// uptimeSlice - used to sort uptimes in chronological order.
-type uptimeSlice []struct {
-	err    error
-	uptime time.Duration
-}
-
-func (ts uptimeSlice) Len() int {
-	return len(ts)
-}
-
-func (ts uptimeSlice) Less(i, j int) bool {
-	return ts[i].uptime < ts[j].uptime
-}
-
-func (ts uptimeSlice) Swap(i, j int) {
-	ts[i], ts[j] = ts[j], ts[i]
-}
-
-// getPeerUptimes - returns the uptime since the last time read quorum
-// was established on success. Otherwise returns errXLReadQuorum.
-func getPeerUptimes(peers adminPeers) (time.Duration, error) {
-	// In a single node Erasure or FS backend setup the uptime of
-	// the setup is the uptime of the single minio server
-	// instance.
-	if !globalIsDistXL {
-		return UTCNow().Sub(globalBootTime), nil
-	}
-
-	uptimes := make(uptimeSlice, len(peers))
-
-	// Get up time of all servers.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			serverInfoData, rpcErr := peer.cmdRunner.ServerInfoData()
-			uptimes[idx].uptime, uptimes[idx].err = serverInfoData.Properties.Uptime, rpcErr
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Sort uptimes in chronological order.
-	sort.Sort(uptimes)
-
-	// Pick the readQuorum'th uptime in chronological order. i.e,
-	// the time at which read quorum was (re-)established.
-	readQuorum := len(uptimes) / 2
-	validCount := 0
-	latestUptime := time.Duration(0)
-	for _, uptime := range uptimes {
-		if uptime.err != nil {
-			logger.LogIf(context.Background(), uptime.err)
-			continue
-		}
-
-		validCount++
-		if validCount >= readQuorum {
-			latestUptime = uptime.uptime
-			break
-		}
-	}
-
-	// Less than readQuorum "Admin.Uptime" RPC call returned
-	// successfully, so read-quorum unavailable.
-	if validCount < readQuorum {
-		return time.Duration(0), InsufficientReadQuorum{}
-	}
-
-	return latestUptime, nil
-}
-
-// getPeerConfig - Fetches config.json from all nodes in the setup and
-// returns the one that occurs in a majority of them.
-func getPeerConfig(peers adminPeers) ([]byte, error) {
-	if !globalIsDistXL {
-		return peers[0].cmdRunner.GetConfig()
-	}
-
-	errs := make([]error, len(peers))
-	configs := make([][]byte, len(peers))
-
-	// Get config from all servers.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			configs[idx], errs[idx] = peer.cmdRunner.GetConfig()
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Find the maximally occurring config among peers in a
-	// distributed setup.
-
-	serverConfigs := make([]serverConfig, len(peers))
-	for i, configBytes := range configs {
-		if errs[i] != nil {
-			continue
-		}
-
-		// Unmarshal the received config files.
-		err := json.Unmarshal(configBytes, &serverConfigs[i])
-		if err != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peers[i].addr)
-			ctx := logger.SetReqInfo(context.Background(), reqInfo)
-			logger.LogIf(ctx, err)
-			return nil, err
-		}
-	}
-
-	configJSON, err := getValidServerConfig(serverConfigs, errs)
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return nil, err
-	}
-
-	// Return the config.json that was present quorum or more
-	// number of disks.
-	return json.Marshal(configJSON)
-}
-
-// getValidServerConfig - finds the server config that is present in
-// quorum or more number of servers.
-func getValidServerConfig(serverConfigs []serverConfig, errs []error) (scv serverConfig, e error) {
-	// majority-based quorum
-	quorum := len(serverConfigs)/2 + 1
-
-	// Count the number of disks a config.json was found in.
-	configCounter := make([]int, len(serverConfigs))
-
-	// We group equal serverConfigs by the lowest index of the
-	// same value;  e.g, let us take the following serverConfigs
-	// in a 4-node setup,
-	// serverConfigs == [c1, c2, c1, c1]
-	// configCounter == [3, 1, 0, 0]
-	// c1, c2 are the only distinct values that appear.  c1 is
-	// identified by 0, the lowest index it appears in and c2 is
-	// identified by 1. So, we need to find the number of times
-	// each of these distinct values occur.
-
-	// Invariants:
-
-	// 1. At the beginning of the i-th iteration, the number of
-	// unique configurations seen so far is equal to the number of
-	// non-zero counter values in config[:i].
-
-	// 2. At the beginning of the i-th iteration, the sum of
-	// elements of configCounter[:i] is equal to the number of
-	// non-error configurations seen so far.
-
-	// For each of the serverConfig ...
-	for i := range serverConfigs {
-		// Skip nodes where getConfig failed.
-		if errs[i] != nil {
-			continue
-		}
-		// Check if it is equal to any of the configurations
-		// seen so far. If j == i is reached then we have an
-		// unseen configuration.
-		for j := 0; j <= i; j++ {
-			if j < i && configCounter[j] == 0 {
-				// serverConfigs[j] is known to be
-				// equal to a value that was already
-				// seen. See example above for
-				// clarity.
-				continue
-			} else if j < i && serverConfigs[i].ConfigDiff(&serverConfigs[j]) == "" {
-				// serverConfigs[i] is equal to
-				// serverConfigs[j], update
-				// serverConfigs[j]'s counter since it
-				// is the lower index.
-				configCounter[j]++
-				break
-			} else if j == i {
-				// serverConfigs[i] is equal to no
-				// other value seen before. It is
-				// unique so far.
-				configCounter[i] = 1
-				break
-			} // else invariants specified above are violated.
-		}
-	}
-
-	// We find the maximally occurring server config and check if
-	// there is quorum.
-	var configJSON serverConfig
-	maxOccurrence := 0
-	for i, count := range configCounter {
-		if maxOccurrence < count {
-			maxOccurrence = count
-			configJSON = serverConfigs[i]
-		}
-	}
-
-	// If quorum nodes don't agree.
-	if maxOccurrence < quorum {
-		return scv, errXLWriteQuorum
-	}
-
-	return configJSON, nil
-}
-
-// Write config contents into a temporary file on all nodes.
-func writeTmpConfigPeers(peers adminPeers, tmpFileName string, configBytes []byte) []error {
-	// For a single-node minio server setup.
-	if !globalIsDistXL {
-		err := peers[0].cmdRunner.WriteTmpConfig(tmpFileName, configBytes)
-		return []error{err}
-	}
-
-	errs := make([]error, len(peers))
-
-	// Write config into temporary file on all nodes.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			errs[idx] = peer.cmdRunner.WriteTmpConfig(tmpFileName, configBytes)
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Return bytes written and errors (if any) during writing
-	// temporary config file.
-	return errs
-}
-
-// Move config contents from the given temporary file onto config.json
-// on all nodes.
-func commitConfigPeers(peers adminPeers, tmpFileName string) []error {
-	// For a single-node minio server setup.
-	if !globalIsDistXL {
-		return []error{peers[0].cmdRunner.CommitConfig(tmpFileName)}
-	}
-
-	errs := make([]error, len(peers))
-
-	// Rename temporary config file into configDir/config.json on
-	// all nodes.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			errs[idx] = peer.cmdRunner.CommitConfig(tmpFileName)
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Return errors (if any) received during rename.
-	return errs
-}

cmd/admin-rpc-server.go

@@ -121,7 +121,7 @@ func NewAdminRPCServer() (*xrpc.Server, error) {
 // registerAdminRPCRouter - creates and registers Admin RPC server and its router.
 func registerAdminRPCRouter(router *mux.Router) {
 	rpcServer, err := NewAdminRPCServer()
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to initialize Lock RPC Server", context.Background())
 	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
 	subrouter.Path(adminServiceSubPath).Handler(rpcServer)
 }

cmd/admin-rpc_test.go

@@ -139,10 +139,8 @@ func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
 	}()
 
 	endpoints := new(EndpointList)
-	notificationSys, err := NewNotificationSys(globalServerConfig, *endpoints)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
+
+	notificationSys := NewNotificationSys(globalServerConfig, *endpoints)
 
 	testCases := []struct {
 		bootTime        time.Time

cmd/api-errors.go

@@ -17,11 +17,13 @@
 package cmd
 
 import (
+	"context"
 	"encoding/xml"
 	"fmt"
 	"net/http"
 
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 )
@@ -191,6 +193,7 @@ const (
 	ErrHealMissingBucket
 	ErrHealAlreadyRunning
 	ErrHealOverlappingPaths
+	ErrIncorrectContinuationToken
 )
 
 // error code to APIError structure, these fields carry respective
@@ -689,7 +692,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	ErrStorageFull: {
 		Code:           "XMinioStorageFull",
 		Description:    "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
-		HTTPStatusCode: http.StatusInternalServerError,
+		HTTPStatusCode: http.StatusInsufficientStorage,
 	},
 	ErrRequestBodyParse: {
 		Code:           "XMinioRequestBodyParse",
@@ -840,7 +843,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Object storage backend is unreachable",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
-
+	ErrIncorrectContinuationToken: {
+		Code:           "InvalidArgument",
+		Description:    "The continuation token provided is incorrect",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	// Add your error structure here.
 }
 
@@ -864,6 +871,29 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminInvalidAccessKey
 	case auth.ErrInvalidSecretKeyLength:
 		apiErr = ErrAdminInvalidSecretKey
+	// SSE errors
+	case errInsecureSSERequest:
+		apiErr = ErrInsecureSSECustomerRequest
+	case errInvalidSSEAlgorithm:
+		apiErr = ErrInvalidSSECustomerAlgorithm
+	case errInvalidSSEKey:
+		apiErr = ErrInvalidSSECustomerKey
+	case errMissingSSEKey:
+		apiErr = ErrMissingSSECustomerKey
+	case errMissingSSEKeyMD5:
+		apiErr = ErrMissingSSECustomerKeyMD5
+	case errSSEKeyMD5Mismatch:
+		apiErr = ErrSSECustomerKeyMD5Mismatch
+	case errObjectTampered:
+		apiErr = ErrObjectTampered
+	case errEncryptedObject:
+		apiErr = ErrSSEEncryptedObject
+	case errInvalidSSEParameters:
+		apiErr = ErrInvalidSSECustomerParameters
+	case errSSEKeyMismatch:
+		apiErr = ErrAccessDenied // no access without correct key
+	case context.Canceled, context.DeadlineExceeded:
+		apiErr = ErrOperationTimedOut
 	}
 
 	if apiErr != ErrNone {
@@ -871,27 +901,10 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		return apiErr
 	}
 
-	switch err { // SSE errors
-	case errInsecureSSERequest:
-		return ErrInsecureSSECustomerRequest
-	case errInvalidSSEAlgorithm:
-		return ErrInvalidSSECustomerAlgorithm
-	case errInvalidSSEKey:
-		return ErrInvalidSSECustomerKey
-	case errMissingSSEKey:
-		return ErrMissingSSECustomerKey
-	case errMissingSSEKeyMD5:
-		return ErrMissingSSECustomerKeyMD5
-	case errSSEKeyMD5Mismatch:
-		return ErrSSECustomerKeyMD5Mismatch
-	case errObjectTampered:
-		return ErrObjectTampered
-	case errEncryptedObject:
-		return ErrSSEEncryptedObject
-	case errInvalidSSEParameters:
-		return ErrInvalidSSECustomerParameters
-	case errSSEKeyMismatch:
-		return ErrAccessDenied // no access without correct key
+	// etcd specific errors, a key is always a bucket for us return
+	// ErrNoSuchBucket in such a case.
+	if err == dns.ErrNoEntriesFound {
+		return ErrNoSuchBucket
 	}
 
 	switch err.(type) {

cmd/api-resources.go

@@ -36,7 +36,17 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
 }
 
 // Parse bucket url queries for ListObjects V2.
-func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string) {
+func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
+	// The continuation-token cannot be empty.
+	if val, ok := values["continuation-token"]; ok {
+		if len(val[0]) == 0 {
+			errCode = ErrIncorrectContinuationToken
+			return
+		}
+	}
+
 	prefix = values.Get("prefix")
 	token = values.Get("continuation-token")
 	startAfter = values.Get("start-after")

cmd/api-resources_test.go

@@ -29,6 +29,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 		fetchOwner                           bool
 		maxKeys                              int
 		encodingType                         string
+		errCode                              APIErrorCode
 	}{
 		{
 			values: url.Values{
@@ -47,6 +48,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 			fetchOwner:   true,
 			maxKeys:      100,
 			encodingType: "gzip",
+			errCode:      ErrNone,
 		},
 		{
 			values: url.Values{
@@ -64,11 +66,34 @@ func TestListObjectsV2Resources(t *testing.T) {
 			fetchOwner:   true,
 			maxKeys:      1000,
 			encodingType: "gzip",
+			errCode:      ErrNone,
+		},
+		{
+			values: url.Values{
+				"prefix":             []string{"photos/"},
+				"continuation-token": []string{""},
+				"start-after":        []string{"start-after"},
+				"delimiter":          []string{"/"},
+				"fetch-owner":        []string{"true"},
+				"encoding-type":      []string{"gzip"},
+			},
+			prefix:       "",
+			token:        "",
+			startAfter:   "",
+			delimiter:    "",
+			fetchOwner:   false,
+			maxKeys:      0,
+			encodingType: "",
+			errCode:      ErrIncorrectContinuationToken,
 		},
 	}
 
 	for i, testCase := range testCases {
-		prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType := getListObjectsV2Args(testCase.values)
+		prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(testCase.values)
+
+		if errCode != testCase.errCode {
+			t.Errorf("Test %d: Expected error code:%d, got %d", i+1, testCase.errCode, errCode)
+		}
 		if prefix != testCase.prefix {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix)
 		}

cmd/api-response.go

@@ -21,6 +21,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/pkg/handlers"
@@ -293,8 +294,10 @@ func getObjectLocation(r *http.Request, domain, bucket, object string) string {
 	}
 	// If domain is set then we need to use bucket DNS style.
 	if domain != "" {
-		u.Host = bucket + "." + domain
-		u.Path = path.Join(slashSeparator, object)
+		if strings.Contains(r.Host, domain) {
+			u.Host = bucket + "." + r.Host
+			u.Path = path.Join(slashSeparator, object)
+		}
 	}
 	return u.String()
 }

cmd/bucket-handlers-listobjects.go

@@ -70,20 +70,19 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	// Extract all the listObjectsV2 query params to their native values.
-	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _ := getListObjectsV2Args(r.URL.Query())
+	urlValues := r.URL.Query()
 
-	// In ListObjectsV2 'continuation-token' is the marker.
-	marker := token
-	// Check if 'continuation-token' is empty.
-	if token == "" {
-		// Then we need to use 'start-after' as marker instead.
-		marker = startAfter
+	// Extract all the listObjectsV2 query params to their native values.
+	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _, errCode := getListObjectsV2Args(urlValues)
+
+	if errCode != ErrNone {
+		writeErrorResponse(w, errCode, r.URL)
+		return
 	}
 
 	// Validate the query params before beginning to serve the request.
 	// fetch-owner is not validated since it is a boolean
-	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
+	if s3Error := validateListObjectsArgs(prefix, token, delimiter, maxKeys); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -93,8 +92,8 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	}
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
-	// marshalled into S3 compatible XML header.
-	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, marker, delimiter, maxKeys, fetchOwner, startAfter)
+	// marshaled into S3 compatible XML header.
+	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -159,7 +158,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
-	// marshalled into S3 compatible XML header.
+	// marshaled into S3 compatible XML header.
 	listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)

cmd/bucket-handlers.go

@@ -17,8 +17,10 @@
 package cmd
 
 import (
+	"context"
 	"encoding/base64"
 	"encoding/xml"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -29,12 +31,58 @@ import (
 	"sync"
 
 	"github.com/gorilla/mux"
+
+	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
+// Check if there are buckets on server without corresponding entry in etcd backend and
+// make entries. Here is the general flow
+// - Range over all the available buckets
+// - Check if a bucket has an entry in etcd backend
+// -- If no, make an entry
+// -- If yes, check if the IP of entry matches local IP. This means entry is for this instance.
+// -- If IP of the entry doesn't match, this means entry is for another instance. Log an error to console.
+func initFederatorBackend(objLayer ObjectLayer) {
+	b, err := objLayer.ListBuckets(context.Background())
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return
+	}
+
+	g := errgroup.WithNErrs(len(b))
+	for index := range b {
+		index := index
+		g.Go(func() error {
+			r, gerr := globalDNSConfig.Get(b[index].Name)
+			if gerr != nil {
+				if gerr == dns.ErrNoEntriesFound {
+					return globalDNSConfig.Put(b[index].Name)
+				}
+				return gerr
+			}
+			if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				// There is already an entry for this bucket, with all IP addresses different. This indicates a bucket name collision. Log an error and continue.
+				return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", b[index].Name, globalDomainIPs.ToSlice())
+			}
+			return nil
+		}, index)
+	}
+
+	for _, err := range g.Wait() {
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			return
+		}
+	}
+}
+
 // GetBucketLocationHandler - GET Bucket location.
 // -------------------------
 // This operation returns bucket location.
@@ -157,12 +205,33 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
-
-	// Invoke the list buckets.
-	bucketsInfo, err := listBuckets(ctx)
-	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
+	// If etcd, dns federation configured list buckets from etcd.
+	var bucketsInfo []BucketInfo
+	if globalDNSConfig != nil {
+		dnsBuckets, err := globalDNSConfig.List()
+		if err != nil && err != dns.ErrNoEntriesFound {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
+		bucketSet := set.NewStringSet()
+		for _, dnsRecord := range dnsBuckets {
+			if bucketSet.Contains(dnsRecord.Key) {
+				continue
+			}
+			bucketsInfo = append(bucketsInfo, BucketInfo{
+				Name:    strings.Trim(dnsRecord.Key, slashSeparator),
+				Created: dnsRecord.CreationDate,
+			})
+			bucketSet.Add(dnsRecord.Key)
+		}
+	} else {
+		// Invoke the list buckets.
+		var err error
+		bucketsInfo, err = listBuckets(ctx)
+		if err != nil {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
 	}
 
 	// Generate response.
@@ -211,7 +280,13 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	}
 
 	// Allocate incoming content length bytes.
-	deleteXMLBytes := make([]byte, r.ContentLength)
+	var deleteXMLBytes []byte
+	const maxBodySize = 2 * 1000 * 1024 // The max. XML contains 1000 object names (each at most 1024 bytes long) + XML overhead
+	if r.ContentLength > maxBodySize {  // Only allocated memory for at most 1000 objects
+		deleteXMLBytes = make([]byte, maxBodySize)
+	} else {
+		deleteXMLBytes = make([]byte, r.ContentLength)
+	}
 
 	// Read incoming body XML bytes.
 	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
@@ -298,7 +373,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	// Get host and port from Request.RemoteAddr failing which
 	// fill them with empty strings.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -353,12 +428,33 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+	if globalDNSConfig != nil {
+		if _, err := globalDNSConfig.Get(bucket); err != nil {
+			if err == dns.ErrNoEntriesFound {
+				// Proceed to creating a bucket.
+				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
+					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					return
+				}
+				if err = globalDNSConfig.Put(bucket); err != nil {
+					objectAPI.DeleteBucket(ctx, bucket)
+					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					return
+				}
+
+				// Make sure to add Location information here only for bucket
+				w.Header().Set("Location", getObjectLocation(r, globalDomainName, bucket, ""))
+
+				writeSuccessResponseHeadersOnly(w)
+				return
+			}
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+
+		}
+		writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL)
 		return
 	}
-	defer bucketLock.Unlock()
 
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
@@ -503,7 +599,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 
 	// Extract metadata to be saved from received Form.
-	metadata, err := extractMetadataFromHeader(ctx, formValues)
+	metadata := make(map[string]string)
+	err = extractMetadataFromMap(ctx, formValues, metadata)
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
@@ -525,7 +622,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
 			}
-			reader, err = newEncryptReader(hashReader, key, metadata)
+			reader, err = newEncryptReader(hashReader, key, bucket, object, metadata)
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -550,7 +647,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	w.Header().Set("Location", location)
 
 	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -655,9 +752,15 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	globalNotificationSys.RemoveNotification(bucket)
 	globalPolicySys.Remove(bucket)
-	for nerr := range globalNotificationSys.DeleteBucket(bucket) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
+	globalNotificationSys.DeleteBucket(ctx, bucket)
+
+	if globalDNSConfig != nil {
+		if err := globalDNSConfig.Delete(bucket); err != nil {
+			// Deleting DNS entry failed, attempt to create the bucket again.
+			objectAPI.MakeBucketWithLocation(ctx, bucket, "")
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
 	}
 
 	// Write success response.

cmd/bucket-notification-handlers.go

@@ -146,10 +146,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 
 	rulesMap := config.ToRulesMap()
 	globalNotificationSys.AddRulesMap(bucketName, rulesMap)
-	for nerr := range globalNotificationSys.PutBucketNotification(bucketName, rulesMap) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.PutBucketNotification(ctx, bucketName, rulesMap)
 
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -224,11 +221,17 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
-	host, e := xnet.ParseHost(r.RemoteAddr)
-	logger.CriticalIf(ctx, e)
+	host, err := xnet.ParseHost(r.RemoteAddr)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
 
-	target, e := target.NewHTTPClientTarget(*host, w)
-	logger.CriticalIf(ctx, e)
+	target, err := target.NewHTTPClientTarget(*host, w)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
 
 	rulesMap := event.NewRulesMap(eventNames, pattern, target.ID())
 
@@ -241,25 +244,24 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
 	defer globalNotificationSys.RemoveRulesMap(bucketName, rulesMap)
 
-	thisAddr, e := xnet.ParseHost(GetLocalPeer(globalEndpoints))
-	logger.CriticalIf(ctx, e)
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
 
-	if err := SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
+	if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
 		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	errCh := globalNotificationSys.ListenBucketNotification(bucketName, eventNames, pattern, target.ID(), *thisAddr)
-	for nerr := range errCh {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.ListenBucketNotification(ctx, bucketName, eventNames, pattern, target.ID(), *thisAddr)
 
 	<-target.DoneCh
 
-	if err := RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
+	if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
 		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)

cmd/bucket-policy-handlers.go

@@ -91,10 +91,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	globalPolicySys.Set(bucket, *bucketPolicy)
-	for nerr := range globalNotificationSys.SetBucketPolicy(bucket, bucketPolicy) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.SetBucketPolicy(ctx, bucket, bucketPolicy)
 
 	// Success.
 	writeSuccessNoContent(w)
@@ -130,10 +127,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	}
 
 	globalPolicySys.Remove(bucket)
-	for nerr := range globalNotificationSys.RemoveBucketPolicy(bucket) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.RemoveBucketPolicy(ctx, bucket)
 
 	// Success.
 	writeSuccessNoContent(w)

cmd/build-constants.go

@@ -22,6 +22,9 @@ var (
 	// GOPATH - GOPATH value at the time of build.
 	GOPATH = ""
 
+	// GOROOT - GOROOT value at the time of build.
+	GOROOT = ""
+
 	// Go get development tag.
 	goGetTag = "DEVELOPMENT.GOGET"
 

cmd/certs.go

@@ -86,10 +86,11 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 		return nil, nil
 	}
 
-	rootCAs, err := x509.SystemCertPool()
-	if err != nil {
-		// In some systems like Windows, system cert pool is not supported.
-		// Hence we create a new cert pool.
+	rootCAs, _ := x509.SystemCertPool()
+	if rootCAs == nil {
+		// In some systems (like Windows) system cert pool is
+		// not supported or no certificates are present on the
+		// system - so we create a new cert pool.
 		rootCAs = x509.NewCertPool()
 	}
 

cmd/common-main.go

@@ -17,16 +17,23 @@
 package cmd
 
 import (
+	"context"
 	"errors"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
+	etcd "github.com/coreos/etcd/clientv3"
+
 	"github.com/minio/cli"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dns"
+
+	"github.com/minio/minio-go/pkg/set"
 )
 
 // Check for updates and print a notification message
@@ -41,12 +48,34 @@ func checkUpdate(mode string) {
 	}
 }
 
+// Initialize and load config from remote etcd or local config directory
 func initConfig() {
-	// Config file does not exist, we create it fresh and return upon success.
+	if globalEtcdClient != nil {
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		resp, err := globalEtcdClient.Get(ctx, getConfigFile())
+		cancel()
+		// This means there are no entries in etcd with config file
+		// So create a new config
+		if err == nil && resp.Count == 0 {
+			logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time.")
+			logger.Info("Created minio configuration file successfully at %v", globalEtcdClient.Endpoints())
+		} else {
+			// This means there is an entry in etcd, update it if required and proceed
+			if err == nil && resp.Count > 0 {
+				logger.FatalIf(migrateConfig(), "Config migration failed.")
+				logger.FatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion)
+			} else {
+				logger.FatalIf(err, "Unable to load config version: '%s'.", serverConfigVersion)
+			}
+		}
+		return
+	}
+
 	if isFile(getConfigFile()) {
 		logger.FatalIf(migrateConfig(), "Config migration failed")
 		logger.FatalIf(loadConfig(), "Unable to load the configuration file")
 	} else {
+		// Config file does not exist, we create it fresh and return upon success.
 		logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time")
 		logger.Info("Created minio configuration file successfully at " + getConfigDir())
 	}
@@ -123,8 +152,37 @@ func handleCommonEnvVars() {
 		logger.FatalIf(err, "error opening file %s", traceFile)
 	}
 
+	etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
+	if ok {
+		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
+		var err error
+		globalEtcdClient, err = etcd.New(etcd.Config{
+			Endpoints:         etcdEndpoints,
+			DialTimeout:       defaultDialTimeout,
+			DialKeepAliveTime: defaultDialKeepAlive,
+		})
+		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
+	}
+
 	globalDomainName, globalIsEnvDomainName = os.LookupEnv("MINIO_DOMAIN")
 
+	minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
+	if ok {
+		minioEndpoints := strings.Split(minioEndpointsEnv, ",")
+		globalDomainIPs = set.NewStringSet()
+		for i, ip := range minioEndpoints {
+			if net.ParseIP(ip) == nil {
+				logger.FatalIf(errInvalidArgument, "Unable to initialize Minio server with invalid MINIO_PUBLIC_IPS[%d]: %s", i, ip)
+			}
+			globalDomainIPs.Add(ip)
+		}
+	}
+	if globalDomainName != "" && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
+		var err error
+		globalDNSConfig, err = dns.NewCoreDNS(globalDomainName, globalDomainIPs, globalMinioPort, globalEtcdClient)
+		logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainName)
+	}
+
 	if drives := os.Getenv("MINIO_CACHE_DRIVES"); drives != "" {
 		driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter))
 		if err != nil {
@@ -150,6 +208,16 @@ func handleCommonEnvVars() {
 		globalCacheExpiry = expiry
 	}
 
+	if maxUseStr := os.Getenv("MINIO_CACHE_MAXUSE"); maxUseStr != "" {
+		maxUse, err := strconv.Atoi(maxUseStr)
+		if err != nil {
+			logger.Fatal(uiErrInvalidCacheMaxUse(err), "Unable to parse MINIO_CACHE_MAXUSE value (`%s`)", maxUseStr)
+		}
+		// maxUse should be a valid percentage.
+		if maxUse > 0 && maxUse <= 100 {
+			globalCacheMaxUse = maxUse
+		}
+	}
 	// In place update is true by default if the MINIO_UPDATE is not set
 	// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
 	// in-place update is off.

cmd/config-current.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"reflect"
@@ -39,9 +40,9 @@ import (
 // 6. Make changes in config-current_test.go for any test change
 
 // Config version
-const serverConfigVersion = "25"
+const serverConfigVersion = "26"
 
-type serverConfig = serverConfigV25
+type serverConfig = serverConfigV26
 
 var (
 	// globalServerConfig server config.
@@ -116,10 +117,11 @@ func (s *serverConfig) GetWorm() bool {
 }
 
 // SetCacheConfig sets the current cache config
-func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int) {
+func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int, maxuse int) {
 	s.Cache.Drives = drives
 	s.Cache.Exclude = exclude
 	s.Cache.Expiry = expiry
+	s.Cache.MaxUse = maxuse
 }
 
 // GetCacheConfig gets the current cache config
@@ -127,10 +129,19 @@ func (s *serverConfig) GetCacheConfig() CacheConfig {
 	return s.Cache
 }
 
-// Save config.
-func (s *serverConfig) Save() error {
-	// Save config file.
-	return quick.Save(getConfigFile(), s)
+// Save config file to corresponding backend
+func Save(configFile string, data interface{}) error {
+	return quick.SaveConfig(data, configFile, globalEtcdClient)
+}
+
+// Load config from backend
+func Load(configFile string, data interface{}) (quick.Config, error) {
+	return quick.LoadConfig(configFile, globalEtcdClient, data)
+}
+
+// GetVersion gets config version from backend
+func GetVersion(configFile string) (string, error) {
+	return quick.GetVersion(configFile, globalEtcdClient)
 }
 
 // Returns the string describing a difference with the given
@@ -196,6 +207,7 @@ func newServerConfig() *serverConfig {
 			Drives:  []string{},
 			Exclude: []string{},
 			Expiry:  globalCacheExpiry,
+			MaxUse:  globalCacheMaxUse,
 		},
 		Notify: notifier{},
 	}
@@ -223,6 +235,7 @@ func newServerConfig() *serverConfig {
 	srvCfg.Cache.Drives = make([]string, 0)
 	srvCfg.Cache.Exclude = make([]string, 0)
 	srvCfg.Cache.Expiry = globalCacheExpiry
+	srvCfg.Cache.MaxUse = globalCacheMaxUse
 	return srvCfg
 }
 
@@ -230,7 +243,10 @@ func newServerConfig() *serverConfig {
 // found, otherwise use default parameters
 func newConfig() error {
 	// Initialize server config.
-	srvCfg := newServerConfig()
+	srvCfg, err := newQuickConfig(newServerConfig())
+	if err != nil {
+		return err
+	}
 
 	// If env is set override the credentials from config file.
 	if globalIsEnvCreds {
@@ -258,7 +274,7 @@ func newConfig() error {
 	}
 
 	if globalIsDiskCacheEnabled {
-		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry)
+		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
 	}
 
 	// hold the mutex lock before a new config is assigned.
@@ -269,7 +285,19 @@ func newConfig() error {
 	globalServerConfigMu.Unlock()
 
 	// Save config into file.
-	return globalServerConfig.Save()
+	return Save(getConfigFile(), globalServerConfig)
+}
+
+// newQuickConfig - initialize a new server config, with an allocated
+// quick.Config interface.
+func newQuickConfig(srvCfg *serverConfig) (*serverConfig, error) {
+	qcfg, err := quick.NewConfig(srvCfg, globalEtcdClient)
+	if err != nil {
+		return nil, err
+	}
+
+	srvCfg.Config = qcfg
+	return srvCfg, nil
 }
 
 // getValidConfig - returns valid server configuration
@@ -279,7 +307,14 @@ func getValidConfig() (*serverConfig, error) {
 		Browser: true,
 	}
 
-	if _, err := quick.Load(getConfigFile(), srvCfg); err != nil {
+	var err error
+	srvCfg, err = newQuickConfig(srvCfg)
+	if err != nil {
+		return nil, err
+	}
+
+	configFile := getConfigFile()
+	if err = srvCfg.Load(configFile); err != nil {
 		return nil, err
 	}
 
@@ -327,7 +362,7 @@ func loadConfig() error {
 	}
 
 	if globalIsDiskCacheEnabled {
-		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry)
+		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
 	}
 
 	// hold the mutex lock before a new config is assigned.
@@ -356,6 +391,7 @@ func loadConfig() error {
 		globalCacheDrives = cacheConf.Drives
 		globalCacheExcludes = cacheConf.Exclude
 		globalCacheExpiry = cacheConf.Expiry
+		globalCacheMaxUse = cacheConf.MaxUse
 	}
 	globalServerConfigMu.Unlock()
 
@@ -367,17 +403,19 @@ func loadConfig() error {
 // * Add a new target in pkg/event/target package.
 // * Add newly added target configuration to serverConfig.Notify.<TARGET_NAME>.
 // * Handle the configuration in this function to create/add into TargetList.
-func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
+func getNotificationTargets(config *serverConfig) *event.TargetList {
 	targetList := event.NewTargetList()
 
 	for id, args := range config.Notify.AMQP {
 		if args.Enable {
 			newTarget, err := target.NewAMQPTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -386,10 +424,14 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewElasticsearchTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
+
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
+
 			}
 		}
 	}
@@ -398,10 +440,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewKafkaTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -410,10 +454,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewMQTTTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -422,10 +468,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewMySQLTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -434,10 +482,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewNATSTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -446,10 +496,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewPostgreSQLTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -458,10 +510,12 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget, err := target.NewRedisTarget(id, args)
 			if err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 			if err = targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
@@ -470,10 +524,11 @@ func getNotificationTargets(config *serverConfig) (*event.TargetList, error) {
 		if args.Enable {
 			newTarget := target.NewWebhookTarget(id, args)
 			if err := targetList.Add(newTarget); err != nil {
-				return nil, err
+				logger.LogIf(context.Background(), err)
+				continue
 			}
 		}
 	}
 
-	return targetList, nil
+	return targetList
 }

cmd/config-current_test.go

@@ -50,7 +50,7 @@ func TestServerConfig(t *testing.T) {
 	}
 
 	// Attempt to save.
-	if err := globalServerConfig.Save(); err != nil {
+	if err := globalServerConfig.Save(getConfigFile()); err != nil {
 		t.Fatalf("Unable to save updated config file %s", err)
 	}
 

cmd/config-migrate.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
 	xnet "github.com/minio/minio/pkg/net"
@@ -42,7 +43,7 @@ func migrateConfig() error {
 	}
 
 	// Load only config version information.
-	version, err := quick.GetVersion(getConfigFile())
+	version, err := GetVersion(getConfigFile())
 	if err != nil {
 		return err
 	}
@@ -182,6 +183,11 @@ func migrateConfig() error {
 			return err
 		}
 		fallthrough
+	case "25":
+		if err = migrateV25ToV26(); err != nil {
+			return err
+		}
+		fallthrough
 	case serverConfigVersion:
 		// No migration needed. this always points to current version.
 		err = nil
@@ -194,8 +200,8 @@ func purgeV1() error {
 	configFile := filepath.Join(getConfigDir(), "fsUsers.json")
 
 	cv1 := &configV1{}
-	_, err := quick.Load(configFile, cv1)
-	if os.IsNotExist(err) {
+	_, err := Load(configFile, cv1)
+	if os.IsNotExist(err) || err == dns.ErrNoEntriesFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘1’. %v", err)
@@ -215,7 +221,7 @@ func migrateV2ToV3() error {
 	configFile := getConfigFile()
 
 	cv2 := &configV2{}
-	_, err := quick.Load(configFile, cv2)
+	_, err := Load(configFile, cv2)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -259,7 +265,7 @@ func migrateV2ToV3() error {
 	}
 	srvConfig.Logger.Syslog = slogger
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv2.Version, srvConfig.Version, err)
 	}
 
@@ -274,7 +280,7 @@ func migrateV3ToV4() error {
 	configFile := getConfigFile()
 
 	cv3 := &configV3{}
-	_, err := quick.Load(configFile, cv3)
+	_, err := Load(configFile, cv3)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -297,7 +303,7 @@ func migrateV3ToV4() error {
 	srvConfig.Logger.File = cv3.Logger.File
 	srvConfig.Logger.Syslog = cv3.Logger.Syslog
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv3.Version, srvConfig.Version, err)
 	}
 
@@ -312,7 +318,7 @@ func migrateV4ToV5() error {
 	configFile := getConfigFile()
 
 	cv4 := &configV4{}
-	_, err := quick.Load(configFile, cv4)
+	_, err := Load(configFile, cv4)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -338,7 +344,7 @@ func migrateV4ToV5() error {
 	srvConfig.Logger.ElasticSearch.Enable = false
 	srvConfig.Logger.Redis.Enable = false
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv4.Version, srvConfig.Version, err)
 	}
 
@@ -353,7 +359,7 @@ func migrateV5ToV6() error {
 	configFile := getConfigFile()
 
 	cv5 := &configV5{}
-	_, err := quick.Load(configFile, cv5)
+	_, err := Load(configFile, cv5)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -427,7 +433,7 @@ func migrateV5ToV6() error {
 		}
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv5.Version, srvConfig.Version, err)
 	}
 
@@ -442,7 +448,7 @@ func migrateV6ToV7() error {
 	configFile := getConfigFile()
 
 	cv6 := &configV6{}
-	_, err := quick.Load(configFile, cv6)
+	_, err := Load(configFile, cv6)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -483,7 +489,7 @@ func migrateV6ToV7() error {
 		srvConfig.Notify.Redis = cv6.Notify.Redis
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv6.Version, srvConfig.Version, err)
 	}
 
@@ -498,7 +504,7 @@ func migrateV7ToV8() error {
 	configFile := getConfigFile()
 
 	cv7 := &serverConfigV7{}
-	_, err := quick.Load(configFile, cv7)
+	_, err := Load(configFile, cv7)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -546,7 +552,7 @@ func migrateV7ToV8() error {
 		srvConfig.Notify.Redis = cv7.Notify.Redis
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv7.Version, srvConfig.Version, err)
 	}
 
@@ -560,7 +566,7 @@ func migrateV8ToV9() error {
 	configFile := getConfigFile()
 
 	cv8 := &serverConfigV8{}
-	_, err := quick.Load(configFile, cv8)
+	_, err := Load(configFile, cv8)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -616,7 +622,7 @@ func migrateV8ToV9() error {
 		srvConfig.Notify.PostgreSQL = cv8.Notify.PostgreSQL
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv8.Version, srvConfig.Version, err)
 	}
 
@@ -630,7 +636,7 @@ func migrateV9ToV10() error {
 	configFile := getConfigFile()
 
 	cv9 := &serverConfigV9{}
-	_, err := quick.Load(configFile, cv9)
+	_, err := Load(configFile, cv9)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -684,7 +690,7 @@ func migrateV9ToV10() error {
 		srvConfig.Notify.PostgreSQL = cv9.Notify.PostgreSQL
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv9.Version, srvConfig.Version, err)
 	}
 
@@ -698,7 +704,7 @@ func migrateV10ToV11() error {
 	configFile := getConfigFile()
 
 	cv10 := &serverConfigV10{}
-	_, err := quick.Load(configFile, cv10)
+	_, err := Load(configFile, cv10)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -755,7 +761,7 @@ func migrateV10ToV11() error {
 	srvConfig.Notify.Kafka = make(map[string]target.KafkaArgs)
 	srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv10.Version, srvConfig.Version, err)
 	}
 
@@ -769,7 +775,7 @@ func migrateV11ToV12() error {
 	configFile := getConfigFile()
 
 	cv11 := &serverConfigV11{}
-	_, err := quick.Load(configFile, cv11)
+	_, err := Load(configFile, cv11)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -853,7 +859,7 @@ func migrateV11ToV12() error {
 		}
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv11.Version, srvConfig.Version, err)
 	}
 
@@ -866,7 +872,7 @@ func migrateV12ToV13() error {
 	configFile := getConfigFile()
 
 	cv12 := &serverConfigV12{}
-	_, err := quick.Load(configFile, cv12)
+	_, err := Load(configFile, cv12)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -933,7 +939,7 @@ func migrateV12ToV13() error {
 	srvConfig.Notify.Webhook = make(map[string]target.WebhookArgs)
 	srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv12.Version, srvConfig.Version, err)
 	}
 
@@ -946,7 +952,7 @@ func migrateV13ToV14() error {
 	configFile := getConfigFile()
 
 	cv13 := &serverConfigV13{}
-	_, err := quick.Load(configFile, cv13)
+	_, err := Load(configFile, cv13)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1018,7 +1024,7 @@ func migrateV13ToV14() error {
 	// Set the new browser parameter to true by default
 	srvConfig.Browser = true
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv13.Version, srvConfig.Version, err)
 	}
 
@@ -1031,7 +1037,7 @@ func migrateV14ToV15() error {
 	configFile := getConfigFile()
 
 	cv14 := &serverConfigV14{}
-	_, err := quick.Load(configFile, cv14)
+	_, err := Load(configFile, cv14)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1107,7 +1113,7 @@ func migrateV14ToV15() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv14.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv14.Version, srvConfig.Version, err)
 	}
 
@@ -1121,7 +1127,7 @@ func migrateV15ToV16() error {
 	configFile := getConfigFile()
 
 	cv15 := &serverConfigV15{}
-	_, err := quick.Load(configFile, cv15)
+	_, err := Load(configFile, cv15)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1197,7 +1203,7 @@ func migrateV15ToV16() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv15.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv15.Version, srvConfig.Version, err)
 	}
 
@@ -1211,7 +1217,7 @@ func migrateV16ToV17() error {
 	configFile := getConfigFile()
 
 	cv16 := &serverConfigV16{}
-	_, err := quick.Load(configFile, cv16)
+	_, err := Load(configFile, cv16)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1318,7 +1324,7 @@ func migrateV16ToV17() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv16.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv16.Version, srvConfig.Version, err)
 	}
 
@@ -1332,7 +1338,7 @@ func migrateV17ToV18() error {
 	configFile := getConfigFile()
 
 	cv17 := &serverConfigV17{}
-	_, err := quick.Load(configFile, cv17)
+	_, err := Load(configFile, cv17)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1422,7 +1428,7 @@ func migrateV17ToV18() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv17.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv17.Version, srvConfig.Version, err)
 	}
 
@@ -1434,7 +1440,7 @@ func migrateV18ToV19() error {
 	configFile := getConfigFile()
 
 	cv18 := &serverConfigV18{}
-	_, err := quick.Load(configFile, cv18)
+	_, err := Load(configFile, cv18)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1528,7 +1534,7 @@ func migrateV18ToV19() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv18.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv18.Version, srvConfig.Version, err)
 	}
 
@@ -1540,7 +1546,7 @@ func migrateV19ToV20() error {
 	configFile := getConfigFile()
 
 	cv19 := &serverConfigV19{}
-	_, err := quick.Load(configFile, cv19)
+	_, err := Load(configFile, cv19)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1633,7 +1639,7 @@ func migrateV19ToV20() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv19.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv19.Version, srvConfig.Version, err)
 	}
 
@@ -1645,7 +1651,7 @@ func migrateV20ToV21() error {
 	configFile := getConfigFile()
 
 	cv20 := &serverConfigV20{}
-	_, err := quick.Load(configFile, cv20)
+	_, err := Load(configFile, cv20)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1737,7 +1743,7 @@ func migrateV20ToV21() error {
 	// Load domain config from existing config in the file.
 	srvConfig.Domain = cv20.Domain
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv20.Version, srvConfig.Version, err)
 	}
 
@@ -1749,7 +1755,7 @@ func migrateV21ToV22() error {
 	configFile := getConfigFile()
 
 	cv21 := &serverConfigV21{}
-	_, err := quick.Load(configFile, cv21)
+	_, err := Load(configFile, cv21)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1841,7 +1847,7 @@ func migrateV21ToV22() error {
 	// Load domain config from existing config in the file.
 	srvConfig.Domain = cv21.Domain
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv21.Version, srvConfig.Version, err)
 	}
 
@@ -1853,7 +1859,7 @@ func migrateV22ToV23() error {
 	configFile := getConfigFile()
 
 	cv22 := &serverConfigV22{}
-	_, err := quick.Load(configFile, cv22)
+	_, err := Load(configFile, cv22)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1954,7 +1960,7 @@ func migrateV22ToV23() error {
 	srvConfig.Cache.Exclude = []string{}
 	srvConfig.Cache.Expiry = globalCacheExpiry
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv22.Version, srvConfig.Version, err)
 	}
 
@@ -1966,7 +1972,7 @@ func migrateV23ToV24() error {
 	configFile := getConfigFile()
 
 	cv23 := &serverConfigV23{}
-	_, err := quick.Load(configFile, cv23)
+	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv23)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -2067,7 +2073,7 @@ func migrateV23ToV24() error {
 	srvConfig.Cache.Exclude = cv23.Cache.Exclude
 	srvConfig.Cache.Expiry = cv23.Cache.Expiry
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv23.Version, srvConfig.Version, err)
 	}
 
@@ -2079,7 +2085,7 @@ func migrateV24ToV25() error {
 	configFile := getConfigFile()
 
 	cv24 := &serverConfigV24{}
-	_, err := quick.Load(configFile, cv24)
+	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv24)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -2185,10 +2191,129 @@ func migrateV24ToV25() error {
 	srvConfig.Cache.Exclude = cv24.Cache.Exclude
 	srvConfig.Cache.Expiry = cv24.Cache.Expiry
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv24.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv24.Version, srvConfig.Version)
 	return nil
 }
+
+func migrateV25ToV26() error {
+	configFile := getConfigFile()
+
+	cv25 := &serverConfigV25{}
+	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv25)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config version ‘25’. %v", err)
+	}
+	if cv25.Version != "25" {
+		return nil
+	}
+
+	// Copy over fields from V25 into V26 config struct
+	srvConfig := &serverConfigV26{
+		Notify: notifier{},
+	}
+	srvConfig.Version = "26"
+	srvConfig.Credential = cv25.Credential
+	srvConfig.Region = cv25.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = globalMinioDefaultRegion
+	}
+
+	if len(cv25.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]target.AMQPArgs)
+		srvConfig.Notify.AMQP["1"] = target.AMQPArgs{}
+	} else {
+		srvConfig.Notify.AMQP = cv25.Notify.AMQP
+	}
+	if len(cv25.Notify.Elasticsearch) == 0 {
+		srvConfig.Notify.Elasticsearch = make(map[string]target.ElasticsearchArgs)
+		srvConfig.Notify.Elasticsearch["1"] = target.ElasticsearchArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Elasticsearch = cv25.Notify.Elasticsearch
+	}
+	if len(cv25.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]target.RedisArgs)
+		srvConfig.Notify.Redis["1"] = target.RedisArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Redis = cv25.Notify.Redis
+	}
+	if len(cv25.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
+		srvConfig.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv25.Notify.PostgreSQL
+	}
+	if len(cv25.Notify.Kafka) == 0 {
+		srvConfig.Notify.Kafka = make(map[string]target.KafkaArgs)
+		srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
+	} else {
+		srvConfig.Notify.Kafka = cv25.Notify.Kafka
+	}
+	if len(cv25.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]target.NATSArgs)
+		srvConfig.Notify.NATS["1"] = target.NATSArgs{}
+	} else {
+		srvConfig.Notify.NATS = cv25.Notify.NATS
+	}
+	if len(cv25.Notify.Webhook) == 0 {
+		srvConfig.Notify.Webhook = make(map[string]target.WebhookArgs)
+		srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
+	} else {
+		srvConfig.Notify.Webhook = cv25.Notify.Webhook
+	}
+	if len(cv25.Notify.MySQL) == 0 {
+		srvConfig.Notify.MySQL = make(map[string]target.MySQLArgs)
+		srvConfig.Notify.MySQL["1"] = target.MySQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.MySQL = cv25.Notify.MySQL
+	}
+
+	if len(cv25.Notify.MQTT) == 0 {
+		srvConfig.Notify.MQTT = make(map[string]target.MQTTArgs)
+		srvConfig.Notify.MQTT["1"] = target.MQTTArgs{}
+	} else {
+		srvConfig.Notify.MQTT = cv25.Notify.MQTT
+	}
+
+	// Load browser config from existing config in the file.
+	srvConfig.Browser = cv25.Browser
+
+	// Load worm config from existing config in the file.
+	srvConfig.Worm = cv25.Worm
+
+	// Load domain config from existing config in the file.
+	srvConfig.Domain = cv25.Domain
+
+	// Load storage class config from existing storage class config in the file.
+	srvConfig.StorageClass.RRS = cv25.StorageClass.RRS
+	srvConfig.StorageClass.Standard = cv25.StorageClass.Standard
+
+	// Load cache config from existing cache config in the file.
+	srvConfig.Cache.Drives = cv25.Cache.Drives
+	srvConfig.Cache.Exclude = cv25.Cache.Exclude
+	srvConfig.Cache.Expiry = cv25.Cache.Expiry
+
+	// Add predefined value to new server config.
+	srvConfig.Cache.MaxUse = globalCacheMaxUse
+
+	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv25.Version, srvConfig.Version, err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, cv25.Version, srvConfig.Version)
+	return nil
+}

cmd/config-versions.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/quick"
 )
 
 /////////////////// Config V1 ///////////////////
@@ -633,6 +634,35 @@ type serverConfigV24 struct {
 // IMPORTANT NOTE: When updating this struct make sure that
 // serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV25 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BoolFlag         `json:"browser"`
+	Worm       BoolFlag         `json:"worm"`
+	Domain     string           `json:"domain"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+}
+
+// serverConfigV26 is just like version '25', stores additionally
+// cache max use value in 'CacheConfig'.
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
+type serverConfigV26 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
 	Version string `json:"version"`
 
 	// S3 API configuration.

cmd/crypto/doc.go

@@ -0,0 +1,116 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package crypto implements AWS S3 related cryptographic building blocks
+// for implementing Server-Side-Encryption (SSE-S3) and Server-Side-Encryption
+// with customer provided keys (SSE-C).
+//
+// All objects are encrypted with an unique and randomly generated 'ObjectKey'.
+// The ObjectKey itself is never stored in plaintext. Instead it is only stored
+// in a sealed from. The sealed 'ObjectKey' is created by encrypting the 'ObjectKey'
+// with an unique key-encryption-key. Given the correct key-encryption-key the
+// sealed 'ObjectKey' can be unsealed and the object can be decrypted.
+//
+//
+// ## SSE-C
+//
+// SSE-C computes the key-encryption-key from the client-provided key, an
+// initialization vector (IV) and the bucket/object path.
+//
+// 1. Encrypt:
+//       Input: ClientKey, bucket, object, metadata, object_data
+//       -              IV := Random({0,1}²⁵⁶)
+//       -       ObjectKey := SHA256(ClientKey || Random({0,1}²⁵⁶))
+//       -       KeyEncKey := HMAC-SHA256(ClientKey, IV || 'SSE-C' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
+//       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
+//       -        metadata <- IV
+//       -        metadata <- SealedKey
+//       Output: enc_object_data, metadata
+//
+// 2. Decrypt:
+//       Input: ClientKey, bucket, object, metadata, enc_object_data
+//       -          IV <- metadata
+//       -   SealedKey <- metadata
+//       -   KeyEncKey := HMAC-SHA256(ClientKey, IV || 'SSE-C' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
+//       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
+//       Output: object_data
+//
+//
+// ## SSE-S3
+//
+// SSE-S3 can use either a master key or a KMS as root-of-trust.
+// The en/decryption slightly depens upon which root-of-trust is used.
+//
+// ### SSE-S3 and single master key
+//
+// The master key is used to derive unique object- and key-encryption-keys.
+// SSE-S3 with a single master key works as SSE-C where the master key is
+// used as the client-provided key.
+//
+// 1. Encrypt:
+//       Input: MasterKey, bucket, object, metadata, object_data
+//       -              IV := Random({0,1}²⁵⁶)
+//       -       ObjectKey := SHA256(MasterKey || Random({0,1}²⁵⁶))
+//       -       KeyEncKey := HMAC-SHA256(MasterKey, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
+//       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
+//       -        metadata <- IV
+//       -        metadata <- SealedKey
+//       Output: enc_object_data, metadata
+//
+// 2. Decrypt:
+//       Input: MasterKey, bucket, object, metadata, enc_object_data
+//       -          IV <- metadata
+//       -   SealedKey <- metadata
+//       -   KeyEncKey := HMAC-SHA256(MasterKey, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
+//       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
+//       Output: object_data
+//
+//
+// ### SSE-S3 and KMS
+//
+// SSE-S3 requires that the KMS provides two functions:
+//       1.       Generate(KeyID) -> (Key, EncKey)
+//       2. Unseal(KeyID, EncKey) -> Key
+//
+// 1. Encrypt:
+//       Input: KeyID, bucket, object, metadata, object_data
+//       -     Key, EncKey := Generate(KeyID)
+//       -              IV := Random({0,1}²⁵⁶)
+//       -       ObjectKey := SHA256(Key, Random({0,1}²⁵⁶))
+//       -       KeyEncKey := HMAC-SHA256(Key, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
+//       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
+//       -        metadata <- IV
+//       -        metadata <- KeyID
+//       -        metadata <- EncKey
+//       -        metadata <- SealedKey
+//       Output: enc_object_data, metadata
+//
+// 2. Decrypt:
+//       Input: bucket, object, metadata, enc_object_data
+//       -      KeyID  <- metadata
+//       -      EncKey <- metadata
+//       -          IV <- metadata
+//       -   SealedKey <- metadata
+//       -         Key := Unseal(KeyID, EncKey)
+//       -   KeyEncKey := HMAC-SHA256(Key, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object)
+//       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
+//       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
+//       Output: object_data
+//
+package crypto

cmd/crypto/error.go

@@ -0,0 +1,23 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import "errors"
+
+var (
+	// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
+	// is not supported.
+	ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
+)

cmd/crypto/header.go

@@ -0,0 +1,49 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"net/http"
+)
+
+// SSEHeader is the general AWS SSE HTTP header key.
+const SSEHeader = "X-Amz-Server-Side-Encryption"
+
+// SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
+// For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
+// For SSE-C  see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
+const SSEAlgorithmAES256 = "AES256"
+
+// S3 represents AWS SSE-S3. It provides functionality to handle
+// SSE-S3 requests.
+var S3 = s3{}
+
+type s3 struct{}
+
+// IsRequested returns true if the HTTP headers indicates that
+// the S3 client requests SSE-S3.
+func (s3) IsRequested(h http.Header) bool {
+	_, ok := h[SSEHeader]
+	return ok
+}
+
+// Parse parses the SSE-S3 related HTTP headers and checks
+// whether they contain valid values.
+func (s3) Parse(h http.Header) (err error) {
+	if h.Get(SSEHeader) != SSEAlgorithmAES256 {
+		err = ErrInvalidEncryptionMethod
+	}
+	return
+}

cmd/crypto/header_test.go

@@ -0,0 +1,56 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"net/http"
+	"testing"
+)
+
+var isRequestedTests = []struct {
+	Header   http.Header
+	Expected bool
+}{
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true},  // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true}, // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true},        // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false},  // 3
+}
+
+func TestS3IsRequested(t *testing.T) {
+	for i, test := range isRequestedTests {
+		if got := S3.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+}
+
+var parseTests = []struct {
+	Header      http.Header
+	ExpectedErr error
+}{
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, ExpectedErr: nil},                         // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, ExpectedErr: ErrInvalidEncryptionMethod},        // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, ExpectedErr: ErrInvalidEncryptionMethod},   // 3
+}
+
+func TestS3Parse(t *testing.T) {
+	for i, test := range parseTests {
+		if err := S3.Parse(test.Header); err != test.ExpectedErr {
+			t.Errorf("Test %d: Wanted '%v' but got '%v'", i, test.ExpectedErr, err)
+		}
+	}
+}

cmd/crypto/key.go

@@ -0,0 +1,93 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/rand"
+	"encoding/binary"
+	"errors"
+	"io"
+	"path/filepath"
+
+	"github.com/minio/minio/cmd/logger"
+	sha256 "github.com/minio/sha256-simd"
+	"github.com/minio/sio"
+)
+
+// ObjectKey is a 256 bit secret key used to encrypt the object.
+// It must never be stored in plaintext.
+type ObjectKey [32]byte
+
+// GenerateKey generates a unique ObjectKey from a 256 bit external key
+// and a source of randomness. If random is nil the default PRNG of system
+// (crypto/rand) is used.
+func GenerateKey(extKey [32]byte, random io.Reader) (key ObjectKey) {
+	if random == nil {
+		random = rand.Reader
+	}
+	var nonce [32]byte
+	if _, err := io.ReadFull(random, nonce[:]); err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to read enough randomness from the system"))
+	}
+	sha := sha256.New()
+	sha.Write(extKey[:])
+	sha.Write(nonce[:])
+	sha.Sum(key[:0])
+	return
+}
+
+// Seal encrypts the ObjectKey using the 256 bit external key and IV. The sealed
+// key is also cryptographically bound to the object's path (bucket/object).
+func (key ObjectKey) Seal(extKey, iv [32]byte, bucket, object string) []byte {
+	var sealedKey bytes.Buffer
+	mac := hmac.New(sha256.New, extKey[:])
+	mac.Write(iv[:])
+	mac.Write([]byte(filepath.Join(bucket, object)))
+
+	if n, err := sio.Encrypt(&sealedKey, bytes.NewReader(key[:]), sio.Config{Key: mac.Sum(nil)}); n != 64 || err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to generate sealed key"))
+	}
+	return sealedKey.Bytes()
+}
+
+// Unseal decrypts a sealed key using the 256 bit external key and IV. Since the sealed key
+// is cryptographically bound to the object's path the same bucket/object as during sealing
+// must be provided. On success the ObjectKey contains the decrypted sealed key.
+func (key *ObjectKey) Unseal(sealedKey []byte, extKey, iv [32]byte, bucket, object string) error {
+	var unsealedKey bytes.Buffer
+	mac := hmac.New(sha256.New, extKey[:])
+	mac.Write(iv[:])
+	mac.Write([]byte(filepath.Join(bucket, object)))
+
+	if n, err := sio.Decrypt(&unsealedKey, bytes.NewReader(sealedKey), sio.Config{Key: mac.Sum(nil)}); n != 32 || err != nil {
+		return err // TODO(aead): upgrade sio to use sio.Error
+	}
+	copy(key[:], unsealedKey.Bytes())
+	return nil
+}
+
+// DerivePartKey derives an unique 256 bit key from an ObjectKey and the part index.
+func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
+	var bin [4]byte
+	binary.LittleEndian.PutUint32(bin[:], id)
+
+	mac := hmac.New(sha256.New, key[:])
+	mac.Write(bin[:])
+	mac.Sum(partKey[:0])
+	return
+}

cmd/crypto/key_test.go

@@ -0,0 +1,127 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"io"
+	"testing"
+)
+
+var shortRandom = func(limit int64) io.Reader { return io.LimitReader(rand.Reader, limit) }
+
+func recoverTest(i int, shouldPass bool, t *testing.T) {
+	if err := recover(); err == nil && !shouldPass {
+		t.Errorf("Test %d should fail but passed successfully", i)
+	} else if err != nil && shouldPass {
+		t.Errorf("Test %d should pass but failed: %v", i, err)
+	}
+}
+
+var generateKeyTests = []struct {
+	ExtKey     [32]byte
+	Random     io.Reader
+	ShouldPass bool
+}{
+	{ExtKey: [32]byte{}, Random: nil, ShouldPass: true},             // 0
+	{ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true},     // 1
+	{ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true}, // 2
+	// {ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3 See: https://github.com/minio/minio/issues/6064
+}
+
+func TestGenerateKey(t *testing.T) {
+	for i, test := range generateKeyTests {
+		func() {
+			defer recoverTest(i, test.ShouldPass, t)
+			key := GenerateKey(test.ExtKey, test.Random)
+			if [32]byte(key) == [32]byte{} {
+				t.Errorf("Test %d: generated key is zero key", i) // check that we generate random and unique key
+			}
+		}()
+	}
+}
+
+var sealUnsealKeyTests = []struct {
+	SealExtKey, SealIV     [32]byte
+	SealBucket, SealObject string
+
+	UnsealExtKey, UnsealIV     [32]byte
+	UnsealBucket, UnsealObject string
+
+	ShouldPass bool
+}{
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "object",
+		ShouldPass: true,
+	}, // 0
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{1}, UnsealIV: [32]byte{0}, UnsealBucket: "bucket", UnsealObject: "object",
+		ShouldPass: false,
+	}, // 1
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{1}, UnsealBucket: "bucket", UnsealObject: "object",
+		ShouldPass: false,
+	}, // 2
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "Bucket", UnsealObject: "object",
+		ShouldPass: false,
+	}, // 3
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "Object",
+		ShouldPass: false,
+	}, // 4
+}
+
+func TestSealUnsealKey(t *testing.T) {
+	for i, test := range sealUnsealKeyTests {
+		key := GenerateKey(test.SealExtKey, rand.Reader)
+		sealedKey := key.Seal(test.SealExtKey, test.SealIV, test.SealBucket, test.SealObject)
+		if err := key.Unseal(sealedKey, test.UnsealExtKey, test.UnsealIV, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass {
+			t.Errorf("Test %d should fail but passed successfully", i)
+		} else if err != nil && test.ShouldPass {
+			t.Errorf("Test %d should pass put failed: %v", i, err)
+		}
+	}
+}
+
+var derivePartKeyTest = []struct {
+	PartID  uint32
+	PartKey string
+}{
+	{PartID: 0, PartKey: "aa7855e13839dd767cd5da7c1ff5036540c9264b7a803029315e55375287b4af"},
+	{PartID: 1, PartKey: "a3e7181c6eed030fd52f79537c56c4d07da92e56d374ff1dd2043350785b37d8"},
+	{PartID: 10000, PartKey: "f86e65c396ed52d204ee44bd1a0bbd86eb8b01b7354e67a3b3ae0e34dd5bd115"},
+}
+
+func TestDerivePartKey(t *testing.T) {
+	var key ObjectKey
+	for i, test := range derivePartKeyTest {
+		expectedPartKey, err := hex.DecodeString(test.PartKey)
+		if err != nil {
+			t.Fatalf("Test %d failed to decode expected part-key: %v", i, err)
+		}
+		partKey := key.DerivePartKey(test.PartID)
+		if !bytes.Equal(partKey[:], expectedPartKey[:]) {
+			t.Errorf("Test %d derives wrong part-key: got '%s' want: '%s'", i, hex.EncodeToString(partKey[:]), test.PartKey)
+		}
+	}
+}

cmd/crypto/sse.go

@@ -0,0 +1,60 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"context"
+	"errors"
+	"io"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/ioutil"
+	"github.com/minio/sio"
+)
+
+const (
+	// S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3.
+	S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
+	// S3KMSKeyID is the metadata key referencing the KMS key-id used to
+	// generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS.
+	S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
+	// S3KMSSealedKey is the metadata key referencing the encrypted key generated
+	// by KMS. It is only used for SSE-S3 + KMS.
+	S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
+)
+
+// EncryptSinglePart encrypts an io.Reader which must be the
+// the body of a single-part PUT request.
+func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {
+	r, err := sio.EncryptReader(r, sio.Config{MinVersion: sio.Version20, Key: key[:]})
+	if err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to encrypt io.Reader using object key"))
+	}
+	return r
+}
+
+// DecryptSinglePart decrypts an io.Writer which must an object
+// uploaded with the single-part PUT API. The offset and length
+// specify the requested range.
+func DecryptSinglePart(w io.Writer, offset, length int64, key ObjectKey) io.WriteCloser {
+	const PayloadSize = 1 << 16 // DARE 2.0
+	w = ioutil.LimitedWriter(w, offset%PayloadSize, length)
+
+	decWriter, err := sio.DecryptWriter(w, sio.Config{Key: key[:]})
+	if err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to decrypt io.Writer using object key"))
+	}
+	return decWriter
+}

cmd/disk-cache-config.go

@@ -19,12 +19,16 @@ package cmd
 import (
 	"encoding/json"
 	"path/filepath"
+	"strings"
+
+	"github.com/minio/minio/pkg/ellipses"
 )
 
 // CacheConfig represents cache config settings
 type CacheConfig struct {
 	Drives  []string `json:"drives"`
 	Expiry  int      `json:"expiry"`
+	MaxUse  int      `json:"maxuse"`
 	Exclude []string `json:"exclude"`
 }
 
@@ -51,12 +55,42 @@ func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {
 
 // Parses given cacheDrivesEnv and returns a list of cache drives.
 func parseCacheDrives(drives []string) ([]string, error) {
+	if len(drives) == 0 {
+		return drives, nil
+	}
+	var endpoints []string
 	for _, d := range drives {
+		if ellipses.HasEllipses(d) {
+			s, err := parseCacheDrivePaths(d)
+			if err != nil {
+				return nil, err
+			}
+			endpoints = append(endpoints, s...)
+		} else {
+			endpoints = append(endpoints, d)
+		}
+	}
+
+	for _, d := range endpoints {
 		if !filepath.IsAbs(d) {
 			return nil, uiErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
 		}
 	}
-	return drives, nil
+	return endpoints, nil
+}
+
+// Parses all arguments and returns a slice of drive paths following the ellipses pattern.
+func parseCacheDrivePaths(arg string) (ep []string, err error) {
+	patterns, perr := ellipses.FindEllipsesPatterns(arg)
+	if perr != nil {
+		return []string{}, uiErrInvalidCacheDrivesValue(nil).Msg(perr.Error())
+	}
+
+	for _, lbls := range patterns.Expand() {
+		ep = append(ep, strings.Join(lbls, ""))
+	}
+
+	return ep, nil
 }
 
 // Parses given cacheExcludesEnv and returns a list of cache exclude patterns.

cmd/disk-cache-config_test.go

@@ -41,12 +41,32 @@ func TestParseCacheDrives(t *testing.T) {
 			expectedPatterns []string
 			success          bool
 		}{"C:/home/drive1;C:/home/drive2;C:/home/drive3", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"C:/home/drive{1...3}", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"C:/home/drive{1..3}", []string{}, false})
 	} else {
 		testCases = append(testCases, struct {
 			driveStr         string
 			expectedPatterns []string
 			success          bool
 		}{"/home/drive1;/home/drive2;/home/drive3", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"/home/drive{1...3}", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"/home/drive{1..3}", []string{}, false})
 	}
 	for i, testCase := range testCases {
 		drives, err := parseCacheDrives(strings.Split(testCase.driveStr, cacheEnvDelimiter))

cmd/disk-cache.go

@@ -44,7 +44,6 @@ const (
 	// disk cache needs to have cacheSizeMultiplier * object size space free for a cache entry to be created.
 	cacheSizeMultiplier  = 100
 	cacheTrashDir        = "trash"
-	cacheMaxDiskUsagePct = 80 // in %
 	cacheCleanupInterval = 10 // in minutes
 )
 
@@ -239,7 +238,7 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 			pipeWriter.CloseWithError(err)
 			return
 		}
-		pipeWriter.Close() // Close writer explicitly signalling we wrote all data.
+		pipeWriter.Close() // Close writer explicitly signaling we wrote all data.
 	}()
 	err = dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(objInfo))
 	if err != nil {
@@ -839,7 +838,7 @@ func newCache(config CacheConfig) (*diskCache, error) {
 		if err := checkAtimeSupport(dir); err != nil {
 			return nil, errors.New("Atime support required for disk caching")
 		}
-		cache, err := newCacheFSObjects(dir, config.Expiry, cacheMaxDiskUsagePct)
+		cache, err := newCacheFSObjects(dir, config.Expiry, config.MaxUse)
 		if err != nil {
 			return nil, err
 		}

cmd/disk-cache_test.go

@@ -28,10 +28,10 @@ import (
 )
 
 // Initialize cache FS objects.
-func initCacheFSObjects(disk string, t *testing.T) (*cacheFSObjects, error) {
+func initCacheFSObjects(disk string, cacheMaxUse int, t *testing.T) (*cacheFSObjects, error) {
 	newTestConfig(globalMinioDefaultRegion)
 	var err error
-	obj, err := newCacheFSObjects(disk, globalCacheExpiry, 100)
+	obj, err := newCacheFSObjects(disk, globalCacheExpiry, cacheMaxUse)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -39,10 +39,10 @@ func initCacheFSObjects(disk string, t *testing.T) (*cacheFSObjects, error) {
 }
 
 // inits diskCache struct for nDisks
-func initDiskCaches(drives []string, t *testing.T) (*diskCache, error) {
+func initDiskCaches(drives []string, cacheMaxUse int, t *testing.T) (*diskCache, error) {
 	var cfs []*cacheFSObjects
 	for _, d := range drives {
-		obj, err := initCacheFSObjects(d, t)
+		obj, err := initCacheFSObjects(d, cacheMaxUse, t)
 		if err != nil {
 			return nil, err
 		}
@@ -59,7 +59,46 @@ func TestGetCacheFS(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		d, err := initDiskCaches(fsDirs, t)
+		d, err := initDiskCaches(fsDirs, 100, t)
+		if err != nil {
+			t.Fatal(err)
+		}
+		bucketName := "testbucket"
+		objectName := "testobject"
+		ctx := context.Background()
+		// find cache drive where object would be hashed
+		index := d.hashIndex(bucketName, objectName)
+		// turn off drive by setting online status to false
+		d.cfs[index].online = false
+		cfs, err := d.getCacheFS(ctx, bucketName, objectName)
+		if n == 1 && err == errDiskNotFound {
+			continue
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		i := -1
+		for j, f := range d.cfs {
+			if f == cfs {
+				i = j
+				break
+			}
+		}
+		if i != (index+1)%n {
+			t.Fatalf("expected next cache location to be picked")
+		}
+	}
+}
+
+// test whether a drive being offline causes
+// getCacheFS to fetch next online drive
+func TestGetCacheFSMaxUse(t *testing.T) {
+	for n := 1; n < 10; n++ {
+		fsDirs, err := getRandomDisks(n)
+		if err != nil {
+			t.Fatal(err)
+		}
+		d, err := initDiskCaches(fsDirs, globalCacheMaxUse, t)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -141,7 +180,7 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	d, err := initDiskCaches(fsDirs, t)
+	d, err := initDiskCaches(fsDirs, 100, t)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -209,6 +248,87 @@ func TestDiskCache(t *testing.T) {
 	}
 }
 
+// Test diskCache with upper bound on max cache use.
+func TestDiskCacheMaxUse(t *testing.T) {
+	fsDirs, err := getRandomDisks(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	d, err := initDiskCaches(fsDirs, globalCacheMaxUse, t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cache := d.cfs[0]
+	ctx := context.Background()
+	bucketName := "testbucket"
+	objectName := "testobject"
+	content := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	etag := "061208c10af71a30c6dcd6cf5d89f0fe"
+	contentType := "application/zip"
+	size := len(content)
+
+	httpMeta := make(map[string]string)
+	httpMeta["etag"] = etag
+	httpMeta["content-type"] = contentType
+
+	objInfo := ObjectInfo{}
+	objInfo.Bucket = bucketName
+	objInfo.Name = objectName
+	objInfo.Size = int64(size)
+	objInfo.ContentType = contentType
+	objInfo.ETag = etag
+	objInfo.UserDefined = httpMeta
+
+	byteReader := bytes.NewReader([]byte(content))
+	hashReader, err := hash.NewReader(byteReader, int64(size), "", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !cache.diskAvailable(int64(size)) {
+		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
+		if err != errDiskFull {
+			t.Fatal("Cache max-use limit violated.")
+		}
+	} else {
+		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
+		if err != nil {
+			t.Fatal(err)
+		}
+		cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !cache.Exists(ctx, bucketName, objectName) {
+			t.Fatal("Expected object to exist on cache")
+		}
+		if cachedObjInfo.ETag != objInfo.ETag {
+			t.Fatal("Expected ETag to match")
+		}
+		if cachedObjInfo.Size != objInfo.Size {
+			t.Fatal("Size mismatch")
+		}
+		if cachedObjInfo.ContentType != objInfo.ContentType {
+			t.Fatal("Cached content-type does not match")
+		}
+		writer := bytes.NewBuffer(nil)
+		err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if ccontent := writer.Bytes(); !bytes.Equal([]byte(content), ccontent) {
+			t.Errorf("wrong cached file content")
+		}
+		err = cache.Delete(ctx, bucketName, objectName)
+		if err != nil {
+			t.Errorf("object missing from cache")
+		}
+		online := cache.IsOnline()
+		if !online {
+			t.Errorf("expected cache drive to be online")
+		}
+	}
+}
+
 func TestIsCacheExcludeDirective(t *testing.T) {
 	testCases := []struct {
 		cacheControlOpt string

cmd/encryption-v1.go

@@ -28,6 +28,7 @@ import (
 	"errors"
 	"io"
 	"net/http"
+	"path"
 	"strconv"
 
 	"github.com/minio/minio/cmd/logger"
@@ -144,9 +145,20 @@ const (
 	ServerSideEncryptionSealedKey = ReservedMetadataPrefix + "Server-Side-Encryption-Sealed-Key"
 )
 
-// SSESealAlgorithmDareSha256 specifies DARE as authenticated en/decryption scheme and SHA256 as cryptographic
-// hash function.
-const SSESealAlgorithmDareSha256 = "DARE-SHA256"
+const (
+	// SSESealAlgorithmDareSha256 specifies DARE as authenticated en/decryption scheme and SHA256 as cryptographic
+	// hash function. The key derivation of DARE-SHA256 is not optimal and does not include the object path.
+	// It is considered legacy and should not be used anymore.
+	SSESealAlgorithmDareSha256 = "DARE-SHA256"
+
+	// SSESealAlgorithmDareV2HmacSha256 specifies DAREv2 as authenticated en/decryption scheme and SHA256 as cryptographic
+	// hash function for the HMAC PRF.
+	SSESealAlgorithmDareV2HmacSha256 = "DAREv2-HMAC-SHA256"
+
+	// SSEDomain specifies the domain for the derived key - in this case the
+	// key should be used for SSE-C.
+	SSEDomain = "SSE-C"
+)
 
 // hasSSECustomerHeader returns true if the given HTTP header
 // contains server-side-encryption with customer provided key fields.
@@ -250,10 +262,11 @@ func ParseSSECustomerHeader(header http.Header) (key []byte, err error) {
 }
 
 // This function rotates old to new key.
-func rotateKey(oldKey []byte, newKey []byte, metadata map[string]string) error {
+func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map[string]string) error {
 	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
 
-	if metadata[ServerSideEncryptionSealAlgorithm] != SSESealAlgorithmDareSha256 { // currently DARE-SHA256 is the only option
+	algorithm := metadata[ServerSideEncryptionSealAlgorithm]
+	if algorithm != SSESealAlgorithmDareSha256 && algorithm != SSESealAlgorithmDareV2HmacSha256 {
 		return errObjectTampered
 	}
 	iv, err := base64.StdEncoding.DecodeString(metadata[ServerSideEncryptionIV])
@@ -265,14 +278,33 @@ func rotateKey(oldKey []byte, newKey []byte, metadata map[string]string) error {
 		return errObjectTampered
 	}
 
-	sha := sha256.New() // derive key encryption key
-	sha.Write(oldKey)
-	sha.Write(iv)
-	keyEncryptionKey := sha.Sum(nil)
+	var (
+		minDAREVersion   byte
+		keyEncryptionKey [32]byte
+	)
+	switch algorithm {
+	default:
+		return errObjectTampered
+	case SSESealAlgorithmDareSha256: // legacy key-encryption-key derivation
+		minDAREVersion = sio.Version10
+		sha := sha256.New()
+		sha.Write(oldKey)
+		sha.Write(iv)
+		sha.Sum(keyEncryptionKey[:0])
+	case SSESealAlgorithmDareV2HmacSha256: // key-encryption-key derivation - See: crypto/doc.go
+		minDAREVersion = sio.Version20
+		mac := hmac.New(sha256.New, oldKey)
+		mac.Write(iv)
+		mac.Write([]byte(SSEDomain))
+		mac.Write([]byte(SSESealAlgorithmDareV2HmacSha256))
+		mac.Write([]byte(path.Join(bucket, object)))
+		mac.Sum(keyEncryptionKey[:0])
+	}
 
 	objectEncryptionKey := bytes.NewBuffer(nil) // decrypt object encryption key
 	n, err := sio.Decrypt(objectEncryptionKey, bytes.NewReader(sealedKey), sio.Config{
-		Key: keyEncryptionKey,
+		MinVersion: minDAREVersion,
+		Key:        keyEncryptionKey[:],
 	})
 	if n != 32 || err != nil { // Either the provided key does not match or the object was tampered.
 		if subtle.ConstantTimeCompare(oldKey, newKey) == 1 {
@@ -280,46 +312,34 @@ func rotateKey(oldKey []byte, newKey []byte, metadata map[string]string) error {
 		}
 		return errSSEKeyMismatch // To provide strict AWS S3 compatibility we return: access denied.
 	}
-	if subtle.ConstantTimeCompare(oldKey, newKey) == 1 {
-		return nil // we don't need to rotate keys if newKey == oldKey
+	if subtle.ConstantTimeCompare(oldKey, newKey) == 1 && algorithm != SSESealAlgorithmDareSha256 {
+		return nil // we don't need to rotate keys if newKey == oldKey but we may have to upgrade KDF algorithm
 	}
 
-	nonce := make([]byte, 32) // generate random values for key derivation
-	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
-		return err
-	}
-
-	niv := sha256.Sum256(nonce[:]) // derive key encryption key
-	sha = sha256.New()
-	sha.Write(newKey)
-	sha.Write(niv[:])
-	keyEncryptionKey = sha.Sum(nil)
+	mac := hmac.New(sha256.New, newKey) // key-encryption-key derivation - See: crypto/doc.go
+	mac.Write(iv)
+	mac.Write([]byte(SSEDomain))
+	mac.Write([]byte(SSESealAlgorithmDareV2HmacSha256))
+	mac.Write([]byte(path.Join(bucket, object)))
+	mac.Sum(keyEncryptionKey[:0])
 
 	sealedKeyW := bytes.NewBuffer(nil) // sealedKey := 16 byte header + 32 byte payload + 16 byte tag
 	n, err = sio.Encrypt(sealedKeyW, bytes.NewReader(objectEncryptionKey.Bytes()), sio.Config{
-		Key: keyEncryptionKey,
+		Key: keyEncryptionKey[:],
 	})
 	if n != 64 || err != nil {
 		return errors.New("failed to seal object encryption key") // if this happens there's a bug in the code (may panic ?)
 	}
 
-	metadata[ServerSideEncryptionIV] = base64.StdEncoding.EncodeToString(niv[:])
-	metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareSha256
+	metadata[ServerSideEncryptionIV] = base64.StdEncoding.EncodeToString(iv[:])
+	metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareV2HmacSha256
 	metadata[ServerSideEncryptionSealedKey] = base64.StdEncoding.EncodeToString(sealedKeyW.Bytes())
 	return nil
 }
 
-func newEncryptMetadata(key []byte, metadata map[string]string) ([]byte, error) {
+func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) {
 	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
-
-	// security notice:
-	//  - If the first 32 bytes of the random value are ever repeated under the same client-provided
-	//    key the encrypted object will not be tamper-proof. [ P(coll) ~= 1 / 2^(256 / 2)]
-	//  - If the last 32 bytes of the random value are ever repeated under the same client-provided
-	//    key an adversary may be able to extract the object encryption key. This depends on the
-	//    authenticated en/decryption scheme. The DARE format will generate an 8 byte nonce which must
-	//    be repeated in addition to reveal the object encryption key.
-	//    [ P(coll) ~= 1 / 2^((256 + 64) / 2) ]
+	// See crypto/doc.go for detailed description
 	nonce := make([]byte, 32+SSEIVSize) // generate random values for key derivation
 	if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
 		return nil, err
@@ -329,11 +349,13 @@ func newEncryptMetadata(key []byte, metadata map[string]string) ([]byte, error)
 	sha.Write(nonce[:32])
 	objectEncryptionKey := sha.Sum(nil)
 
-	iv := sha256.Sum256(nonce[32:]) // derive key encryption key
-	sha = sha256.New()
-	sha.Write(key)
-	sha.Write(iv[:])
-	keyEncryptionKey := sha.Sum(nil)
+	iv := sha256.Sum256(nonce[32:]) // key-encryption-key derivation - See: crypto/doc.go
+	mac := hmac.New(sha256.New, key)
+	mac.Write(iv[:])
+	mac.Write([]byte(SSEDomain))
+	mac.Write([]byte(SSESealAlgorithmDareV2HmacSha256))
+	mac.Write([]byte(path.Join(bucket, object)))
+	keyEncryptionKey := mac.Sum(nil)
 
 	sealedKey := bytes.NewBuffer(nil) // sealedKey := 16 byte header + 32 byte payload + 16 byte tag
 	n, err := sio.Encrypt(sealedKey, bytes.NewReader(objectEncryptionKey), sio.Config{
@@ -344,14 +366,14 @@ func newEncryptMetadata(key []byte, metadata map[string]string) ([]byte, error)
 	}
 
 	metadata[ServerSideEncryptionIV] = base64.StdEncoding.EncodeToString(iv[:])
-	metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareSha256
+	metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareV2HmacSha256
 	metadata[ServerSideEncryptionSealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Bytes())
 
 	return objectEncryptionKey, nil
 }
 
-func newEncryptReader(content io.Reader, key []byte, metadata map[string]string) (io.Reader, error) {
-	objectEncryptionKey, err := newEncryptMetadata(key, metadata)
+func newEncryptReader(content io.Reader, key []byte, bucket, object string, metadata map[string]string) (io.Reader, error) {
+	objectEncryptionKey, err := newEncryptMetadata(key, bucket, object, metadata)
 	if err != nil {
 		return nil, err
 	}
@@ -367,29 +389,26 @@ func newEncryptReader(content io.Reader, key []byte, metadata map[string]string)
 // EncryptRequest takes the client provided content and encrypts the data
 // with the client provided key. It also marks the object as client-side-encrypted
 // and sets the correct headers.
-func EncryptRequest(content io.Reader, r *http.Request, metadata map[string]string) (io.Reader, error) {
+func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, metadata map[string]string) (io.Reader, error) {
 	key, err := ParseSSECustomerRequest(r)
 	if err != nil {
 		return nil, err
 	}
-	return newEncryptReader(content, key, metadata)
+	return newEncryptReader(content, key, bucket, object, metadata)
 }
 
 // DecryptCopyRequest decrypts the object with the client provided key. It also removes
 // the client-side-encryption metadata from the object and sets the correct headers.
-func DecryptCopyRequest(client io.Writer, r *http.Request, metadata map[string]string) (io.WriteCloser, error) {
+func DecryptCopyRequest(client io.Writer, r *http.Request, bucket, object string, metadata map[string]string) (io.WriteCloser, error) {
 	key, err := ParseSSECopyCustomerRequest(r)
 	if err != nil {
 		return nil, err
 	}
 	delete(metadata, SSECopyCustomerKey) // make sure we do not save the key by accident
-	return newDecryptWriter(client, key, 0, metadata)
+	return newDecryptWriter(client, key, bucket, object, 0, metadata)
 }
 
-func decryptObjectInfo(key []byte, metadata map[string]string) ([]byte, error) {
-	if metadata[ServerSideEncryptionSealAlgorithm] != SSESealAlgorithmDareSha256 { // currently DARE-SHA256 is the only option
-		return nil, errObjectTampered
-	}
+func decryptObjectInfo(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) {
 	iv, err := base64.StdEncoding.DecodeString(metadata[ServerSideEncryptionIV])
 	if err != nil || len(iv) != SSEIVSize {
 		return nil, errObjectTampered
@@ -399,14 +418,33 @@ func decryptObjectInfo(key []byte, metadata map[string]string) ([]byte, error) {
 		return nil, errObjectTampered
 	}
 
-	sha := sha256.New() // derive key encryption key
-	sha.Write(key)
-	sha.Write(iv)
-	keyEncryptionKey := sha.Sum(nil)
+	var (
+		minDAREVersion   byte
+		keyEncryptionKey [32]byte
+	)
+	switch algorithm := metadata[ServerSideEncryptionSealAlgorithm]; algorithm {
+	default:
+		return nil, errObjectTampered
+	case SSESealAlgorithmDareSha256: // legacy key-encryption-key derivation
+		minDAREVersion = sio.Version10
+		sha := sha256.New()
+		sha.Write(key)
+		sha.Write(iv)
+		sha.Sum(keyEncryptionKey[:0])
+	case SSESealAlgorithmDareV2HmacSha256: // key-encryption-key derivation - See: crypto/doc.go
+		minDAREVersion = sio.Version20
+		mac := hmac.New(sha256.New, key)
+		mac.Write(iv)
+		mac.Write([]byte(SSEDomain))
+		mac.Write([]byte(SSESealAlgorithmDareV2HmacSha256))
+		mac.Write([]byte(path.Join(bucket, object)))
+		mac.Sum(keyEncryptionKey[:0])
+	}
 
 	objectEncryptionKey := bytes.NewBuffer(nil) // decrypt object encryption key
 	n, err := sio.Decrypt(objectEncryptionKey, bytes.NewReader(sealedKey), sio.Config{
-		Key: keyEncryptionKey,
+		MinVersion: minDAREVersion,
+		Key:        keyEncryptionKey[:],
 	})
 	if n != 32 || err != nil {
 		// Either the provided key does not match or the object was tampered.
@@ -416,11 +454,10 @@ func decryptObjectInfo(key []byte, metadata map[string]string) ([]byte, error) {
 	return objectEncryptionKey.Bytes(), nil
 }
 
-func newDecryptWriter(client io.Writer, key []byte, seqNumber uint32, metadata map[string]string) (io.WriteCloser, error) {
-	objectEncryptionKey, err := decryptObjectInfo(key, metadata)
+func newDecryptWriter(client io.Writer, key []byte, bucket, object string, seqNumber uint32, metadata map[string]string) (io.WriteCloser, error) {
+	objectEncryptionKey, err := decryptObjectInfo(key, bucket, object, metadata)
 	if err != nil {
 		return nil, err
-
 	}
 	return newDecryptWriterWithObjectKey(client, objectEncryptionKey, seqNumber, metadata)
 }
@@ -443,19 +480,19 @@ func newDecryptWriterWithObjectKey(client io.Writer, objectEncryptionKey []byte,
 
 // DecryptRequestWithSequenceNumber decrypts the object with the client provided key. It also removes
 // the client-side-encryption metadata from the object and sets the correct headers.
-func DecryptRequestWithSequenceNumber(client io.Writer, r *http.Request, seqNumber uint32, metadata map[string]string) (io.WriteCloser, error) {
+func DecryptRequestWithSequenceNumber(client io.Writer, r *http.Request, bucket, object string, seqNumber uint32, metadata map[string]string) (io.WriteCloser, error) {
 	key, err := ParseSSECustomerRequest(r)
 	if err != nil {
 		return nil, err
 	}
 	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
-	return newDecryptWriter(client, key, seqNumber, metadata)
+	return newDecryptWriter(client, key, bucket, object, seqNumber, metadata)
 }
 
 // DecryptRequest decrypts the object with the client provided key. It also removes
 // the client-side-encryption metadata from the object and sets the correct headers.
-func DecryptRequest(client io.Writer, r *http.Request, metadata map[string]string) (io.WriteCloser, error) {
-	return DecryptRequestWithSequenceNumber(client, r, 0, metadata)
+func DecryptRequest(client io.Writer, r *http.Request, bucket, object string, metadata map[string]string) (io.WriteCloser, error) {
+	return DecryptRequestWithSequenceNumber(client, r, bucket, object, 0, metadata)
 }
 
 // DecryptBlocksWriter - decrypts multipart parts, while implementing a io.Writer compatible interface.
@@ -469,9 +506,10 @@ type DecryptBlocksWriter struct {
 	// Current part index
 	partIndex int
 	// Parts information
-	parts    []objectPartInfo
-	req      *http.Request
-	metadata map[string]string
+	parts          []objectPartInfo
+	req            *http.Request
+	bucket, object string
+	metadata       map[string]string
 
 	partEncRelOffset int64
 
@@ -499,7 +537,7 @@ func (w *DecryptBlocksWriter) buildDecrypter(partID int) error {
 		return err
 	}
 
-	objectEncryptionKey, err := decryptObjectInfo(key, m)
+	objectEncryptionKey, err := decryptObjectInfo(key, w.bucket, w.object, m)
 	if err != nil {
 		return err
 	}
@@ -594,29 +632,26 @@ func (w *DecryptBlocksWriter) Close() error {
 // DecryptAllBlocksCopyRequest - setup a struct which can decrypt many concatenated encrypted data
 // parts information helps to know the boundaries of each encrypted data block, this function decrypts
 // all parts starting from part-1.
-func DecryptAllBlocksCopyRequest(client io.Writer, r *http.Request, objInfo ObjectInfo) (io.WriteCloser, int64, error) {
-	w, _, size, err := DecryptBlocksRequest(client, r, 0, objInfo.Size, objInfo, true)
+func DecryptAllBlocksCopyRequest(client io.Writer, r *http.Request, bucket, object string, objInfo ObjectInfo) (io.WriteCloser, int64, error) {
+	w, _, size, err := DecryptBlocksRequest(client, r, bucket, object, 0, objInfo.Size, objInfo, true)
 	return w, size, err
 }
 
 // DecryptBlocksRequest - setup a struct which can decrypt many concatenated encrypted data
 // parts information helps to know the boundaries of each encrypted data block.
-func DecryptBlocksRequest(client io.Writer, r *http.Request, startOffset, length int64, objInfo ObjectInfo, copySource bool) (io.WriteCloser, int64, int64, error) {
-	seqNumber, encStartOffset, encLength := getEncryptedStartOffset(startOffset, length)
-
-	// Encryption length cannot be bigger than the file size, if it is
-	// which is allowed in AWS S3, we simply default to EncryptedSize().
-	if encLength+encStartOffset > objInfo.EncryptedSize() {
-		encLength = objInfo.EncryptedSize() - encStartOffset
-	}
+func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object string, startOffset, length int64, objInfo ObjectInfo, copySource bool) (io.WriteCloser, int64, int64, error) {
+	var seqNumber uint32
+	var encStartOffset, encLength int64
 
 	if len(objInfo.Parts) == 0 || !objInfo.IsEncryptedMultipart() {
+		seqNumber, encStartOffset, encLength = getEncryptedSinglePartOffsetLength(startOffset, length, objInfo)
+
 		var writer io.WriteCloser
 		var err error
 		if copySource {
-			writer, err = DecryptCopyRequest(client, r, objInfo.UserDefined)
+			writer, err = DecryptCopyRequest(client, r, bucket, object, objInfo.UserDefined)
 		} else {
-			writer, err = DecryptRequestWithSequenceNumber(client, r, seqNumber, objInfo.UserDefined)
+			writer, err = DecryptRequestWithSequenceNumber(client, r, bucket, object, seqNumber, objInfo.UserDefined)
 		}
 		if err != nil {
 			return nil, 0, 0, err
@@ -624,6 +659,7 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, startOffset, length
 		return writer, encStartOffset, encLength, nil
 	}
 
+	seqNumber, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
 	var partStartIndex int
 	var partStartOffset = startOffset
 	// Skip parts until final offset maps to a particular part offset.
@@ -656,6 +692,8 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, startOffset, length
 		parts:             objInfo.Parts,
 		partIndex:         partStartIndex,
 		req:               r,
+		bucket:            bucket,
+		object:            object,
 		customerKeyHeader: r.Header.Get(SSECustomerKey),
 		copySource:        copySource,
 	}
@@ -676,15 +714,62 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, startOffset, length
 		w.customerKeyHeader = r.Header.Get(SSECopyCustomerKey)
 	}
 
-	if err := w.buildDecrypter(partStartIndex + 1); err != nil {
+	if err := w.buildDecrypter(w.parts[w.partIndex].Number); err != nil {
 		return nil, 0, 0, err
 	}
 
 	return w, encStartOffset, encLength, nil
 }
 
-// getEncryptedStartOffset - fetch sequence number, encrypted start offset and encrypted length.
-func getEncryptedStartOffset(offset, length int64) (seqNumber uint32, encOffset int64, encLength int64) {
+// getEncryptedMultipartsOffsetLength - fetch sequence number, encrypted start offset and encrypted length.
+func getEncryptedMultipartsOffsetLength(offset, length int64, obj ObjectInfo) (uint32, int64, int64) {
+
+	// Calculate encrypted offset of a multipart object
+	computeEncOffset := func(off int64, obj ObjectInfo) (seqNumber uint32, encryptedOffset int64, err error) {
+		var curPartEndOffset uint64
+		var prevPartsEncSize int64
+		for _, p := range obj.Parts {
+			size, decErr := sio.DecryptedSize(uint64(p.Size))
+			if decErr != nil {
+				err = errObjectTampered // assign correct error type
+				return
+			}
+			if off < int64(curPartEndOffset+size) {
+				seqNumber, encryptedOffset, _ = getEncryptedSinglePartOffsetLength(off-int64(curPartEndOffset), 1, obj)
+				encryptedOffset += int64(prevPartsEncSize)
+				break
+			}
+			curPartEndOffset += size
+			prevPartsEncSize += p.Size
+		}
+		return
+	}
+
+	// Calculate the encrypted start offset corresponding to the plain offset
+	seqNumber, encStartOffset, _ := computeEncOffset(offset, obj)
+	// Calculate also the encrypted end offset corresponding to plain offset + plain length
+	_, encEndOffset, _ := computeEncOffset(offset+length-1, obj)
+
+	// encLength is the diff between encrypted end offset and encrypted start offset + one package size
+	// to ensure all encrypted data are covered
+	encLength := encEndOffset - encStartOffset + (64*1024 + 32)
+
+	// Calculate total size of all parts
+	var totalPartsLength int64
+	for _, p := range obj.Parts {
+		totalPartsLength += p.Size
+	}
+
+	// Set encLength to maximum possible value if it exceeded total parts size
+	if encLength+encStartOffset > totalPartsLength {
+		encLength = totalPartsLength - encStartOffset
+	}
+
+	return seqNumber, encStartOffset, encLength
+}
+
+// getEncryptedSinglePartOffsetLength - fetch sequence number, encrypted start offset and encrypted length.
+func getEncryptedSinglePartOffsetLength(offset, length int64, objInfo ObjectInfo) (seqNumber uint32, encOffset int64, encLength int64) {
 	onePkgSize := int64(sseDAREPackageBlockSize + sseDAREPackageMetaSize)
 
 	seqNumber = uint32(offset / sseDAREPackageBlockSize)
@@ -702,6 +787,9 @@ func getEncryptedStartOffset(offset, length int64) (seqNumber uint32, encOffset
 		encLength += onePkgSize
 	}
 
+	if encLength+encOffset > objInfo.EncryptedSize() {
+		encLength = objInfo.EncryptedSize() - encOffset
+	}
 	return seqNumber, encOffset, encLength
 }
 
@@ -746,11 +834,23 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	if !o.IsEncrypted() {
 		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}
-	size, err := sio.DecryptedSize(uint64(o.Size))
-	if err != nil {
-		err = errObjectTampered // assign correct error type
+	if len(o.Parts) == 0 || !o.IsEncryptedMultipart() {
+		size, err := sio.DecryptedSize(uint64(o.Size))
+		if err != nil {
+			err = errObjectTampered // assign correct error type
+		}
+		return int64(size), err
 	}
-	return int64(size), err
+
+	var size int64
+	for _, part := range o.Parts {
+		partSize, err := sio.DecryptedSize(uint64(part.Size))
+		if err != nil {
+			return 0, errObjectTampered
+		}
+		size += int64(partSize)
+	}
+	return size, nil
 }
 
 // EncryptedSize returns the size of the object after encryption.

cmd/encryption-v1_test.go

@@ -308,7 +308,7 @@ func TestEncryptRequest(t *testing.T) {
 		for k, v := range test.header {
 			req.Header.Set(k, v)
 		}
-		_, err := EncryptRequest(content, req, test.metadata)
+		_, err := EncryptRequest(content, req, "bucket", "object", test.metadata)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
 		}
@@ -328,11 +328,14 @@ func TestEncryptRequest(t *testing.T) {
 }
 
 var decryptRequestTests = []struct {
-	header     map[string]string
-	metadata   map[string]string
-	shouldFail bool
+	bucket, object string
+	header         map[string]string
+	metadata       map[string]string
+	shouldFail     bool
 }{
 	{
+		bucket: "bucket",
+		object: "object",
 		header: map[string]string{
 			SSECustomerAlgorithm: "AES256",
 			SSECustomerKey:       "MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=",
@@ -346,6 +349,23 @@ var decryptRequestTests = []struct {
 		shouldFail: false,
 	},
 	{
+		bucket: "bucket",
+		object: "object",
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=",
+			SSECustomerKeyMD5:    "7PpPLAK26ONlVUGOWlusfg==",
+		},
+		metadata: map[string]string{
+			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareV2HmacSha256,
+			ServerSideEncryptionIV:            "qEqmsONcorqlcZXJxaw32H04eyXyXwUgjHzlhkaIYrU=",
+			ServerSideEncryptionSealedKey:     "IAAfAIM14ugTGcM/dIrn4iQMrkl1sjKyeBQ8FBEvRebYj8vWvxG+0cJRpC6NXRU1wJN50JaUOATjO7kz0wZ2mA==",
+		},
+		shouldFail: false,
+	},
+	{
+		bucket: "bucket",
+		object: "object",
 		header: map[string]string{
 			SSECustomerAlgorithm: "AES256",
 			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
@@ -359,6 +379,8 @@ var decryptRequestTests = []struct {
 		shouldFail: true,
 	},
 	{
+		bucket: "bucket",
+		object: "object",
 		header: map[string]string{
 			SSECustomerAlgorithm: "AES256",
 			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
@@ -372,6 +394,8 @@ var decryptRequestTests = []struct {
 		shouldFail: true,
 	},
 	{
+		bucket: "bucket",
+		object: "object",
 		header: map[string]string{
 			SSECustomerAlgorithm: "AES256",
 			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
@@ -384,21 +408,39 @@ var decryptRequestTests = []struct {
 		},
 		shouldFail: true,
 	},
+	{
+		bucket: "bucket",
+		object: "object-2",
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=",
+			SSECustomerKeyMD5:    "7PpPLAK26ONlVUGOWlusfg==",
+		},
+		metadata: map[string]string{
+			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareV2HmacSha256,
+			ServerSideEncryptionIV:            "qEqmsONcorqlcZXJxaw32H04eyXyXwUgjHzlhkaIYrU=",
+			ServerSideEncryptionSealedKey:     "IAAfAIM14ugTGcM/dIrn4iQMrkl1sjKyeBQ8FBEvRebYj8vWvxG+0cJRpC6NXRU1wJN50JaUOATjO7kz0wZ2mA==",
+		},
+		shouldFail: true,
+	},
 }
 
 func TestDecryptRequest(t *testing.T) {
 	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
 	globalIsSSL = true
-	for i, test := range decryptRequestTests {
+	for i, test := range decryptRequestTests[1:] {
 		client := bytes.NewBuffer(nil)
 		req := &http.Request{Header: http.Header{}}
 		for k, v := range test.header {
 			req.Header.Set(k, v)
 		}
-		_, err := DecryptRequest(client, req, test.metadata)
+		_, err := DecryptRequest(client, req, test.bucket, test.object, test.metadata)
 		if err != nil && !test.shouldFail {
 			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
 		}
+		if err == nil && test.shouldFail {
+			t.Fatalf("Test %d: should fail but passed", i)
+		}
 		if key, ok := test.metadata[SSECustomerKey]; ok {
 			t.Errorf("Test %d: Client provided key survived in metadata - key: %s", i, key)
 		}

cmd/endpoint.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"os"
 	"path"
 	"path/filepath"
 	"runtime"
@@ -442,6 +443,8 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		return serverAddr, endpoints, setupType, err
 	}
 
+	updateDomainIPs(uniqueArgs)
+
 	setupType = DistXLSetupType
 	return serverAddr, endpoints, setupType, nil
 }
@@ -493,3 +496,22 @@ func GetRemotePeers(endpoints EndpointList) []string {
 
 	return peerSet.ToSlice()
 }
+
+// In federated and distributed setup, update IP addresses of the hosts passed in command line
+// if MINIO_PUBLIC_IPS are not set manually
+func updateDomainIPs(endPoints set.StringSet) {
+	_, dok := os.LookupEnv("MINIO_DOMAIN")
+	_, eok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
+	_, iok := os.LookupEnv("MINIO_PUBLIC_IPS")
+	if dok && eok && !iok {
+		globalDomainIPs = set.NewStringSet()
+		for e := range endPoints {
+			host, _, _ := net.SplitHostPort(e)
+			ipList, _ := getHostIP4(host)
+			remoteIPList := ipList.FuncMatch(func(ip string, matchString string) bool {
+				return !strings.HasPrefix(ip, "127.")
+			}, "")
+			globalDomainIPs.Add(remoteIPList.ToSlice()[0])
+		}
+	}
+}

cmd/fs-v1-helpers.go

@@ -178,7 +178,6 @@ func fsStatVolume(ctx context.Context, volume string) (os.FileInfo, error) {
 	}
 
 	if !fi.IsDir() {
-		logger.LogIf(ctx, errVolumeAccessDenied)
 		return nil, errVolumeAccessDenied
 	}
 
@@ -200,7 +199,7 @@ func osErrToFSFileErr(err error) error {
 		return errFileAccessDenied
 	}
 	if isSysErrNotDir(err) {
-		return errFileAccessDenied
+		return errFileNotFound
 	}
 	if isSysErrPathNotFound(err) {
 		return errFileNotFound
@@ -219,8 +218,7 @@ func fsStatDir(ctx context.Context, statDir string) (os.FileInfo, error) {
 		return nil, err
 	}
 	if !fi.IsDir() {
-		logger.LogIf(ctx, errFileAccessDenied)
-		return nil, errFileAccessDenied
+		return nil, errFileNotFound
 	}
 	return fi, nil
 }
@@ -245,8 +243,7 @@ func fsStatFile(ctx context.Context, statFile string) (os.FileInfo, error) {
 		return nil, err
 	}
 	if fi.IsDir() {
-		logger.LogIf(ctx, errFileAccessDenied)
-		return nil, errFileAccessDenied
+		return nil, errFileNotFound
 	}
 	return fi, nil
 }

cmd/fs-v1-helpers_test.go

@@ -136,7 +136,7 @@ func TestFSStats(t *testing.T) {
 			srcFSPath:   path,
 			srcVol:      "success-vol",
 			srcPath:     "path",
-			expectedErr: errFileAccessDenied,
+			expectedErr: errFileNotFound,
 		},
 		// Test case - 6.
 		// Test case with src path segment > 255.

cmd/fs-v1-multipart.go

@@ -264,7 +264,7 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
 			}
 			return
 		}
-		// Close writer explicitly signalling we wrote all data.
+		// Close writer explicitly signaling we wrote all data.
 		if gerr := srcInfo.Writer.Close(); gerr != nil {
 			logger.LogIf(ctx, gerr)
 			return

cmd/fs-v1.go

@@ -34,6 +34,7 @@ import (
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/mimedb"
+	"github.com/minio/minio/pkg/mountinfo"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -44,8 +45,6 @@ var defaultEtag = "00000000000000000000000000000000-1"
 type FSObjects struct {
 	// Disk usage metrics
 	totalUsed uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
-	// Disk usage running routine
-	usageRunning int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
 
 	// Path to be exported over S3 API.
 	fsPath string
@@ -64,6 +63,8 @@ type FSObjects struct {
 	// ListObjects pool management.
 	listPool *treeWalkPool
 
+	diskMount bool
+
 	appendFileMap   map[string]*fsAppendFile
 	appendFileMapMu sync.Mutex
 
@@ -109,6 +110,9 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 
 	var err error
 	if fsPath, err = getValidPath(fsPath); err != nil {
+		if err == errMinDiskSize {
+			return nil, err
+		}
 		return nil, uiErrUnableToWriteInBackend(err)
 	}
 
@@ -138,6 +142,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		nsMutex:       newNSLock(false),
 		listPool:      newTreeWalkPool(globalLookupTimeout),
 		appendFileMap: make(map[string]*fsAppendFile),
+		diskMount:     mountinfo.IsLikelyMountPoint(fsPath),
 	}
 
 	// Once the filesystem has initialized hold the read lock for
@@ -146,17 +151,10 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 	// or cause changes on backend format.
 	fs.fsFormatRlk = rlk
 
-	// Initialize notification system.
-	if err = globalNotificationSys.Init(fs); err != nil {
-		return nil, uiErrUnableToReadFromBackend(err).Msg("Unable to initialize notification system")
+	if !fs.diskMount {
+		go fs.diskUsage(globalServiceDoneCh)
 	}
 
-	// Initialize policy system.
-	if err = globalPolicySys.Init(fs); err != nil {
-		return nil, uiErrUnableToReadFromBackend(err).Msg("Unable to initialize policy system")
-	}
-
-	go fs.diskUsage(globalServiceDoneCh)
 	go fs.cleanupStaleMultipartUploads(ctx, globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh)
 
 	// Return successfully initialized object layer.
@@ -173,32 +171,40 @@ func (fs *FSObjects) Shutdown(ctx context.Context) error {
 
 // diskUsage returns du information for the posix path, in a continuous routine.
 func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
-	ticker := time.NewTicker(globalUsageCheckInterval)
-	defer ticker.Stop()
-
 	usageFn := func(ctx context.Context, entry string) error {
-		var fi os.FileInfo
-		var err error
-		if hasSuffix(entry, slashSeparator) {
-			fi, err = fsStatDir(ctx, entry)
-		} else {
-			fi, err = fsStatFile(ctx, entry)
+		if globalHTTPServer != nil {
+			// Wait at max 1 minute for an inprogress request
+			// before proceeding to count the usage.
+			waitCount := 60
+			// Any requests in progress, delay the usage.
+			for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
+				waitCount--
+				time.Sleep(1 * time.Second)
+			}
 		}
-		if err != nil {
-			return err
+
+		select {
+		case <-doneCh:
+			return errWalkAbort
+		default:
+			var fi os.FileInfo
+			var err error
+			if hasSuffix(entry, slashSeparator) {
+				fi, err = fsStatDir(ctx, entry)
+			} else {
+				fi, err = fsStatFile(ctx, entry)
+			}
+			if err != nil {
+				return err
+			}
+			atomic.AddUint64(&fs.totalUsed, uint64(fi.Size()))
 		}
-		atomic.AddUint64(&fs.totalUsed, uint64(fi.Size()))
 		return nil
 	}
 
-	// Check if disk usage routine is running, if yes then return.
-	if atomic.LoadInt32(&fs.usageRunning) == 1 {
-		return
-	}
-	atomic.StoreInt32(&fs.usageRunning, 1)
-	defer atomic.StoreInt32(&fs.usageRunning, 0)
-
-	if err := getDiskUsage(context.Background(), fs.fsPath, usageFn); err != nil {
+	// Return this routine upon errWalkAbort, continue for any other error on purpose
+	// so that we can start the routine freshly in another 12 hours.
+	if err := getDiskUsage(context.Background(), fs.fsPath, usageFn); err == errWalkAbort {
 		return
 	}
 
@@ -206,15 +212,20 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
 		select {
 		case <-doneCh:
 			return
-		case <-ticker.C:
-			// Check if disk usage routine is running, if yes let it finish.
-			if atomic.LoadInt32(&fs.usageRunning) == 1 {
-				continue
-			}
-			atomic.StoreInt32(&fs.usageRunning, 1)
-
+		case <-time.After(globalUsageCheckInterval):
 			var usage uint64
 			usageFn = func(ctx context.Context, entry string) error {
+				if globalHTTPServer != nil {
+					// Wait at max 1 minute for an inprogress request
+					// before proceeding to count the usage.
+					waitCount := 60
+					// Any requests in progress, delay the usage.
+					for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
+						waitCount--
+						time.Sleep(1 * time.Second)
+					}
+				}
+
 				var fi os.FileInfo
 				var err error
 				if hasSuffix(entry, slashSeparator) {
@@ -230,11 +241,8 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
 			}
 
 			if err := getDiskUsage(context.Background(), fs.fsPath, usageFn); err != nil {
-				atomic.StoreInt32(&fs.usageRunning, 0)
 				continue
 			}
-
-			atomic.StoreInt32(&fs.usageRunning, 0)
 			atomic.StoreUint64(&fs.totalUsed, usage)
 		}
 	}
@@ -242,8 +250,16 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
 
 // StorageInfo - returns underlying storage statistics.
 func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo {
+	di, err := getDiskInfo(fs.fsPath)
+	if err != nil {
+		return StorageInfo{}
+	}
+	used := di.Total - di.Free
+	if !fs.diskMount {
+		used = atomic.LoadUint64(&fs.totalUsed)
+	}
 	storageInfo := StorageInfo{
-		Used: atomic.LoadUint64(&fs.totalUsed),
+		Used: used,
 	}
 	storageInfo.Backend.Type = FS
 	return storageInfo
@@ -479,7 +495,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 			}
 			return
 		}
-		// Close writer explicitly signalling we wrote all data.
+		// Close writer explicitly signaling we wrote all data.
 		if gerr := srcInfo.Writer.Close(); gerr != nil {
 			logger.LogIf(ctx, gerr)
 			return
@@ -695,7 +711,7 @@ func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object s
 	}
 
 	if _, err := fs.statBucketDir(ctx, bucket); err != nil {
-		return oi, toObjectErr(err, bucket)
+		return oi, err
 	}
 
 	if strings.HasSuffix(object, slashSeparator) && !fs.isObjectDir(bucket, object) {
@@ -1214,7 +1230,7 @@ func (fs *FSObjects) SetBucketPolicy(ctx context.Context, bucket string, policy
 
 // GetBucketPolicy will get policy on bucket
 func (fs *FSObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
-	return GetPolicyConfig(fs, bucket)
+	return getPolicyConfig(fs, bucket)
 }
 
 // DeleteBucketPolicy deletes all policies on bucket
@@ -1224,7 +1240,12 @@ func (fs *FSObjects) DeleteBucketPolicy(ctx context.Context, bucket string) erro
 
 // ListObjectsV2 lists all blobs in bucket filtered by prefix
 func (fs *FSObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
-	loi, err := fs.ListObjects(ctx, bucket, prefix, continuationToken, delimiter, maxKeys)
+	marker := continuationToken
+	if marker == "" {
+		marker = startAfter
+	}
+
+	loi, err := fs.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
 		return result, err
 	}

cmd/gateway-common.go

@@ -154,6 +154,7 @@ func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
 		UserDefined:     userDefined,
 		ContentType:     oi.ContentType,
 		ContentEncoding: oi.Metadata.Get("Content-Encoding"),
+		StorageClass:    oi.StorageClass,
 	}
 }
 

cmd/gateway-main.go

@@ -34,7 +34,7 @@ import (
 )
 
 func init() {
-	logger.Init(GOPATH)
+	logger.Init(GOPATH, GOROOT)
 	logger.RegisterUIError(fmtError)
 }
 
@@ -170,8 +170,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	initNSLock(false) // Enable local namespace lock.
 
 	// Create new notification system.
-	globalNotificationSys, err = NewNotificationSys(globalServerConfig, EndpointList{})
-	logger.FatalIf(err, "Unable to create new notification system")
+	globalNotificationSys = NewNotificationSys(globalServerConfig, EndpointList{})
 
 	// Create new policy system.
 	globalPolicySys = NewPolicySys()
@@ -197,7 +196,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		getCert = globalTLSCerts.GetCertificate
 	}
 
-	globalHTTPServer = xhttp.NewServer([]string{gatewayAddr}, registerHandlers(router, globalHandlers...), getCert)
+	globalHTTPServer = xhttp.NewServer([]string{gatewayAddr}, criticalErrorHandler{registerHandlers(router, globalHandlers...)}, getCert)
 	globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
 	globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes
 	go func() {
@@ -215,6 +214,13 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.FatalIf(err, "Unable to initialize gateway backend")
 	}
 
+	if gw.Name() != "nas" {
+		// Initialize policy sys for all gateways. NAS gateway already
+		// initializes policy sys internally, avoid double initialization.
+		// Additionally also don't block the initialization of gateway.
+		go globalPolicySys.Init(newObject)
+	}
+
 	// Once endpoints are finalized, initialize the new object api.
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = newObject

cmd/gateway/azure/gateway-azure.go

@@ -82,6 +82,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
 EXAMPLES:
   1. Start minio gateway server for Azure Blob Storage backend.
@@ -100,6 +101,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}}
 `
 
@@ -238,6 +240,8 @@ func s3MetaToAzureProperties(ctx context.Context, s3Metadata map[string]string)
 			props.ContentMD5 = v
 		case k == "Content-Type":
 			props.ContentType = v
+		case k == "Content-Language":
+			props.ContentLanguage = v
 		}
 	}
 	return blobMeta, props, nil
@@ -292,6 +296,9 @@ func azurePropertiesToS3Meta(meta storage.BlobMetadata, props storage.BlobProper
 	if props.ContentType != "" {
 		s3Metadata["Content-Type"] = props.ContentType
 	}
+	if props.ContentLanguage != "" {
+		s3Metadata["Content-Language"] = props.ContentLanguage
+	}
 	return s3Metadata
 }
 
@@ -476,7 +483,6 @@ func (a *azureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi min
 			} // else continue
 		}
 	}
-	logger.LogIf(ctx, minio.BucketNotFound{Bucket: bucket})
 	return bi, minio.BucketNotFound{Bucket: bucket}
 }
 
@@ -604,7 +610,7 @@ func (a *azureObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
 // ListObjectsV2 - list all blobs in Azure bucket filtered by prefix
 func (a *azureObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result minio.ListObjectsV2Info, err error) {
 	marker := continuationToken
-	if startAfter != "" {
+	if marker == "" {
 		marker = startAfter
 	}
 
@@ -1124,7 +1130,6 @@ func (a *azureObjects) GetBucketPolicy(ctx context.Context, bucket string) (*pol
 	}
 
 	if perm.AccessType == storage.ContainerAccessTypePrivate {
-		logger.LogIf(ctx, minio.BucketPolicyNotFound{Bucket: bucket})
 		return nil, minio.BucketPolicyNotFound{Bucket: bucket}
 	} else if perm.AccessType != storage.ContainerAccessTypeContainer {
 		logger.LogIf(ctx, minio.NotImplemented{})

cmd/gateway/b2/gateway-b2.go

@@ -72,6 +72,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
 EXAMPLES:
   1. Start minio gateway server for B2 backend.
@@ -85,6 +86,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}}
 `
 	minio.RegisterGatewayCommand(cli.Command{
@@ -276,7 +278,6 @@ func (l *b2Objects) Bucket(ctx context.Context, bucket string) (*b2.Bucket, erro
 			return bkt, nil
 		}
 	}
-	logger.LogIf(ctx, minio.BucketNotFound{Bucket: bucket})
 	return nil, minio.BucketNotFound{Bucket: bucket}
 }
 
@@ -353,12 +354,20 @@ func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix strin
 // ListObjectsV2 lists all objects in B2 bucket filtered by prefix, returns upto max 1000 entries at a time.
 func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int,
 	fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
-	// fetchOwner, startAfter are not supported and unused.
+	// fetchOwner is not supported and unused.
+	marker := continuationToken
+	if marker == "" {
+		// B2's continuation token is an object name to "start at" rather than "start after"
+		// startAfter plus the lowest character B2 supports is used so that the startAfter
+		// object isn't included in the results
+		marker = startAfter + " "
+	}
+
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
 		return loi, err
 	}
-	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, continuationToken, prefix, delimiter)
+	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
 	if lerr != nil {
 		logger.LogIf(ctx, lerr)
 		return loi, b2ToObjectError(lerr, bucket)
@@ -770,7 +779,6 @@ func (l *b2Objects) GetBucketPolicy(ctx context.Context, bucket string) (*policy
 	// just return back as policy not found for all cases.
 	// CreateBucket always sets the value to allPrivate by default.
 	if bkt.Type != bucketTypeReadOnly {
-		logger.LogIf(ctx, minio.BucketPolicyNotFound{Bucket: bucket})
 		return nil, minio.BucketPolicyNotFound{Bucket: bucket}
 	}
 

cmd/gateway/gcs/gateway-gcs.go

@@ -118,6 +118,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
   GCS credentials file:
      GOOGLE_APPLICATION_CREDENTIALS: Path to credentials.json
@@ -137,6 +138,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}} mygcsprojectid
 `
 
@@ -656,6 +658,10 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri
 
 // ListObjectsV2 - lists all blobs in GCS bucket filtered by prefix
 func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (minio.ListObjectsV2Info, error) {
+	if continuationToken == "" && startAfter != "" {
+		continuationToken = toGCSPageToken(startAfter)
+	}
+
 	it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
 		Delimiter: delimiter,
 		Prefix:    prefix,
@@ -1009,12 +1015,8 @@ func (l *gcsGateway) AbortMultipartUpload(ctx context.Context, bucket string, ke
 
 // CompleteMultipartUpload completes ongoing multipart upload and finalizes object
 // Note that there is a limit (currently 32) to the number of components that can
-// be composed in a single operation. There is a limit (currently 1024) to the total
-// number of components for a given composite object. This means you can append to
-// each object at most 1023 times. There is a per-project rate limit (currently 200)
-// to the number of components you can compose per second. This rate counts both the
-// components being appended to a composite object as well as the components being
-// copied when the composite object of which they are a part is copied.
+// be composed in a single operation. There is a per-project rate limit (currently 200)
+// to the number of source objects you can compose per second.
 func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string, key string, uploadID string, uploadedParts []minio.CompletePart) (minio.ObjectInfo, error) {
 	meta := gcsMultipartMetaName(uploadID)
 	object := l.client.Bucket(bucket).Object(meta)
@@ -1134,7 +1136,6 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
 func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
 	policyInfo, err := minio.PolicyToBucketAccessPolicy(bucketPolicy)
 	if err != nil {
-		// This should not happen.
 		logger.LogIf(ctx, err)
 		return gcsToObjectError(err, bucket)
 	}
@@ -1190,7 +1191,6 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketP
 func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
 	rules, err := l.client.Bucket(bucket).ACL().List(l.ctx)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		return nil, gcsToObjectError(err, bucket)
 	}
 
@@ -1225,7 +1225,6 @@ func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*polic
 
 	// Return NoSuchBucketPolicy error, when policy is not set
 	if len(actionSet) == 0 {
-		logger.LogIf(ctx, minio.BucketPolicyNotFound{})
 		return nil, gcsToObjectError(minio.BucketPolicyNotFound{}, bucket)
 	}
 
@@ -1250,7 +1249,6 @@ func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*polic
 func (l *gcsGateway) DeleteBucketPolicy(ctx context.Context, bucket string) error {
 	// This only removes the storage.AllUsers policies
 	if err := l.client.Bucket(bucket).ACL().Delete(l.ctx, storage.AllUsers); err != nil {
-		logger.LogIf(ctx, err)
 		return gcsToObjectError(err, bucket)
 	}
 

cmd/gateway/manta/gateway-manta.go

@@ -78,6 +78,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
 EXAMPLES:
   1. Start minio gateway server for Manta Object Storage backend.
@@ -102,6 +103,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}}
 `
 
@@ -345,6 +347,13 @@ func (t *tritonObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
 		dirName = path.Join(mantaRoot, bucket, pathDir)
 	}
 
+	if marker != "" {
+		// Manta uses the marker as the key to start at rather than start after
+		// A space is appended to the marker so that the corresponding object is not
+		// included in the results
+		marker += " "
+	}
+
 	input = &storage.ListDirectoryInput{
 		DirectoryName: dirName,
 		Limit:         uint64(maxKeys),
@@ -417,6 +426,18 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti
 		pathBase = path.Base(prefix)
 	)
 
+	marker := continuationToken
+	if marker == "" {
+		marker = startAfter
+	}
+
+	if marker != "" {
+		// Manta uses the marker as the key to start at rather than start after.
+		// A space is appended to the marker so that the corresponding object is not
+		// included in the results
+		marker += " "
+	}
+
 	if pathDir := path.Dir(prefix); pathDir == "." {
 		dirName = path.Join(mantaRoot, bucket)
 	} else {
@@ -426,7 +447,7 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti
 	input = &storage.ListDirectoryInput{
 		DirectoryName: dirName,
 		Limit:         uint64(maxKeys),
-		Marker:        continuationToken,
+		Marker:        marker,
 	}
 	objs, err = t.client.Dir().List(ctx, input)
 	if err != nil {

cmd/gateway/nas/gateway-nas.go

@@ -17,12 +17,9 @@
 package nas
 
 import (
-	"context"
-
 	"github.com/minio/cli"
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/policy"
 )
 
 const (
@@ -57,6 +54,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
 EXAMPLES:
   1. Start minio gateway server for NAS backend.
@@ -70,6 +68,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}} /shared/nasvol
 `
 
@@ -126,8 +125,3 @@ type nasObjects struct {
 func (l *nasObjects) IsNotificationSupported() bool {
 	return false
 }
-
-// GetBucketPolicy will get policy on bucket
-func (l *nasObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
-	return minio.GetPolicyConfig(l, bucket)
-}

cmd/gateway/oss/gateway-oss.go

@@ -74,6 +74,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
 EXAMPLES:
   1. Start minio gateway server for Aliyun OSS backend.
@@ -92,6 +93,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}}
 `
 
@@ -477,8 +479,11 @@ func ossListObjects(ctx context.Context, client *oss.Client, bucket, prefix, mar
 // ossListObjectsV2 lists all blobs in OSS bucket filtered by prefix.
 func ossListObjectsV2(ctx context.Context, client *oss.Client, bucket, prefix, continuationToken, delimiter string, maxKeys int,
 	fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
-	// fetchOwner and startAfter are not supported and unused.
+	// fetchOwner is not supported and unused.
 	marker := continuationToken
+	if marker == "" {
+		marker = startAfter
+	}
 
 	resultV1, err := ossListObjects(ctx, client, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
@@ -1016,7 +1021,6 @@ func (l *ossObjects) GetBucketPolicy(ctx context.Context, bucket string) (*polic
 	switch result.ACL {
 	case string(oss.ACLPrivate):
 		// By default, all buckets starts with a "private" policy.
-		logger.LogIf(ctx, minio.BucketPolicyNotFound{})
 		return nil, ossToObjectError(minio.BucketPolicyNotFound{}, bucket)
 	case string(oss.ACLPublicRead):
 		readOnly = true

cmd/gateway/s3/gateway-s3.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -65,6 +65,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
 EXAMPLES:
   1. Start minio gateway server for AWS S3 backend.
@@ -83,6 +84,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}}
 `
 
@@ -97,12 +99,16 @@ EXAMPLES:
 
 // Handler for 'minio gateway s3' command line.
 func s3GatewayMain(ctx *cli.Context) {
-	// Validate gateway arguments.
-	host := ctx.Args().First()
-	// Validate gateway arguments.
-	logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
+	args := ctx.Args()
+	if !ctx.Args().Present() {
+		args = cli.Args{"https://s3.amazonaws.com"}
+	}
 
-	minio.StartGateway(ctx, &S3{host})
+	// Validate gateway arguments.
+	logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), args.First()), "Invalid argument")
+
+	// Start the gateway..
+	minio.StartGateway(ctx, &S3{args.First()})
 }
 
 // S3 implements Gateway.
@@ -115,34 +121,46 @@ func (g *S3) Name() string {
 	return s3Backend
 }
 
-// NewGatewayLayer returns s3 ObjectLayer.
-func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
-	var err error
-	var endpoint string
-	var secure = true
+// newS3 - Initializes a new client by auto probing S3 server signature.
+func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
+	if url == "" {
+		url = "https://s3.amazonaws.com"
+	}
 
-	// Validate host parameters.
-	if g.host != "" {
-		// Override default params if the host is provided
-		endpoint, secure, err = minio.ParseGatewayEndpoint(g.host)
+	// Override default params if the host is provided
+	endpoint, secure, err := minio.ParseGatewayEndpoint(url)
+	if err != nil {
+		return nil, err
+	}
+
+	clnt, err := miniogo.NewV4(endpoint, accessKey, secretKey, secure)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err = clnt.BucketExists("probe-bucket-sign"); err != nil {
+		clnt, err = miniogo.NewV2(endpoint, accessKey, secretKey, secure)
 		if err != nil {
 			return nil, err
 		}
+		if _, err = clnt.BucketExists("probe-bucket-sign"); err != nil {
+			return nil, err
+		}
 	}
 
-	// Default endpoint parameters
-	if endpoint == "" {
-		endpoint = "s3.amazonaws.com"
-	}
+	return &miniogo.Core{Client: clnt}, nil
+}
 
-	// Initialize minio client object.
-	client, err := miniogo.NewCore(endpoint, creds.AccessKey, creds.SecretKey, secure)
+// NewGatewayLayer returns s3 ObjectLayer.
+func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
+	// Probe S3 signature with input credentials.
+	clnt, err := newS3(g.host, creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		return nil, err
 	}
 
 	return &s3Objects{
-		Client: client,
+		Client: clnt,
 	}, nil
 }
 
@@ -209,7 +227,6 @@ func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.
 		}, nil
 	}
 
-	logger.LogIf(ctx, minio.BucketNotFound{Bucket: bucket})
 	return bi, minio.BucketNotFound{Bucket: bucket}
 }
 
@@ -255,7 +272,7 @@ func (l *s3Objects) ListObjects(ctx context.Context, bucket string, prefix strin
 
 // ListObjectsV2 lists all blobs in S3 bucket filtered by prefix
 func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, e error) {
-	result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys)
+	result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys, startAfter)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return loi, minio.ErrorRespToObjectError(err, bucket)
@@ -446,7 +463,6 @@ func (l *s3Objects) SetBucketPolicy(ctx context.Context, bucket string, bucketPo
 func (l *s3Objects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
 	data, err := l.Client.GetBucketPolicy(bucket)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		return nil, minio.ErrorRespToObjectError(err, bucket)
 	}
 

cmd/gateway/sia/gateway-sia.go

@@ -79,6 +79,7 @@ ENVIRONMENT VARIABLES: (Default values in parenthesis)
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
   SIA_TEMP_DIR:        The name of the local Sia temporary storage directory. (.sia_temp)
   SIA_API_PASSWORD:    API password for Sia daemon. (default is empty)
@@ -91,6 +92,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}}
 `
 

cmd/generic-handlers.go

@@ -19,13 +19,18 @@ package cmd
 import (
 	"bufio"
 	"context"
+	"fmt"
 	"net"
 	"net/http"
 	"strings"
 	"time"
 
+	"github.com/minio/minio-go/pkg/set"
+
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/dns"
+	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/sys"
 	"github.com/rs/cors"
 	"golang.org/x/time/rate"
@@ -611,14 +616,81 @@ func (h pathValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	h.handler.ServeHTTP(w, r)
 }
 
+// To forward the path style requests on a bucket to the right
+// configured server, bucket to IP configuration is obtained
+// from centralized etcd configuration service.
+type bucketForwardingHandler struct {
+	fwd     *handlers.Forwarder
+	handler http.Handler
+}
+
+func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if globalDNSConfig == nil || globalDomainName == "" || guessIsBrowserReq(r) || guessIsHealthCheckReq(r) || guessIsMetricsReq(r) || guessIsRPCReq(r) {
+		f.handler.ServeHTTP(w, r)
+		return
+	}
+	bucket, object := urlPath2BucketObjectName(r.URL.Path)
+	// ListBucket requests should be handled at current endpoint as
+	// all buckets data can be fetched from here.
+	if r.Method == http.MethodGet && bucket == "" && object == "" {
+		f.handler.ServeHTTP(w, r)
+		return
+	}
+
+	// MakeBucket requests should be handled at current endpoint
+	if r.Method == http.MethodPut && bucket != "" && object == "" {
+		f.handler.ServeHTTP(w, r)
+		return
+	}
+
+	// CopyObject requests should be handled at current endpoint as path style
+	// requests have target bucket and object in URI and source details are in
+	// header fields
+	if r.Method == http.MethodPut && r.Header.Get("X-Amz-Copy-Source") != "" {
+		f.handler.ServeHTTP(w, r)
+		return
+	}
+	sr, err := globalDNSConfig.Get(bucket)
+	if err != nil {
+		if err == dns.ErrNoEntriesFound {
+			writeErrorResponse(w, ErrNoSuchBucket, r.URL)
+		} else {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		}
+		return
+	}
+	if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
+		host, port := getRandomHostPort(sr)
+		r.URL.Scheme = "http"
+		if globalIsSSL {
+			r.URL.Scheme = "https"
+		}
+		r.URL.Host = fmt.Sprintf("%s:%d", host, port)
+		f.fwd.ServeHTTP(w, r)
+		return
+	}
+	f.handler.ServeHTTP(w, r)
+}
+
+// setBucketForwardingHandler middleware forwards the path style requests
+// on a bucket to the right bucket location, bucket to IP configuration
+// is obtained from centralized etcd configuration service.
+func setBucketForwardingHandler(h http.Handler) http.Handler {
+	fwd := handlers.NewForwarder(&handlers.Forwarder{
+		PassHost:     true,
+		RoundTripper: NewCustomHTTPTransport(),
+	})
+	return bucketForwardingHandler{fwd, h}
+}
+
 // setRateLimitHandler middleware limits the throughput to h using a
 // rate.Limiter token bucket configured with maxOpenFileLimit and
 // burst set to 1. The request will idle for up to 1*time.Second.
 // If the limiter detects the deadline will be exceeded, the request is
-// cancelled immediately.
+// canceled immediately.
 func setRateLimitHandler(h http.Handler) http.Handler {
 	_, maxLimit, err := sys.GetMaxOpenFileLimit()
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to get maximum open file limit", context.Background())
 	// Burst value is set to 1 to allow only maxOpenFileLimit
 	// requests to happen at once.
 	l := rate.NewLimiter(rate.Limit(maxLimit), 1)
@@ -665,3 +737,20 @@ func (s securityHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
 	header.Set("Content-Security-Policy", "block-all-mixed-content") // prevent mixed (HTTP / HTTPS content)
 	s.handler.ServeHTTP(w, r)
 }
+
+// criticalErrorHandler handles critical server failures caused by
+// `panic(logger.ErrCritical)` as done by `logger.CriticalIf`.
+//
+// It should be always the first / highest HTTP handler.
+type criticalErrorHandler struct{ handler http.Handler }
+
+func (h criticalErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	defer func() {
+		if err := recover(); err == logger.ErrCritical { // handle
+			writeErrorResponse(w, ErrInternalError, r.URL)
+		} else if err != nil {
+			panic(err) // forward other panic calls
+		}
+	}()
+	h.handler.ServeHTTP(w, r)
+}

cmd/globals.go

@@ -22,11 +22,15 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/minio/minio-go/pkg/set"
+
+	etcd "github.com/coreos/etcd/clientv3"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/fatih/color"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/certs"
+	"github.com/minio/minio/pkg/dns"
 )
 
 // minio configuration related constants.
@@ -159,7 +163,8 @@ var (
 	globalPublicCerts []*x509.Certificate
 
 	globalIsEnvDomainName bool
-	globalDomainName      string // Root domain for virtual host style requests
+	globalDomainName      string        // Root domain for virtual host style requests
+	globalDomainIPs       set.StringSet // Root domain IP address(s) for a distributed Minio deployment
 
 	globalListingTimeout   = newDynamicTimeout( /*30*/ 600*time.Second /*5*/, 600*time.Second) // timeout for listing related ops
 	globalObjectTimeout    = newDynamicTimeout( /*1*/ 10*time.Minute /*10*/, 600*time.Second)  // timeout for Object API related ops
@@ -174,17 +179,23 @@ var (
 	// Set to store standard storage class
 	globalStandardStorageClass storageClass
 
-	globalIsEnvWORM   bool
+	globalIsEnvWORM bool
+	// Is worm enabled
 	globalWORMEnabled bool
 
 	// Is Disk Caching set up
 	globalIsDiskCacheEnabled bool
+
 	// Disk cache drives
 	globalCacheDrives []string
+
 	// Disk cache excludes
 	globalCacheExcludes []string
+
 	// Disk cache expiry
 	globalCacheExpiry = 90
+	// Max allowed disk cache percentage
+	globalCacheMaxUse = 80
 
 	// RPC V1 - Initial version
 	// RPC V2 - format.json XL version changed to 2
@@ -192,12 +203,18 @@ var (
 	// Current RPC version
 	globalRPCAPIVersion = RPCVersion{3, 0, 0}
 
-	// Add new variable global values here.
+	// Allocated etcd endpoint for config and bucket DNS.
+	globalEtcdClient *etcd.Client
+
+	// Allocated DNS config wrapper over etcd client.
+	globalDNSConfig dns.Config
 
 	// Default usage check interval value.
 	globalDefaultUsageCheckInterval = 12 * time.Hour // 12 hours
 	// Usage check interval value.
 	globalUsageCheckInterval = globalDefaultUsageCheckInterval
+
+	// Add new variable global values here.
 )
 
 // global colors.

cmd/handler-utils.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/handlers"
 	httptracer "github.com/minio/minio/pkg/handlers"
 )
 
@@ -113,40 +114,54 @@ var userMetadataKeyPrefixes = []string{
 	"X-Minio-Meta-",
 }
 
-// extractMetadataFromHeader extracts metadata from HTTP header.
-func extractMetadataFromHeader(ctx context.Context, header http.Header) (map[string]string, error) {
-	if header == nil {
-		logger.LogIf(ctx, errInvalidArgument)
-		return nil, errInvalidArgument
+// extractMetadata extracts metadata from HTTP header and HTTP queryString.
+func extractMetadata(ctx context.Context, r *http.Request) (metadata map[string]string, err error) {
+	query := r.URL.Query()
+	header := r.Header
+	metadata = make(map[string]string)
+	// Extract all query values.
+	err = extractMetadataFromMap(ctx, query, metadata)
+	if err != nil {
+		return nil, err
 	}
-	metadata := make(map[string]string)
 
+	// Extract all header values.
+	err = extractMetadataFromMap(ctx, header, metadata)
+	if err != nil {
+		return nil, err
+	}
+
+	// Success.
+	return metadata, nil
+}
+
+// extractMetadata extracts metadata from map values.
+func extractMetadataFromMap(ctx context.Context, v map[string][]string, m map[string]string) error {
+	if v == nil {
+		logger.LogIf(ctx, errInvalidArgument)
+		return errInvalidArgument
+	}
 	// Save all supported headers.
 	for _, supportedHeader := range supportedHeaders {
-		canonicalHeader := http.CanonicalHeaderKey(supportedHeader)
-		// HTTP headers are case insensitive, look for both canonical
-		// and non canonical entries.
-		if _, ok := header[canonicalHeader]; ok {
-			metadata[supportedHeader] = header.Get(canonicalHeader)
-		} else if _, ok := header[supportedHeader]; ok {
-			metadata[supportedHeader] = header.Get(supportedHeader)
+		if value, ok := v[http.CanonicalHeaderKey(supportedHeader)]; ok {
+			m[supportedHeader] = value[0]
+		} else if value, ok := v[supportedHeader]; ok {
+			m[supportedHeader] = value[0]
 		}
 	}
-
-	// Go through all other headers for any additional headers that needs to be saved.
-	for key := range header {
-		if key != http.CanonicalHeaderKey(key) {
-			logger.LogIf(ctx, errInvalidArgument)
-			return nil, errInvalidArgument
-		}
+	for key := range v {
 		for _, prefix := range userMetadataKeyPrefixes {
-			if strings.HasPrefix(key, prefix) {
-				metadata[key] = header.Get(key)
+			if !strings.HasPrefix(strings.ToLower(key), strings.ToLower(prefix)) {
+				continue
+			}
+			value, ok := v[key]
+			if ok {
+				m[key] = strings.Join(value, ",")
 				break
 			}
 		}
 	}
-	return metadata, nil
+	return nil
 }
 
 // The Query string for the redirect URL the client is
@@ -167,7 +182,7 @@ func extractReqParams(r *http.Request) map[string]string {
 
 	// Success.
 	return map[string]string{
-		"sourceIPAddress": r.RemoteAddr,
+		"sourceIPAddress": handlers.GetSourceIP(r),
 		// Add more fields here.
 	}
 }

cmd/handler-utils_test.go

@@ -165,9 +165,9 @@ func TestExtractMetadataHeaders(t *testing.T) {
 				"x-amz-meta-appid": []string{"amz-meta"},
 			},
 			metadata: map[string]string{
-				"X-Amz-Meta-Appid": "amz-meta",
+				"x-amz-meta-appid": "amz-meta",
 			},
-			shouldFail: true,
+			shouldFail: false,
 		},
 		// Empty header input returns empty metadata.
 		{
@@ -179,7 +179,8 @@ func TestExtractMetadataHeaders(t *testing.T) {
 
 	// Validate if the extracting headers.
 	for i, testCase := range testCases {
-		metadata, err := extractMetadataFromHeader(context.Background(), testCase.header)
+		metadata := make(map[string]string)
+		err := extractMetadataFromMap(context.Background(), testCase.header, metadata)
 		if err != nil && !testCase.shouldFail {
 			t.Fatalf("Test %d failed to extract metadata: %v", i+1, err)
 		}

cmd/http/server.go

@@ -59,7 +59,12 @@ type Server struct {
 	listenerMutex          *sync.Mutex   // to guard 'listener' field.
 	listener               *httpListener // HTTP listener for all 'Addrs' field.
 	inShutdown             uint32        // indicates whether the server is in shutdown or not
-	requestCount           int32         // counter holds no. of request in process.
+	requestCount           int32         // counter holds no. of request in progress.
+}
+
+// GetRequestCount - returns number of request in progress.
+func (srv *Server) GetRequestCount() int32 {
+	return atomic.LoadInt32(&srv.requestCount)
 }
 
 // Start - start HTTP server

cmd/lock-rpc-server.go

@@ -124,7 +124,10 @@ func (l *lockRPCReceiver) lockMaintenance(interval time.Duration) {
 	for _, nlrip := range nlripLongLived {
 		// Initialize client based on the long live locks.
 		host, err := xnet.ParseHost(nlrip.lri.node)
-		logger.CriticalIf(context.Background(), err)
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			continue
+		}
 		c, err := NewLockRPCClient(host)
 		if err != nil {
 			logger.LogIf(context.Background(), err)
@@ -183,7 +186,7 @@ func NewLockRPCServer() (*xrpc.Server, error) {
 // Register distributed NS lock handlers.
 func registerDistNSLockRouter(router *mux.Router) {
 	rpcServer, err := NewLockRPCServer()
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to initialize Lock RPC Server", context.Background())
 
 	// Start lock maintenance from all lock servers.
 	go startLockMaintenance(globalLockServer)

cmd/logger/logger.go

@@ -52,6 +52,25 @@ var matchingFuncNames = [...]string{
 	"http.HandlerFunc.ServeHTTP",
 	"cmd.serverMain",
 	"cmd.StartGateway",
+	"cmd.(*webAPIHandlers).ListBuckets",
+	"cmd.(*webAPIHandlers).MakeBucket",
+	"cmd.(*webAPIHandlers).DeleteBucket",
+	"cmd.(*webAPIHandlers).ListObjects",
+	"cmd.(*webAPIHandlers).RemoveObject",
+	"cmd.(*webAPIHandlers).Login",
+	"cmd.(*webAPIHandlers).GenerateAuth",
+	"cmd.(*webAPIHandlers).SetAuth",
+	"cmd.(*webAPIHandlers).GetAuth",
+	"cmd.(*webAPIHandlers).CreateURLToken",
+	"cmd.(*webAPIHandlers).Upload",
+	"cmd.(*webAPIHandlers).Download",
+	"cmd.(*webAPIHandlers).DownloadZip",
+	"cmd.(*webAPIHandlers).GetBucketPolicy",
+	"cmd.(*webAPIHandlers).ListAllBucketPolicies",
+	"cmd.(*webAPIHandlers).SetBucketPolicy",
+	"cmd.(*webAPIHandlers).PresignedGet",
+	"cmd.(*webAPIHandlers).ServerInfo",
+	"cmd.(*webAPIHandlers).StorageInfo",
 	// add more here ..
 }
 
@@ -142,22 +161,24 @@ func RegisterUIError(f func(string, error, bool) string) {
 // and GOROOT directories. Also append github.com/minio/minio
 // This is done to clean up the filename, when stack trace is
 // displayed when an error happens.
-func Init(goPath string) {
+func Init(goPath string, goRoot string) {
 
 	var goPathList []string
+	var goRootList []string
 	var defaultgoPathList []string
+	var defaultgoRootList []string
+	pathSeperator := ":"
 	// Add all possible GOPATH paths into trimStrings
 	// Split GOPATH depending on the OS type
 	if runtime.GOOS == "windows" {
-		goPathList = strings.Split(goPath, ";")
-		defaultgoPathList = strings.Split(build.Default.GOPATH, ";")
-	} else {
-		// All other types of OSs
-		goPathList = strings.Split(goPath, ":")
-		defaultgoPathList = strings.Split(build.Default.GOPATH, ":")
-
+		pathSeperator = ";"
 	}
 
+	goPathList = strings.Split(goPath, pathSeperator)
+	goRootList = strings.Split(goRoot, pathSeperator)
+	defaultgoPathList = strings.Split(build.Default.GOPATH, pathSeperator)
+	defaultgoRootList = strings.Split(build.Default.GOROOT, pathSeperator)
+
 	// Add trim string "{GOROOT}/src/" into trimStrings
 	trimStrings = []string{filepath.Join(runtime.GOROOT(), "src") + string(filepath.Separator)}
 
@@ -167,10 +188,21 @@ func Init(goPath string) {
 		trimStrings = append(trimStrings, filepath.Join(goPathString, "src")+string(filepath.Separator))
 	}
 
+	for _, goRootString := range goRootList {
+		trimStrings = append(trimStrings, filepath.Join(goRootString, "src")+string(filepath.Separator))
+	}
+
 	for _, defaultgoPathString := range defaultgoPathList {
 		trimStrings = append(trimStrings, filepath.Join(defaultgoPathString, "src")+string(filepath.Separator))
 	}
 
+	for _, defaultgoRootString := range defaultgoRootList {
+		trimStrings = append(trimStrings, filepath.Join(defaultgoRootString, "src")+string(filepath.Separator))
+	}
+
+	// Remove duplicate entries.
+	trimStrings = uniqueEntries(trimStrings)
+
 	// Add "github.com/minio/minio" as the last to cover
 	// paths like "{GOROOT}/src/github.com/minio/minio"
 	// and "{GOPATH}/src/github.com/minio/minio"
@@ -200,7 +232,7 @@ func getTrace(traceLevel int) []string {
 	var trace []string
 	pc, file, lineNumber, ok := runtime.Caller(traceLevel)
 
-	for ok {
+	for ok && file != "" {
 		// Clean up the common prefixes
 		file = trimTrace(file)
 		// Get the function name
@@ -330,13 +362,15 @@ func LogIf(ctx context.Context, err error) {
 	fmt.Println(output)
 }
 
-// CriticalIf :
-// Like LogIf with exit
-// It'll be called for fatal error conditions during run-time
+// ErrCritical is the value panic'd whenever CriticalIf is called.
+var ErrCritical struct{}
+
+// CriticalIf logs the provided error on the console. It fails the
+// current go-routine by causing a `panic(ErrCritical)`.
 func CriticalIf(ctx context.Context, err error) {
 	if err != nil {
 		LogIf(ctx, err)
-		os.Exit(1)
+		panic(ErrCritical)
 	}
 }
 

cmd/logger/utils.go

@@ -50,3 +50,16 @@ func ansiSaveAttributes() {
 func ansiRestoreAttributes() {
 	ansiEscape("8")
 }
+
+func uniqueEntries(paths []string) []string {
+	found := map[string]bool{}
+	unqiue := []string{}
+
+	for v := range paths {
+		if _, ok := found[paths[v]]; !ok {
+			found[paths[v]] = true
+			unqiue = append(unqiue, paths[v])
+		}
+	}
+	return unqiue
+}

cmd/namespace-lock.go

@@ -86,9 +86,9 @@ func newDsyncNodes(endpoints EndpointList) (clnts []dsync.NetLocker, myNode int)
 			locker = &(receiver.ll)
 		} else {
 			host, err := xnet.ParseHost(endpoint.Host)
-			logger.CriticalIf(context.Background(), err)
+			logger.FatalIf(err, "Unable to parse Lock RPC Host", context.Background())
 			locker, err = NewLockRPCClient(host)
-			logger.CriticalIf(context.Background(), err)
+			logger.FatalIf(err, "Unable to initialize Lock RPC Client", context.Background())
 		}
 
 		clnts = append(clnts, locker)

cmd/namespace-lock_test.go

@@ -226,7 +226,7 @@ func TestNamespaceForceUnlockTest(t *testing.T) {
 
 	select {
 	case <-ch:
-		// Signalled so all is fine.
+		// Signaled so all is fine.
 		break
 
 	case <-time.After(100 * time.Millisecond):

cmd/notification.go

@@ -68,28 +68,21 @@ type NotificationPeerErr struct {
 }
 
 // DeleteBucket - calls DeleteBucket RPC call on all peers.
-func (sys *NotificationSys) DeleteBucket(bucketName string) <-chan NotificationPeerErr {
-	errCh := make(chan NotificationPeerErr)
+func (sys *NotificationSys) DeleteBucket(ctx context.Context, bucketName string) {
 	go func() {
-		defer close(errCh)
-
 		var wg sync.WaitGroup
 		for addr, client := range sys.peerRPCClientMap {
 			wg.Add(1)
 			go func(addr xnet.Host, client *PeerRPCClient) {
 				defer wg.Done()
 				if err := client.DeleteBucket(bucketName); err != nil {
-					errCh <- NotificationPeerErr{
-						Host: addr,
-						Err:  err,
-					}
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
+					logger.LogIf(ctx, err)
 				}
 			}(addr, client)
 		}
 		wg.Wait()
 	}()
-
-	return errCh
 }
 
 // SetCredentials - calls SetCredentials RPC call on all peers.
@@ -120,104 +113,76 @@ func (sys *NotificationSys) SetCredentials(credentials auth.Credentials) map[xne
 }
 
 // SetBucketPolicy - calls SetBucketPolicy RPC call on all peers.
-func (sys *NotificationSys) SetBucketPolicy(bucketName string, bucketPolicy *policy.Policy) <-chan NotificationPeerErr {
-	errCh := make(chan NotificationPeerErr)
+func (sys *NotificationSys) SetBucketPolicy(ctx context.Context, bucketName string, bucketPolicy *policy.Policy) {
 	go func() {
-		defer close(errCh)
-
 		var wg sync.WaitGroup
 		for addr, client := range sys.peerRPCClientMap {
 			wg.Add(1)
 			go func(addr xnet.Host, client *PeerRPCClient) {
 				defer wg.Done()
 				if err := client.SetBucketPolicy(bucketName, bucketPolicy); err != nil {
-					errCh <- NotificationPeerErr{
-						Host: addr,
-						Err:  err,
-					}
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
+					logger.LogIf(ctx, err)
 				}
 			}(addr, client)
 		}
 		wg.Wait()
 	}()
-
-	return errCh
 }
 
 // RemoveBucketPolicy - calls RemoveBucketPolicy RPC call on all peers.
-func (sys *NotificationSys) RemoveBucketPolicy(bucketName string) <-chan NotificationPeerErr {
-	errCh := make(chan NotificationPeerErr)
+func (sys *NotificationSys) RemoveBucketPolicy(ctx context.Context, bucketName string) {
 	go func() {
-		defer close(errCh)
-
 		var wg sync.WaitGroup
 		for addr, client := range sys.peerRPCClientMap {
 			wg.Add(1)
 			go func(addr xnet.Host, client *PeerRPCClient) {
 				defer wg.Done()
 				if err := client.RemoveBucketPolicy(bucketName); err != nil {
-					errCh <- NotificationPeerErr{
-						Host: addr,
-						Err:  err,
-					}
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
+					logger.LogIf(ctx, err)
 				}
 			}(addr, client)
 		}
 		wg.Wait()
 	}()
-
-	return errCh
 }
 
 // PutBucketNotification - calls PutBucketNotification RPC call on all peers.
-func (sys *NotificationSys) PutBucketNotification(bucketName string, rulesMap event.RulesMap) <-chan NotificationPeerErr {
-	errCh := make(chan NotificationPeerErr)
+func (sys *NotificationSys) PutBucketNotification(ctx context.Context, bucketName string, rulesMap event.RulesMap) {
 	go func() {
-		defer close(errCh)
-
 		var wg sync.WaitGroup
 		for addr, client := range sys.peerRPCClientMap {
 			wg.Add(1)
 			go func(addr xnet.Host, client *PeerRPCClient, rulesMap event.RulesMap) {
 				defer wg.Done()
 				if err := client.PutBucketNotification(bucketName, rulesMap); err != nil {
-					errCh <- NotificationPeerErr{
-						Host: addr,
-						Err:  err,
-					}
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
+					logger.LogIf(ctx, err)
 				}
 			}(addr, client, rulesMap.Clone())
 		}
 		wg.Wait()
 	}()
-
-	return errCh
 }
 
 // ListenBucketNotification - calls ListenBucketNotification RPC call on all peers.
-func (sys *NotificationSys) ListenBucketNotification(bucketName string, eventNames []event.Name, pattern string,
-	targetID event.TargetID, localPeer xnet.Host) <-chan NotificationPeerErr {
-	errCh := make(chan NotificationPeerErr)
+func (sys *NotificationSys) ListenBucketNotification(ctx context.Context, bucketName string, eventNames []event.Name, pattern string,
+	targetID event.TargetID, localPeer xnet.Host) {
 	go func() {
-		defer close(errCh)
-
 		var wg sync.WaitGroup
 		for addr, client := range sys.peerRPCClientMap {
 			wg.Add(1)
 			go func(addr xnet.Host, client *PeerRPCClient) {
 				defer wg.Done()
 				if err := client.ListenBucketNotification(bucketName, eventNames, pattern, targetID, localPeer); err != nil {
-					errCh <- NotificationPeerErr{
-						Host: addr,
-						Err:  err,
-					}
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", addr.Name)
+					logger.LogIf(ctx, err)
 				}
 			}(addr, client)
 		}
 		wg.Wait()
 	}()
-
-	return errCh
 }
 
 // AddRemoteTarget - adds event rules map, HTTP/PeerRPC client target to bucket name.
@@ -464,12 +429,8 @@ func (sys *NotificationSys) Send(args eventArgs) []event.TargetIDErr {
 }
 
 // NewNotificationSys - creates new notification system object.
-func NewNotificationSys(config *serverConfig, endpoints EndpointList) (*NotificationSys, error) {
-	targetList, err := getNotificationTargets(config)
-	if err != nil {
-		return nil, err
-	}
-
+func NewNotificationSys(config *serverConfig, endpoints EndpointList) *NotificationSys {
+	targetList := getNotificationTargets(config)
 	peerRPCClientMap := makeRemoteRPCClients(endpoints)
 
 	// bucketRulesMap/bucketRemoteTargetRulesMap are initialized by NotificationSys.Init()
@@ -478,7 +439,7 @@ func NewNotificationSys(config *serverConfig, endpoints EndpointList) (*Notifica
 		bucketRulesMap:             make(map[string]event.RulesMap),
 		bucketRemoteTargetRulesMap: make(map[string]map[event.TargetID]event.RulesMap),
 		peerRPCClientMap:           peerRPCClientMap,
-	}, nil
+	}
 }
 
 type eventArgs struct {
@@ -556,13 +517,16 @@ func sendEvent(args eventArgs) {
 		return
 	}
 
-	for _, err := range globalNotificationSys.Send(args) {
-		reqInfo := &logger.ReqInfo{BucketName: args.BucketName, ObjectName: args.Object.Name}
-		reqInfo.AppendTags("EventName", args.EventName.String())
-		reqInfo.AppendTags("targetID", err.ID.Name)
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogOnceIf(ctx, err.Err, err.ID)
-	}
+	notifyCh := globalNotificationSys.Send(args)
+	go func() {
+		for _, err := range notifyCh {
+			reqInfo := &logger.ReqInfo{BucketName: args.BucketName, ObjectName: args.Object.Name}
+			reqInfo.AppendTags("EventName", args.EventName.String())
+			reqInfo.AppendTags("targetID", err.ID.Name)
+			ctx := logger.SetReqInfo(context.Background(), reqInfo)
+			logger.LogOnceIf(ctx, err.Err, err.ID)
+		}
+	}()
 }
 
 func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error {

cmd/object-api-utils.go

@@ -20,12 +20,15 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
+	"math/rand"
 	"path"
 	"runtime"
 	"strings"
+	"time"
 	"unicode/utf8"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/skyrings/skyring-common/tools/uuid"
 )
 
@@ -277,6 +280,22 @@ func isMinioReservedBucket(bucketName string) bool {
 	return bucketName == minioReservedBucket
 }
 
+// returns a slice of hosts by reading a slice of DNS records
+func getHostsSlice(records []dns.SrvRecord) []string {
+	var hosts []string
+	for _, r := range records {
+		hosts = append(hosts, r.Host)
+	}
+	return hosts
+}
+
+// returns a random host (and corresponding port) from a slice of DNS records
+func getRandomHostPort(records []dns.SrvRecord) (string, int) {
+	rand.Seed(time.Now().Unix())
+	srvRecord := records[rand.Intn(len(records))]
+	return srvRecord.Host, srvRecord.Port
+}
+
 // byBucketName is a collection satisfying sort.Interface.
 type byBucketName []BucketInfo
 

cmd/object-handlers-common.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/handlers"
 )
 
 // Validates the preconditions for CopyObjectPart, returns true if CopyObjectPart
@@ -243,7 +244,7 @@ func deleteObject(ctx context.Context, obj ObjectLayer, cache CacheObjectLayer,
 	}
 
 	// Get host and port from Request.RemoteAddr.
-	host, port, _ := net.SplitHostPort(r.RemoteAddr)
+	host, port, _ := net.SplitHostPort(handlers.GetSourceIP(r))
 
 	// Notify object deleted event.
 	sendEvent(eventArgs{

cmd/object-handlers.go

@@ -32,8 +32,11 @@ import (
 	"strconv"
 
 	"github.com/gorilla/mux"
+	miniogo "github.com/minio/minio-go"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/ioutil"
 	"github.com/minio/minio/pkg/policy"
@@ -157,7 +160,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 			// additionally also skipping mod(offset)64KiB boundaries.
 			writer = ioutil.LimitedWriter(writer, startOffset%(64*1024), length)
 
-			writer, startOffset, length, err = DecryptBlocksRequest(writer, r, startOffset, length, objInfo, false)
+			writer, startOffset, length, err = DecryptBlocksRequest(writer, r, bucket, object, startOffset, length, objInfo, false)
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -194,7 +197,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -266,7 +269,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 			writeErrorResponse(w, apiErr, r.URL)
 			return
 		} else if encrypted {
-			if _, err = DecryptRequest(w, r, objInfo.UserDefined); err != nil {
+			if _, err = DecryptRequest(w, r, bucket, object, objInfo.UserDefined); err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
 			}
@@ -290,7 +293,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 	w.WriteHeader(http.StatusOK)
 
 	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -309,7 +312,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 
 // Extract metadata relevant for an CopyObject operation based on conditional
 // header values specified in X-Amz-Metadata-Directive.
-func getCpObjMetadataFromHeader(ctx context.Context, header http.Header, userMeta map[string]string) (map[string]string, error) {
+func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta map[string]string) (map[string]string, error) {
 	// Make a copy of the supplied metadata to avoid
 	// to change the original one.
 	defaultMeta := make(map[string]string, len(userMeta))
@@ -319,13 +322,13 @@ func getCpObjMetadataFromHeader(ctx context.Context, header http.Header, userMet
 
 	// if x-amz-metadata-directive says REPLACE then
 	// we extract metadata from the input headers.
-	if isMetadataReplace(header) {
-		return extractMetadataFromHeader(ctx, header)
+	if isMetadataReplace(r.Header) {
+		return extractMetadata(ctx, r)
 	}
 
 	// if x-amz-metadata-directive says COPY then we
 	// return the default metadata.
-	if isMetadataCopy(header) {
+	if isMetadataCopy(r.Header) {
 		return defaultMeta, nil
 	}
 
@@ -459,7 +462,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			for k, v := range srcInfo.UserDefined {
 				encMetadata[k] = v
 			}
-			if err = rotateKey(oldKey, newKey, encMetadata); err != nil {
+			if err = rotateKey(oldKey, newKey, srcBucket, srcObject, encMetadata); err != nil {
 				pipeWriter.CloseWithError(err)
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -471,7 +474,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			if sseCopyC {
 				// Source is encrypted make sure to save the encrypted size.
 				writer = ioutil.LimitedWriter(writer, 0, srcInfo.Size)
-				writer, srcInfo.Size, err = DecryptAllBlocksCopyRequest(writer, r, srcInfo)
+				writer, srcInfo.Size, err = DecryptAllBlocksCopyRequest(writer, r, srcBucket, srcObject, srcInfo)
 				if err != nil {
 					pipeWriter.CloseWithError(err)
 					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -486,7 +489,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 				}
 			}
 			if sseC {
-				reader, err = newEncryptReader(pipeReader, newKey, encMetadata)
+				reader, err = newEncryptReader(reader, newKey, dstBucket, dstObject, encMetadata)
 				if err != nil {
 					pipeWriter.CloseWithError(err)
 					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -510,7 +513,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	}
 	srcInfo.Writer = writer
 
-	srcInfo.UserDefined, err = getCpObjMetadataFromHeader(ctx, r.Header, srcInfo.UserDefined)
+	srcInfo.UserDefined, err = getCpObjMetadataFromHeader(ctx, r, srcInfo.UserDefined)
 	if err != nil {
 		pipeWriter.CloseWithError(err)
 		writeErrorResponse(w, ErrInternalError, r.URL)
@@ -536,13 +539,73 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	// Copy source object to destination, if source and destination
-	// object is same then only metadata is updated.
-	objInfo, err := objectAPI.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo)
-	if err != nil {
-		pipeWriter.CloseWithError(err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
+	var objInfo ObjectInfo
+
+	// Checks if a remote putobject call is needed for CopyObject operation
+	// 1. If source and destination bucket names are same, it means no call needed to etcd to get destination info
+	// 2. If destination bucket doesn't exist locally, only then a etcd call is needed
+	var isRemoteCallRequired = func(ctx context.Context, src, dst string, objAPI ObjectLayer) bool {
+		if src == dst {
+			return false
+		}
+		_, berr := objAPI.GetBucketInfo(ctx, dst)
+		return berr == toObjectErr(errVolumeNotFound, dst)
+	}
+
+	// Returns a minio-go Client configured to access remote host described by destDNSRecord
+	// Applicable only in a federated deployment
+	var getRemoteInstanceClient = func(host string, port int) (*miniogo.Core, error) {
+		// In a federated deployment, all the instances share config files and hence expected to have same
+		// credentials. So, access current instances creds and use it to create client for remote instance
+		endpoint := net.JoinHostPort(host, strconv.Itoa(port))
+		accessKey := globalServerConfig.Credential.AccessKey
+		secretKey := globalServerConfig.Credential.SecretKey
+		return miniogo.NewCore(endpoint, accessKey, secretKey, globalIsSSL)
+	}
+
+	if isRemoteCallRequired(ctx, srcBucket, dstBucket, objectAPI) {
+		if globalDNSConfig == nil {
+			writeErrorResponse(w, ErrNoSuchBucket, r.URL)
+			return
+		}
+		var dstRecords []dns.SrvRecord
+		if dstRecords, err = globalDNSConfig.Get(dstBucket); err == nil {
+			go func() {
+				if gerr := objectAPI.GetObject(ctx, srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag); gerr != nil {
+					pipeWriter.CloseWithError(gerr)
+					writeErrorResponse(w, ErrInternalError, r.URL)
+					return
+				}
+				// Close writer explicitly to indicate data has been written
+				srcInfo.Writer.Close()
+			}()
+
+			// Send PutObject request to appropriate instance (in federated deployment)
+			host, port := getRandomHostPort(dstRecords)
+			client, rerr := getRemoteInstanceClient(host, port)
+			if rerr != nil {
+				pipeWriter.CloseWithError(rerr)
+				writeErrorResponse(w, ErrInternalError, r.URL)
+				return
+			}
+			remoteObjInfo, rerr := client.PutObject(dstBucket, dstObject, srcInfo.Reader, srcInfo.Size, "", "", srcInfo.UserDefined)
+			if rerr != nil {
+				pipeWriter.CloseWithError(rerr)
+				writeErrorResponse(w, ErrInternalError, r.URL)
+				return
+			}
+			objInfo.ETag = remoteObjInfo.ETag
+			objInfo.ModTime = remoteObjInfo.LastModified
+		}
+	} else {
+		// Copy source object to destination, if source and destination
+		// object is same then only metadata is updated.
+		objInfo, err = objectAPI.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo)
+		if err != nil {
+			pipeWriter.CloseWithError(err)
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
 	}
 
 	pipeReader.Close()
@@ -554,7 +617,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	writeSuccessResponseXML(w, encodedSuccessResponse)
 
 	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -635,12 +698,12 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	// Extract metadata to be saved from incoming HTTP header.
-	metadata, err := extractMetadataFromHeader(ctx, r.Header)
+	metadata, err := extractMetadata(ctx, r)
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
+
 	if rAuthType == authTypeStreamingSigned {
 		if contentEncoding, ok := metadata["content-encoding"]; ok {
 			contentEncoding = trimAwsChunkedContentEncoding(contentEncoding)
@@ -725,7 +788,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 
 	if objectAPI.IsEncryptionSupported() {
 		if hasSSECustomerHeader(r.Header) && !hasSuffix(object, slashSeparator) { // handle SSE-C requests
-			reader, err = EncryptRequest(hashReader, r, metadata)
+			reader, err = EncryptRequest(hashReader, r, bucket, object, metadata)
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -742,6 +805,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	if api.CacheAPI() != nil && !hasSSECustomerHeader(r.Header) {
 		putObject = api.CacheAPI().PutObject
 	}
+
 	// Create the object..
 	objInfo, err := putObject(ctx, bucket, object, hashReader, metadata)
 	if err != nil {
@@ -760,7 +824,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseHeadersOnly(w)
 
 	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -824,7 +888,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
 			}
-			_, err = newEncryptMetadata(key, encMetadata)
+			_, err = newEncryptMetadata(key, bucket, object, encMetadata)
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -837,7 +901,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 	}
 
 	// Extract metadata that needs to be saved.
-	metadata, err := extractMetadataFromHeader(ctx, r.Header)
+	metadata, err := extractMetadata(ctx, r)
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
@@ -973,6 +1037,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 
 	var writer io.WriteCloser = pipeWriter
 	var reader io.Reader = pipeReader
+	var getLength = length
 	srcInfo.Reader, err = hash.NewReader(reader, length, "", "")
 	if err != nil {
 		pipeWriter.CloseWithError(err)
@@ -992,7 +1057,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			// Response writer should be limited early on for decryption upto required length,
 			// additionally also skipping mod(offset)64KiB boundaries.
 			writer = ioutil.LimitedWriter(writer, startOffset%(64*1024), length)
-			writer, startOffset, length, err = DecryptBlocksRequest(pipeWriter, r, startOffset, length, srcInfo, true)
+			writer, startOffset, getLength, err = DecryptBlocksRequest(writer, r, srcBucket, srcObject, startOffset, length, srcInfo, true)
 			if err != nil {
 				pipeWriter.CloseWithError(err)
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -1012,28 +1077,29 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 				return
 			}
 
-			// Calculating object encryption key
 			var objectEncryptionKey []byte
-			objectEncryptionKey, err = decryptObjectInfo(key, li.UserDefined)
+			objectEncryptionKey, err = decryptObjectInfo(key, dstBucket, dstObject, li.UserDefined)
 			if err != nil {
 				pipeWriter.CloseWithError(err)
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
 			}
 
-			reader, err = sio.EncryptReader(pipeReader, sio.Config{Key: objectEncryptionKey})
+			var partIDbin [4]byte
+			binary.LittleEndian.PutUint32(partIDbin[:], uint32(partID)) // marshal part ID
+
+			mac := hmac.New(sha256.New, objectEncryptionKey) // derive part encryption key from part ID and object key
+			mac.Write(partIDbin[:])
+			partEncryptionKey := mac.Sum(nil)
+			reader, err = sio.EncryptReader(reader, sio.Config{Key: partEncryptionKey})
 			if err != nil {
 				pipeWriter.CloseWithError(err)
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
 			}
 
-			size := length
-			if !sseCopyC {
-				info := ObjectInfo{Size: length}
-				size = info.EncryptedSize()
-			}
-
+			info := ObjectInfo{Size: length}
+			size := info.EncryptedSize()
 			srcInfo.Reader, err = hash.NewReader(reader, size, "", "")
 			if err != nil {
 				pipeWriter.CloseWithError(err)
@@ -1047,7 +1113,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	// Copy source object to destination, if source and destination
 	// object is same then only metadata is updated.
 	partInfo, err := objectAPI.CopyObjectPart(ctx, srcBucket, srcObject, dstBucket,
-		dstObject, uploadID, partID, startOffset, length, srcInfo)
+		dstObject, uploadID, partID, startOffset, getLength, srcInfo)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -1217,7 +1283,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 
 			// Calculating object encryption key
 			var objectEncryptionKey []byte
-			objectEncryptionKey, err = decryptObjectInfo(key, li.UserDefined)
+			objectEncryptionKey, err = decryptObjectInfo(key, bucket, object, li.UserDefined)
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -1431,7 +1497,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	writeSuccessResponseXML(w, encodedSuccessResponse)
 
 	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
 		host, port = "", ""
 	}
@@ -1477,6 +1543,27 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
+	if globalDNSConfig != nil {
+		_, err := globalDNSConfig.Get(bucket)
+		if err != nil {
+			if err == dns.ErrNoEntriesFound {
+				writeErrorResponse(w, ErrNoSuchBucket, r.URL)
+			} else {
+				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			}
+			return
+		}
+	} else {
+		getBucketInfo := objectAPI.GetBucketInfo
+		if api.CacheAPI() != nil {
+			getBucketInfo = api.CacheAPI().GetBucketInfo
+		}
+		if _, err := getBucketInfo(ctx, bucket); err != nil {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
+	}
+
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
 	// Ignore delete object errors while replying to client, since we are
 	// suppposed to reply only 204. Additionally log the error for

cmd/peer-rpc-client.go

@@ -166,9 +166,9 @@ func makeRemoteRPCClients(endpoints EndpointList) map[xnet.Host]*PeerRPCClient {
 	peerRPCClientMap := make(map[xnet.Host]*PeerRPCClient)
 	for _, hostStr := range GetRemotePeers(endpoints) {
 		host, err := xnet.ParseHost(hostStr)
-		logger.CriticalIf(context.Background(), err)
+		logger.FatalIf(err, "Unable to parse peer RPC Host", context.Background())
 		rpcClient, err := NewPeerRPCClient(host)
-		logger.CriticalIf(context.Background(), err)
+		logger.FatalIf(err, "Unable to parse peer RPC Client", context.Background())
 		peerRPCClientMap[*host] = rpcClient
 	}
 

cmd/peer-rpc-server.go

@@ -178,7 +178,7 @@ func (receiver *peerRPCReceiver) SetCredentials(args *SetCredentialsArgs, reply
 	prevCred := globalServerConfig.SetCredential(args.Credentials)
 
 	// Save credentials to config file
-	if err := globalServerConfig.Save(); err != nil {
+	if err := globalServerConfig.Save(getConfigFile()); err != nil {
 		// As saving configurstion failed, restore previous credential in memory.
 		globalServerConfig.SetCredential(prevCred)
 
@@ -201,7 +201,7 @@ func NewPeerRPCServer() (*xrpc.Server, error) {
 // registerPeerRPCRouter - creates and registers Peer RPC server and its router.
 func registerPeerRPCRouter(router *mux.Router) {
 	rpcServer, err := NewPeerRPCServer()
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to initialize peer RPC Server", context.Background())
 	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
 	subrouter.Path(peerServiceSubPath).Handler(rpcServer)
 }

cmd/policy.go

@@ -99,7 +99,7 @@ func (sys *PolicySys) refresh(objAPI ObjectLayer) error {
 	}
 	sys.removeDeletedBuckets(buckets)
 	for _, bucket := range buckets {
-		config, err := GetPolicyConfig(objAPI, bucket.Name)
+		config, err := objAPI.GetBucketPolicy(context.Background(), bucket.Name)
 		if err != nil {
 			if _, ok := err.(BucketPolicyNotFound); ok {
 				sys.Remove(bucket.Name)
@@ -187,8 +187,8 @@ func getConditionValues(request *http.Request, locationConstraint string) map[st
 	return args
 }
 
-// GetPolicyConfig - get policy config for given bucket name.
-func GetPolicyConfig(objAPI ObjectLayer, bucketName string) (*policy.Policy, error) {
+// getPolicyConfig - get policy config for given bucket name.
+func getPolicyConfig(objAPI ObjectLayer, bucketName string) (*policy.Policy, error) {
 	// Construct path to policy.json for the given bucket.
 	configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
 

cmd/posix.go

@@ -34,11 +34,12 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/mountinfo"
 )
 
 const (
-	diskMinFreeSpace  = 1 * humanize.GiByte // Min 1GiB free space.
-	diskMinTotalSpace = diskMinFreeSpace    // Min 1GiB total space.
+	diskMinFreeSpace  = 900 * humanize.MiByte // Min 900MiB free space.
+	diskMinTotalSpace = diskMinFreeSpace      // Min 900MiB total space.
 	maxAllowedIOError = 5
 )
 
@@ -60,15 +61,15 @@ func isValidVolname(volname string) bool {
 // posix - implements StorageAPI interface.
 type posix struct {
 	// Disk usage metrics
-	totalUsed uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
-	// Disk usage running routine
-	usageRunning int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
-	ioErrCount   int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
+	totalUsed  uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
+	ioErrCount int32  // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
 
 	diskPath  string
 	pool      sync.Pool
 	connected bool
 
+	diskMount bool // indicates if the path is an actual mount.
+
 	// Disk usage metrics
 	stopUsageCh chan struct{}
 }
@@ -135,10 +136,10 @@ func getValidPath(path string) (string, error) {
 	if err != nil {
 		return path, err
 	}
+	defer os.Remove(pathJoin(path, ".writable-check.tmp"))
 	file.Close()
 
-	err = os.Remove(pathJoin(path, ".writable-check.tmp"))
-	return path, err
+	return path, nil
 }
 
 // isDirEmpty - returns whether given directory is empty or not.
@@ -183,9 +184,12 @@ func newPosix(path string) (*posix, error) {
 			},
 		},
 		stopUsageCh: make(chan struct{}),
+		diskMount:   mountinfo.IsLikelyMountPoint(path),
 	}
 
-	go p.diskUsage(globalServiceDoneCh)
+	if !p.diskMount {
+		go p.diskUsage(globalServiceDoneCh)
+	}
 
 	// Success.
 	return p, nil
@@ -218,7 +222,7 @@ func checkDiskMinTotal(di disk.Info) (err error) {
 	// used for journalling, inodes etc.
 	totalDiskSpace := float64(di.Total) * 0.95
 	if int64(totalDiskSpace) <= diskMinTotalSpace {
-		return errDiskFull
+		return errMinDiskSize
 	}
 	return nil
 }
@@ -293,10 +297,14 @@ func (s *posix) DiskInfo() (info DiskInfo, err error) {
 	if err != nil {
 		return info, err
 	}
+	used := di.Total - di.Free
+	if !s.diskMount {
+		used = atomic.LoadUint64(&s.totalUsed)
+	}
 	return DiskInfo{
 		Total: di.Total,
 		Free:  di.Free,
-		Used:  atomic.LoadUint64(&s.totalUsed),
+		Used:  used,
 	}, nil
 
 }
@@ -336,7 +344,20 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
 	defer ticker.Stop()
 
 	usageFn := func(ctx context.Context, entry string) error {
+		if globalHTTPServer != nil {
+			// Wait at max 1 minute for an inprogress request
+			// before proceeding to count the usage.
+			waitCount := 60
+			// Any requests in progress, delay the usage.
+			for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
+				waitCount--
+				time.Sleep(1 * time.Second)
+			}
+		}
+
 		select {
+		case <-doneCh:
+			return errWalkAbort
 		case <-s.stopUsageCh:
 			return errWalkAbort
 		default:
@@ -349,14 +370,9 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
 		}
 	}
 
-	// Check if disk usage routine is running, if yes then return.
-	if atomic.LoadInt32(&s.usageRunning) == 1 {
-		return
-	}
-	atomic.StoreInt32(&s.usageRunning, 1)
-	defer atomic.StoreInt32(&s.usageRunning, 0)
-
-	if err := getDiskUsage(context.Background(), s.diskPath, usageFn); err != nil {
+	// Return this routine upon errWalkAbort, continue for any other error on purpose
+	// so that we can start the routine freshly in another 12 hours.
+	if err := getDiskUsage(context.Background(), s.diskPath, usageFn); err == errWalkAbort {
 		return
 	}
 
@@ -366,16 +382,20 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
 			return
 		case <-doneCh:
 			return
-		case <-ticker.C:
-			// Check if disk usage routine is running, if yes let it
-			// finish, before starting a new one.
-			if atomic.LoadInt32(&s.usageRunning) == 1 {
-				continue
-			}
-			atomic.StoreInt32(&s.usageRunning, 1)
-
+		case <-time.After(globalUsageCheckInterval):
 			var usage uint64
 			usageFn = func(ctx context.Context, entry string) error {
+				if globalHTTPServer != nil {
+					// Wait at max 1 minute for an inprogress request
+					// before proceeding to count the usage.
+					waitCount := 60
+					// Any requests in progress, delay the usage.
+					for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
+						waitCount--
+						time.Sleep(1 * time.Second)
+					}
+				}
+
 				select {
 				case <-s.stopUsageCh:
 					return errWalkAbort
@@ -390,11 +410,9 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
 			}
 
 			if err := getDiskUsage(context.Background(), s.diskPath, usageFn); err != nil {
-				atomic.StoreInt32(&s.usageRunning, 0)
 				continue
 			}
 
-			atomic.StoreInt32(&s.usageRunning, 0)
 			atomic.StoreUint64(&s.totalUsed, usage)
 		}
 	}

cmd/posix_test.go

@@ -470,7 +470,7 @@ func TestPosixMakeVol(t *testing.T) {
 	}
 }
 
-// TestPosixDeleteVol - Validates the expected behaviour of posix.DeleteVol for various cases.
+// TestPosixDeleteVol - Validates the expected behavior of posix.DeleteVol for various cases.
 func TestPosixDeleteVol(t *testing.T) {
 	// create posix test setup
 	posixStorage, path, err := newPosixTestSetup()
@@ -1948,7 +1948,7 @@ func TestCheckDiskTotalMin(t *testing.T) {
 				FSType: "XFS",
 				Files:  9999,
 			},
-			err: errDiskFull,
+			err: errMinDiskSize,
 		},
 	}
 

cmd/posix_windows_test.go

@@ -69,7 +69,7 @@ func TestUNCPaths(t *testing.T) {
 	}
 }
 
-// Test to validate posix behaviour on windows when a non-final path component is a file.
+// Test to validate posix behavior on windows when a non-final path component is a file.
 func TestUNCPathENOTDIR(t *testing.T) {
 	var err error
 	// Instantiate posix object to manage a disk

cmd/routers.go

@@ -49,6 +49,8 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
 var globalHandlers = []HandlerFunc{
 	// set HTTP security headers such as Content-Security-Policy.
 	addSecurityHeaders,
+	// Forward path style requests to actual host in a bucket federated setup.
+	setBucketForwardingHandler,
 	// Ratelimit the incoming requests using a token bucket algorithm
 	setRateLimitHandler,
 	// Validate all the incoming paths.

cmd/server-main.go

@@ -33,7 +33,7 @@ import (
 )
 
 func init() {
-	logger.Init(GOPATH)
+	logger.Init(GOPATH, GOROOT)
 	logger.RegisterUIError(fmtError)
 }
 
@@ -82,6 +82,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
   DOMAIN:
      MINIO_DOMAIN: To enable virtual-host-style requests, set this value to Minio host domain name.
@@ -89,6 +90,11 @@ ENVIRONMENT VARIABLES:
   WORM:
      MINIO_WORM: To turn on Write-Once-Read-Many in server, set this value to "on".
 
+  BUCKET-DNS:
+     MINIO_DOMAIN:    To enable bucket DNS requests, set this value to Minio host domain name.
+     MINIO_PUBLIC_IPS: To enable bucket DNS requests, set this value to list of Minio host public IP(s) delimited by ",".
+     MINIO_ETCD_ENDPOINTS: To enable bucket DNS requests, set this value to list of etcd endpoints delimited by ",".
+
 EXAMPLES:
   1. Start minio server on "/home/shared" directory.
      $ {{.HelpName}} /home/shared
@@ -113,6 +119,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}} /home/shared
 `,
 }
@@ -264,15 +271,6 @@ func serverMain(ctx *cli.Context) {
 		logger.Fatal(uiErrUnexpectedError(err), "Unable to configure one of server's RPC services")
 	}
 
-	// Create new notification system.
-	globalNotificationSys, err = NewNotificationSys(globalServerConfig, globalEndpoints)
-	if err != nil {
-		logger.Fatal(err, "Unable to initialize the notification system")
-	}
-
-	// Create new policy system.
-	globalPolicySys = NewPolicySys()
-
 	// Initialize Admin Peers inter-node communication only in distributed setup.
 	initGlobalAdminPeers(globalEndpoints)
 
@@ -281,7 +279,7 @@ func serverMain(ctx *cli.Context) {
 		getCert = globalTLSCerts.GetCertificate
 	}
 
-	globalHTTPServer = xhttp.NewServer([]string{globalMinioAddr}, handler, getCert)
+	globalHTTPServer = xhttp.NewServer([]string{globalMinioAddr}, criticalErrorHandler{handler}, getCert)
 	globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
 	globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes
 	go func() {
@@ -303,6 +301,30 @@ func serverMain(ctx *cli.Context) {
 	globalObjectAPI = newObject
 	globalObjLayerMutex.Unlock()
 
+	// Populate existing buckets to the etcd backend
+	if globalDNSConfig != nil {
+		initFederatorBackend(newObject)
+	}
+
+	// Re-enable logging
+	logger.Disable = false
+
+	// Create new policy system.
+	globalPolicySys = NewPolicySys()
+
+	// Initialize policy system.
+	if err := globalPolicySys.Init(newObjectLayerFn()); err != nil {
+		logger.Fatal(err, "Unable to initialize policy system")
+	}
+
+	// Create new notification system.
+	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
+
+	// Initialize notification system.
+	if err := globalNotificationSys.Init(newObjectLayerFn()); err != nil {
+		logger.Fatal(err, "Unable to initialize notification system")
+	}
+
 	// Prints the formatted startup message once object layer is initialized.
 	apiEndpoints := getAPIEndpoints(globalMinioAddr)
 	printStartupMessage(apiEndpoints)
@@ -310,9 +332,6 @@ func serverMain(ctx *cli.Context) {
 	// Set uptime time after object layer has initialized.
 	globalBootTime = UTCNow()
 
-	// Re-enable logging
-	logger.Disable = false
-
 	handleSignals()
 }
 

cmd/server_test.go

@@ -2420,7 +2420,7 @@ func (s *TestSuiteCommon) TestBucketMultipartList(c *check) {
 	// unmarshalling works from a client perspective, specifically
 	// while unmarshalling time.Time type for 'Initiated' field.
 	// time.Time does not honor xml marshaler, it means that we need
-	// to encode/format it before giving it to xml marshalling.
+	// to encode/format it before giving it to xml marshaling.
 
 	// This below check adds client side verification to see if its
 	// truly parseable.

cmd/signature-v4-utils.go

@@ -39,6 +39,9 @@ func skipContentSha256Cksum(r *http.Request) bool {
 
 	if isRequestPresignedSignatureV4(r) {
 		v, ok = r.URL.Query()["X-Amz-Content-Sha256"]
+		if !ok {
+			v, ok = r.Header["X-Amz-Content-Sha256"]
+		}
 	} else {
 		v, ok = r.Header["X-Amz-Content-Sha256"]
 	}
@@ -62,6 +65,9 @@ func getContentSha256Cksum(r *http.Request) string {
 		// will default to 'UNSIGNED-PAYLOAD'.
 		defaultSha256Cksum = unsignedPayload
 		v, ok = r.URL.Query()["X-Amz-Content-Sha256"]
+		if !ok {
+			v, ok = r.Header["X-Amz-Content-Sha256"]
+		}
 	} else {
 		// X-Amz-Content-Sha256, if not set in signed requests, checksum
 		// will default to sha256([]byte("")).

cmd/signature-v4.go

@@ -249,6 +249,12 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 
 	// Save other headers available in the request parameters.
 	for k, v := range req.URL.Query() {
+
+		// Handle the metadata in presigned put query string
+		if strings.Contains(strings.ToLower(k), "x-amz-meta-") {
+			query.Set(k, v[0])
+		}
+
 		if strings.HasPrefix(strings.ToLower(k), "x-amz") {
 			continue
 		}

cmd/storage-errors.go

@@ -76,6 +76,9 @@ var errBitrotHashAlgoInvalid = errors.New("bit-rot hash algorithm is invalid")
 // errCrossDeviceLink - rename across devices not allowed.
 var errCrossDeviceLink = errors.New("Rename across devices not allowed, please fix your backend configuration")
 
+// errMinDiskSize - cannot create volume or files when disk size is less than threshold.
+var errMinDiskSize = errors.New("The disk size is less than the minimum threshold")
+
 // hashMisMatchError - represents a bit-rot hash verification failure
 // error.
 type hashMismatchError struct {

cmd/storage-rpc-client.go

@@ -316,9 +316,9 @@ func NewStorageRPCClient(host *xnet.Host, endpointPath string) (*StorageRPCClien
 // Initialize new storage rpc client.
 func newStorageRPC(endpoint Endpoint) *StorageRPCClient {
 	host, err := xnet.ParseHost(endpoint.Host)
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to parse storage RPC Host", context.Background())
 	rpcClient, err := NewStorageRPCClient(host, endpoint.Path)
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to initialize storage RPC client", context.Background())
 	rpcClient.connected = rpcClient.Call(storageServiceName+".Connect", &AuthArgs{}, &VoidReply{}) == nil
 	return rpcClient
 }

cmd/test-utils_test.go

@@ -353,10 +353,8 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
 	globalMinioHost = host
 	globalMinioPort = port
 	globalMinioAddr = getEndpointsLocalAddr(testServer.Disks)
-	globalNotificationSys, err = NewNotificationSys(globalServerConfig, testServer.Disks)
-	if err != nil {
-		t.Fatalf("Unable to create new notification system. %v", err)
-	}
+
+	globalNotificationSys = NewNotificationSys(globalServerConfig, testServer.Disks)
 
 	// Create new policy system.
 	globalPolicySys = NewPolicySys()
@@ -605,7 +603,7 @@ func newTestConfig(bucketLocation string) (rootPath string, err error) {
 	globalServerConfig.SetRegion(bucketLocation)
 
 	// Save config.
-	if err = globalServerConfig.Save(); err != nil {
+	if err = globalServerConfig.Save(getConfigFile()); err != nil {
 		return "", err
 	}
 
@@ -1720,9 +1718,7 @@ func newTestObjectLayer(endpoints EndpointList) (newObject ObjectLayer, err erro
 	}
 
 	// Create new notification system.
-	if globalNotificationSys, err = NewNotificationSys(globalServerConfig, endpoints); err != nil {
-		return nil, err
-	}
+	globalNotificationSys = NewNotificationSys(globalServerConfig, endpoints)
 
 	// Create new policy system.
 	globalPolicySys = NewPolicySys()

cmd/typed-errors.go

@@ -67,3 +67,6 @@ var errNotFirstDisk = errors.New("Not first disk")
 
 // error returned by first disk waiting to initialize other servers.
 var errFirstDiskWait = errors.New("Waiting on other disks")
+
+// error returned when a bucket already exists
+var errBucketAlreadyExists = errors.New("Your previous request to create the named bucket succeeded and you already own it")

cmd/ui-errors.go

@@ -53,6 +53,12 @@ var (
 		"MINIO_CACHE_EXPIRY: Valid cache expiry duration is in days.",
 	)
 
+	uiErrInvalidCacheMaxUse = newUIErrFn(
+		"Invalid cache max-use value",
+		"Please check the passed value",
+		"MINIO_CACHE_MAXUSE: Valid cache max-use value between 0-100.",
+	)
+
 	uiErrInvalidCredentials = newUIErrFn(
 		"Invalid credentials",
 		"Please provide correct credentials",

cmd/update-main.go

@@ -458,7 +458,7 @@ func getUpdateInfo(timeout time.Duration, mode string) (updateMsg string, sha256
 
 func doUpdate(sha256Hex string, latestReleaseTime time.Time, ok bool) (successMsg string, err error) {
 	if !ok {
-		successMsg = greenColorSprintf("Minio update to version RELEASE.%s cancelled.",
+		successMsg = greenColorSprintf("Minio update to version RELEASE.%s canceled.",
 			latestReleaseTime.Format(minioReleaseTagTimeLayout))
 		return successMsg, nil
 	}

cmd/utils.go

@@ -34,6 +34,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/handlers"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
@@ -141,6 +142,10 @@ const (
 	// Maximum Part ID for multipart upload is 10000
 	// (Acceptable values range from 1 to 10000 inclusive)
 	globalMaxPartID = 10000
+
+	// Default values used while communicating with the cloud backends
+	defaultDialTimeout   = 30 * time.Second
+	defaultDialKeepAlive = 30 * time.Second
 )
 
 // isMaxObjectSize - verify if max object size
@@ -258,12 +263,12 @@ func ToS3ETag(etag string) string {
 // used while communicating with the cloud backends.
 // This sets the value for MaxIdleConnsPerHost from 2 (go default)
 // to 100.
-func NewCustomHTTPTransport() http.RoundTripper {
+func NewCustomHTTPTransport() *http.Transport {
 	return &http.Transport{
 		Proxy: http.ProxyFromEnvironment,
 		DialContext: (&net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: 30 * time.Second,
+			Timeout:   defaultDialTimeout,
+			KeepAlive: defaultDialKeepAlive,
 		}).DialContext,
 		MaxIdleConns:          1024,
 		MaxIdleConnsPerHost:   1024,
@@ -335,7 +340,13 @@ func newContext(r *http.Request, api string) context.Context {
 	if prefix != "" {
 		object = prefix
 	}
-	reqInfo := &logger.ReqInfo{RemoteHost: r.RemoteAddr, UserAgent: r.Header.Get("user-agent"), API: api, BucketName: bucket, ObjectName: object}
+	reqInfo := &logger.ReqInfo{
+		RemoteHost: handlers.GetSourceIP(r),
+		UserAgent:  r.Header.Get("user-agent"),
+		API:        api,
+		BucketName: bucket,
+		ObjectName: object,
+	}
 	return logger.SetReqInfo(context.Background(), reqInfo)
 }
 

cmd/web-handlers.go

@@ -39,6 +39,7 @@ import (
 	"github.com/minio/minio/browser"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/policy"
@@ -135,6 +136,26 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep
 		return toJSONError(errInvalidBucketName)
 	}
 
+	if globalDNSConfig != nil {
+		if _, err := globalDNSConfig.Get(args.BucketName); err != nil {
+			if err == dns.ErrNoEntriesFound {
+				// Proceed to creating a bucket.
+				if err = objectAPI.MakeBucketWithLocation(context.Background(), args.BucketName, globalServerConfig.GetRegion()); err != nil {
+					return toJSONError(err)
+				}
+				if err = globalDNSConfig.Put(args.BucketName); err != nil {
+					objectAPI.DeleteBucket(context.Background(), args.BucketName)
+					return toJSONError(err)
+				}
+
+				reply.UIVersion = browser.UIVersion
+				return nil
+			}
+			return toJSONError(err)
+		}
+		return toJSONError(errBucketAlreadyExists)
+	}
+
 	if err := objectAPI.MakeBucketWithLocation(context.Background(), args.BucketName, globalServerConfig.GetRegion()); err != nil {
 		return toJSONError(err, args.BucketName)
 	}
@@ -171,9 +192,14 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs,
 
 	globalNotificationSys.RemoveNotification(args.BucketName)
 	globalPolicySys.Remove(args.BucketName)
-	for nerr := range globalNotificationSys.DeleteBucket(args.BucketName) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
+	globalNotificationSys.DeleteBucket(ctx, args.BucketName)
+
+	if globalDNSConfig != nil {
+		if err := globalDNSConfig.Delete(args.BucketName); err != nil {
+			// Deleting DNS entry failed, attempt to create the bucket again.
+			objectAPI.MakeBucketWithLocation(ctx, args.BucketName, "")
+			return toJSONError(err)
+		}
 	}
 
 	reply.UIVersion = browser.UIVersion
@@ -208,16 +234,32 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re
 	if authErr != nil {
 		return toJSONError(authErr)
 	}
-	buckets, err := listBuckets(context.Background())
-	if err != nil {
-		return toJSONError(err)
-	}
-	for _, bucket := range buckets {
-		reply.Buckets = append(reply.Buckets, WebBucketInfo{
-			Name:         bucket.Name,
-			CreationDate: bucket.Created,
-		})
+	// If etcd, dns federation configured list buckets from etcd.
+	if globalDNSConfig != nil {
+		dnsBuckets, err := globalDNSConfig.List()
+		if err != nil {
+			return toJSONError(err)
+		}
+		for _, dnsRecord := range dnsBuckets {
+			bucketName := strings.Trim(dnsRecord.Key, "/")
+			reply.Buckets = append(reply.Buckets, WebBucketInfo{
+				Name:         bucketName,
+				CreationDate: dnsRecord.CreationDate,
+			})
+		}
+	} else {
+		buckets, err := listBuckets(context.Background())
+		if err != nil {
+			return toJSONError(err)
+		}
+		for _, bucket := range buckets {
+			reply.Buckets = append(reply.Buckets, WebBucketInfo{
+				Name:         bucket.Name,
+				CreationDate: bucket.Created,
+			})
+		}
 	}
+
 	reply.UIVersion = browser.UIVersion
 	return nil
 }
@@ -412,11 +454,6 @@ type LoginRep struct {
 func (web *webAPIHandlers) Login(r *http.Request, args *LoginArgs, reply *LoginRep) error {
 	token, err := authenticateWeb(args.Username, args.Password)
 	if err != nil {
-		// Make sure to log errors related to browser login,
-		// for security and auditing reasons.
-		reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", r.RemoteAddr)
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogIf(ctx, err)
 		return toJSONError(err)
 	}
 
@@ -437,7 +474,9 @@ func (web webAPIHandlers) GenerateAuth(r *http.Request, args *WebGenericArgs, re
 		return toJSONError(errAuthentication)
 	}
 	cred, err := auth.GetNewCredentials()
-	logger.CriticalIf(context.Background(), err)
+	if err != nil {
+		return toJSONError(err)
+	}
 	reply.AccessKey = cred.AccessKey
 	reply.SecretKey = cred.SecretKey
 	reply.UIVersion = browser.UIVersion
@@ -480,9 +519,9 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se
 	// Update credentials in memory
 	prevCred := globalServerConfig.SetCredential(creds)
 
-	// Save credentials to config file
-	if err := globalServerConfig.Save(); err != nil {
-		// As saving configurstion failed, restore previous credential in memory.
+	// Persist updated credentials.
+	if err = globalServerConfig.Save(getConfigFile()); err != nil {
+		// Save the current creds when failed to update.
 		globalServerConfig.SetCredential(prevCred)
 		logger.LogIf(context.Background(), err)
 		return toJSONError(err)
@@ -589,7 +628,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Extract incoming metadata if any.
-	metadata, err := extractMetadataFromHeader(context.Background(), r.Header)
+	metadata, err := extractMetadata(context.Background(), r)
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
@@ -936,10 +975,7 @@ func (web *webAPIHandlers) SetBucketPolicy(r *http.Request, args *SetBucketPolic
 	}
 
 	globalPolicySys.Set(args.BucketName, *bucketPolicy)
-	for nerr := range globalNotificationSys.SetBucketPolicy(args.BucketName, bucketPolicy) {
-		logger.GetReqInfo(ctx).AppendTags("remotePeer", nerr.Host.Name)
-		logger.LogIf(ctx, nerr.Err)
-	}
+	globalNotificationSys.SetBucketPolicy(ctx, args.BucketName, bucketPolicy)
 
 	return nil
 }

cmd/xl-sets.go

@@ -263,16 +263,6 @@ func newXLSets(endpoints EndpointList, format *formatXLV3, setCount int, drivesP
 	// Connect disks right away.
 	s.connectDisks()
 
-	// Initialize notification system.
-	if err := globalNotificationSys.Init(s); err != nil {
-		return nil, fmt.Errorf("Unable to initialize notification system. %v", err)
-	}
-
-	// Initialize policy system.
-	if err := globalPolicySys.Init(s); err != nil {
-		return nil, fmt.Errorf("Unable to initialize policy system. %v", err)
-	}
-
 	// Start the disk monitoring and connect routine.
 	go s.monitorAndConnectEndpoints(defaultMonitorConnectEndpointInterval)
 
@@ -454,7 +444,12 @@ func (s *xlSets) GetBucketInfo(ctx context.Context, bucket string) (bucketInfo B
 
 // ListObjectsV2 lists all objects in bucket filtered by prefix
 func (s *xlSets) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
-	loi, err := s.ListObjects(ctx, bucket, prefix, continuationToken, delimiter, maxKeys)
+	marker := continuationToken
+	if marker == "" {
+		marker = startAfter
+	}
+
+	loi, err := s.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
 		return result, err
 	}
@@ -476,7 +471,7 @@ func (s *xlSets) SetBucketPolicy(ctx context.Context, bucket string, policy *pol
 
 // GetBucketPolicy will return a policy on a bucket
 func (s *xlSets) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
-	return GetPolicyConfig(s, bucket)
+	return getPolicyConfig(s, bucket)
 }
 
 // DeleteBucketPolicy deletes all policies on bucket
@@ -614,7 +609,7 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke
 			}
 			return
 		}
-		// Close writer explicitly signalling we wrote all data.
+		// Close writer explicitly signaling we wrote all data.
 		if gerr := srcInfo.Writer.Close(); gerr != nil {
 			logger.LogIf(ctx, gerr)
 			return

cmd/xl-v1-bucket.go

@@ -284,7 +284,7 @@ func (xl xlObjects) SetBucketPolicy(ctx context.Context, bucket string, policy *
 
 // GetBucketPolicy will get policy on bucket
 func (xl xlObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
-	return GetPolicyConfig(xl, bucket)
+	return getPolicyConfig(xl, bucket)
 }
 
 // DeleteBucketPolicy deletes all policies on bucket

cmd/xl-v1-metadata.go

@@ -463,7 +463,7 @@ func writeXLMetadata(ctx context.Context, disk StorageAPI, bucket, prefix string
 		logger.LogIf(ctx, err)
 		return err
 	}
-	// Persist marshalled data.
+	// Persist marshaled data.
 	err = disk.AppendFile(bucket, jsonFile, metadataBytes)
 	logger.LogIf(ctx, err)
 	return err

cmd/xl-v1-multipart.go

@@ -277,7 +277,7 @@ func (xl xlObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, ds
 			}
 			return
 		}
-		// Close writer explicitly signalling we wrote all data.
+		// Close writer explicitly signaling we wrote all data.
 		if gerr := srcInfo.Writer.Close(); gerr != nil {
 			logger.LogIf(ctx, gerr)
 			return
@@ -375,8 +375,17 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	}
 
 	// Fetch buffer for I/O, returns from the pool if not allocates a new one and returns.
-	buffer := xl.bp.Get()
-	defer xl.bp.Put(buffer)
+	var buffer []byte
+	switch size := data.Size(); {
+	case size == 0:
+		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
+	case size < blockSizeV1:
+		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
+		buffer = make([]byte, size, 2*size)
+	default:
+		buffer = xl.bp.Get()
+		defer xl.bp.Put(buffer)
+	}
 
 	file, err := storage.CreateFile(ctx, data, minioMetaTmpBucket, tmpPartPath, buffer, DefaultBitrotAlgorithm, writeQuorum)
 	if err != nil {
@@ -473,7 +482,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 // to indicate where the listing should begin from.
 //
 // Implements S3 compatible ListObjectParts API. The resulting
-// ListPartsInfo structure is marshalled directly into XML and
+// ListPartsInfo structure is marshaled directly into XML and
 // replied back to the client.
 func (xl xlObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int) (result ListPartsInfo, e error) {
 	if err := checkListPartsArgs(ctx, bucket, object, xl); err != nil {

cmd/xl-v1-object.go

@@ -142,7 +142,7 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc
 			pipeWriter.CloseWithError(toObjectErr(gerr, srcBucket, srcObject))
 			return
 		}
-		pipeWriter.Close() // Close writer explicitly signalling we wrote all data.
+		pipeWriter.Close() // Close writer explicitly signaling we wrote all data.
 	}()
 
 	hashReader, err := hash.NewReader(pipeReader, length, "", "")
@@ -611,8 +611,17 @@ func (xl xlObjects) putObject(ctx context.Context, bucket string, object string,
 	}
 
 	// Fetch buffer for I/O, returns from the pool if not allocates a new one and returns.
-	buffer := xl.bp.Get()
-	defer xl.bp.Put(buffer)
+	var buffer []byte
+	switch size := data.Size(); {
+	case size == 0:
+		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
+	case size < blockSizeV1:
+		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
+		buffer = make([]byte, size, 2*size)
+	default:
+		buffer = xl.bp.Get()
+		defer xl.bp.Put(buffer)
+	}
 
 	// Read data and split into parts - similar to multipart mechanism
 	for partIdx := 1; ; partIdx++ {
@@ -846,7 +855,12 @@ func (xl xlObjects) DeleteObject(ctx context.Context, bucket, object string) (er
 
 // ListObjectsV2 lists all blobs in bucket filtered by prefix
 func (xl xlObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
-	loi, err := xl.ListObjects(ctx, bucket, prefix, continuationToken, delimiter, maxKeys)
+	marker := continuationToken
+	if marker == "" {
+		marker = startAfter
+	}
+
+	loi, err := xl.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
 		return result, err
 	}

dockerscripts/healthcheck.sh

@@ -48,7 +48,7 @@ healthcheck_main () {
             http_response=$(curl -s -k -o /dev/null -w "%{http_code}" ${scheme}${address}${resource})
         fi
 
-        # If http_repsonse is 200 - server is up.
+        # If http_response is 200 - server is up.
         [ "$http_response" = "200" ]
     fi
 }

docs/bucket/notifications/README.md

@@ -20,7 +20,7 @@ Bucket events can be published to the following targets:
 
 ## Prerequisites
 
-* Install and configure Minio Server from [here](http://docs.minio.io/docs/minio-quickstart-guide).
+* Install and configure Minio Server from [here](https://docs.minio.io/docs/minio-quickstart-guide).
 * Install and configure Minio Client from [here](https://docs.minio.io/docs/minio-client-quickstart-guide).
 
 <a name="AMQP"></a>

docs/config/README.md

@@ -10,7 +10,7 @@ minio server --config-dir /etc/minio /data
 ```
 
 ### Certificate Directory
-TLS certificates are stored under ``${HOME}/.minio/certs`` directory. You need to place certificates here to enable `HTTPS` based access. Read more about [How to secure access to Minio server with TLS](http://docs.minio.io/docs/how-to-secure-access-to-minio-server-with-tls).
+TLS certificates are stored under ``${HOME}/.minio/certs`` directory. You need to place certificates here to enable `HTTPS` based access. Read more about [How to secure access to Minio server with TLS](https://docs.minio.io/docs/how-to-secure-access-to-minio-server-with-tls).
 
 Following is the directory structure for Minio server with TLS certificates.
 
@@ -110,20 +110,21 @@ By default, parity for objects with standard storage class is set to `N/2`, and
 |``drives``| _[]string_ | List of mounted file system drives with [`atime`](http://kerolasa.github.io/filetimes.html) support enabled|
 |``exclude`` | _[]string_ | List of wildcard patterns for prefixes to exclude from cache |
 |``expiry`` | _int_ | Days to cache expiry |
+|``maxuse`` | _int_ | Percentage of disk available to cache |
 
 #### Notify
 |Field|Type|Description|
 |:---|:---|:---|
 |``notify``| |Notify enables bucket notification events for lambda computing via the following targets.|
-|``notify.amqp``| |[Configure to publish Minio events via AMQP target.](http://docs.minio.io/docs/minio-bucket-notification-guide#AMQP)|
-|``notify.nats``| |[Configure to publish Minio events via NATS target.](http://docs.minio.io/docs/minio-bucket-notification-guide#NATS)|
-|``notify.elasticsearch``| |[Configure to publish Minio events via Elasticsearch target.](http://docs.minio.io/docs/minio-bucket-notification-guide#Elasticsearch)|
-|``notify.redis``| |[Configure to publish Minio events via Redis target.](http://docs.minio.io/docs/minio-bucket-notification-guide#Redis)|
-|``notify.postgresql``| |[Configure to publish Minio events via PostgreSQL target.](http://docs.minio.io/docs/minio-bucket-notification-guide#PostgreSQL)|
-|``notify.kafka``| |[Configure to publish Minio events via Apache Kafka target.](http://docs.minio.io/docs/minio-bucket-notification-guide#apache-kafka)|
-|``notify.webhook``| |[Configure to publish Minio events via Webhooks target.](http://docs.minio.io/docs/minio-bucket-notification-guide#webhooks)|
+|``notify.amqp``| |[Configure to publish Minio events via AMQP target.](https://docs.minio.io/docs/minio-bucket-notification-guide#AMQP)|
+|``notify.nats``| |[Configure to publish Minio events via NATS target.](https://docs.minio.io/docs/minio-bucket-notification-guide#NATS)|
+|``notify.elasticsearch``| |[Configure to publish Minio events via Elasticsearch target.](https://docs.minio.io/docs/minio-bucket-notification-guide#Elasticsearch)|
+|``notify.redis``| |[Configure to publish Minio events via Redis target.](https://docs.minio.io/docs/minio-bucket-notification-guide#Redis)|
+|``notify.postgresql``| |[Configure to publish Minio events via PostgreSQL target.](https://docs.minio.io/docs/minio-bucket-notification-guide#PostgreSQL)|
+|``notify.kafka``| |[Configure to publish Minio events via Apache Kafka target.](https://docs.minio.io/docs/minio-bucket-notification-guide#apache-kafka)|
+|``notify.webhook``| |[Configure to publish Minio events via Webhooks target.](https://docs.minio.io/docs/minio-bucket-notification-guide#webhooks)|
 |``notify.mysql``| |[Configure to publish Minio events via MySql target.](https://docs.minio.io/docs/minio-bucket-notification-guide#MySQL)|
-|``notify.mqtt``| |[Configure to publish Minio events via MQTT target.](http://docs.minio.io/docs/minio-bucket-notification-guide#MQTT)|
+|``notify.mqtt``| |[Configure to publish Minio events via MQTT target.](https://docs.minio.io/docs/minio-bucket-notification-guide#MQTT)|
 
 ## Explore Further
 * [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide)

docs/config/config.sample.json

@@ -1,11 +1,12 @@
 {
-    "version": "24",
+    "version": "26",
     "credential": {
         "accessKey": "USWUXHGYZQYFYFFIT3RE",
         "secretKey": "MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03"
     },
     "region": "us-east-1",
     "browser": "on",
+    "worm": "off",
     "domain": "",
     "storageclass": {
         "standard": "",
@@ -14,11 +15,12 @@
     "cache": {
         "drives": [],
         "expiry": 90,
-        "exclude": []
+        "exclude": [],
+        "maxuse": 80
     },
     "usage": {
         "interval": "3h"
-    }
+    },
     "notify": {
         "amqp": {
             "1": {
@@ -124,3 +126,4 @@
         }
     }
 }
+

docs/deployment/kernel-tuning/disk-tuning.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-## Minio Cloud Storage, (C) 2017 Minio, Inc.
+## Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
 ##
 ## Licensed under the Apache License, Version 2.0 (the "License");
 ## you may not use this file except in compliance with the License.
@@ -16,9 +16,9 @@
 
 # This script changes protected files, and must be run as root
 
-for i in $(ls -d /sys/block/*/queue/iosched 2>/dev/null); do
-    iosched_dir=$(echo $i | awk '/iosched/ {print $1}')
-    [ -z $iosched_dir ] && {
+for i in $(echo /sys/block/*/queue/iosched 2>/dev/null); do
+    iosched_dir=$(echo "${i}" | awk '/iosched/ {print $1}')
+    [ -z "${iosched_dir}" ] && {
         continue
     }
     ## Change each disk ioscheduler to be "deadline"
@@ -29,22 +29,22 @@ for i in $(ls -d /sys/block/*/queue/iosched 2>/dev/null); do
     ## see whether write requests have been starved for too
     ## long, and then decides whether to start a new batch
     ## of reads or writes
-    path=$(dirname $iosched_dir)
-    [ -f $path/scheduler ] && {
-        echo "deadline" > $path/scheduler
+    path=$(dirname "${iosched_dir}")
+    [ -f "${path}/scheduler" ] && {
+        echo "deadline" > "${path}/scheduler" 2>/dev/null || true
     }
     ## This controls how many requests may be allocated
     ## in the block layer for read or write requests.
     ## Note that the total allocated number may be twice
     ## this amount, since it applies only to reads or
     ## writes (not the accumulate sum).
-    [ -f $path/nr_requests ] && {
-        echo "256" > $path/nr_requests
+    [ -f "${path}/nr_requests" ] && {
+        echo "256" > "${path}/nr_requests" 2>/dev/null || true
     }
     ## This is the maximum number of kilobytes
     ## supported in a single data transfer at
     ## block layer.
-    [ -f $path/max_sectors_kb ] && {
-        echo "1024" > $path/max_sectors_kb || true
+    [ -f "${path}/max_sectors_kb" ] && {
+        echo "1024" > "${path}/max_sectors_kb" 2>/dev/null || true
     }
 done