docs: fix all Chinese doc links for the new docs site (#6097)

Additionally fix typos, default to US locale words

Makefile

@@ -50,9 +50,11 @@ deadcode:
 	@${GOPATH}/bin/deadcode -test $(shell go list ./...) || true
 
 spelling:
-	@${GOPATH}/bin/misspell -error `find cmd/`
-	@${GOPATH}/bin/misspell -error `find pkg/`
-	@${GOPATH}/bin/misspell -error `find docs/`
+	@${GOPATH}/bin/misspell -locale US -error `find cmd/`
+	@${GOPATH}/bin/misspell -locale US -error `find pkg/`
+	@${GOPATH}/bin/misspell -locale US -error `find docs/`
+	@${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
+	@${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
 
 # Builds minio, runs the verifiers then runs the tests.
 check: test
@@ -89,7 +91,7 @@ pkg-list:
 # Builds minio and installs it to $GOPATH/bin.
 install: build
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
-	@cp $(PWD)/minio $(GOPATH)/bin/minio
+	@mkdir -p $(GOPATH)/bin && cp $(PWD)/minio $(GOPATH)/bin/minio
 	@echo "Installation successful. To learn more, try \"minio --help\"."
 
 clean:

browser/app/js/browser/Login.js

@@ -25,14 +25,35 @@ import web from "../web"
 import { Redirect } from "react-router-dom"
 
 export class Login extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      accessKey: "",
+      secretKey: ""
+    }
+  }
+
+  // Handle field changes
+  accessKeyChange(e) {
+    this.setState({
+      accessKey: e.target.value
+    })
+  }
+
+  secretKeyChange(e) {
+    this.setState({
+      secretKey: e.target.value
+    })
+  }
+
   handleSubmit(event) {
     event.preventDefault()
     const { showAlert, history } = this.props
     let message = ""
-    if (!document.getElementById("accessKey").value) {
+    if (this.state.accessKey === "") {
       message = "Access Key cannot be empty"
     }
-    if (!document.getElementById("secretKey").value) {
+    if (this.state.secretKey === "") {
       message = "Secret Key cannot be empty"
     }
     if (message) {
@@ -41,8 +62,8 @@ export class Login extends React.Component {
     }
     web
       .Login({
-        username: document.getElementById("accessKey").value,
-        password: document.getElementById("secretKey").value
+        username: this.state.accessKey,
+        password: this.state.secretKey
       })
       .then(res => {
         history.push("/")
@@ -77,6 +98,8 @@ export class Login extends React.Component {
         <div className="l-wrap">
           <form onSubmit={this.handleSubmit.bind(this)}>
             <InputGroup
+              value={this.state.accessKey}
+              onChange={this.accessKeyChange.bind(this)}
               className="ig-dark"
               label="Access Key"
               id="accessKey"
@@ -87,6 +110,8 @@ export class Login extends React.Component {
               autoComplete="username"
             />
             <InputGroup
+              value={this.state.secretKey}
+              onChange={this.secretKeyChange.bind(this)}
               className="ig-dark"
               label="Secret Key"
               id="secretKey"

browser/app/js/browser/__tests__/Login.test.js

@@ -60,20 +60,25 @@ describe("Login", () => {
         alert={{ show: false, type: "danger"}}
         showAlert={showAlertMock}
         clearAlert={clearAlertMock}
-      />,
-      { attachTo: document.body }
+      />
     )
     // case where both keys are empty - displays the second warning
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Secret Key cannot be empty")
 
     // case where access key is empty
-    document.getElementById("secretKey").value = "secretKey"
+    wrapper.setState({
+      accessKey: "",
+      secretKey: "secretKey"
+    })
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Access Key cannot be empty")
 
     // case where secret key is empty
-    document.getElementById("accessKey").value = "accessKey"
+    wrapper.setState({
+      accessKey: "accessKey",
+      secretKey: ""
+    })
     wrapper.find("form").simulate("submit")
     expect(showAlertMock).toHaveBeenCalledWith("danger", "Secret Key cannot be empty")
   })
@@ -85,11 +90,12 @@ describe("Login", () => {
         alert={{ show: false, type: "danger"}}
         showAlert={showAlertMock}
         clearAlert={clearAlertMock}
-      />,
-      { attachTo: document.body }
+      />
     )
-    document.getElementById("accessKey").value = "accessKey"
-    document.getElementById("secretKey").value = "secretKey"
+    wrapper.setState({
+      accessKey: "accessKey",
+      secretKey: "secretKey"
+    })
     wrapper.find("form").simulate("submit")
     expect(web.Login).toHaveBeenCalledWith({
       "username": "accessKey", 

browser/app/js/objects/ObjectItem.js

@@ -15,7 +15,6 @@
  */
 
 import React from "react"
-import classNames from "classnames"
 import { connect } from "react-redux"
 import humanize from "humanize"
 import Moment from "moment"
@@ -35,22 +34,19 @@ export const ObjectItem = ({
   onClick
 }) => {
   return (
-    <div 
-      className={
-        "fesl-row " + 
-        classNames({
-          "fesl-row-selected": checked
-        })
-      }
-      data-type={getDataType(name, contentType)}
-    >
+    <div className={"fesl-row"} data-type={getDataType(name, contentType)}>
       <div className="fesl-item fesl-item-icon">
         <div className="fi-select">
           <input
             type="checkbox"
             name={name}
+            checked={checked}
+            onChange={() => {
+              checked ? uncheckObject(name) : checkObject(name)
+            }}
           />
           <i className="fis-icon" />
+          <i className="fis-helper" />
         </div>
       </div>
       <div className="fesl-item fesl-item-name">
@@ -58,7 +54,9 @@ export const ObjectItem = ({
           href="#"
           onClick={e => {
             e.preventDefault()
-            checked ? uncheckObject(name) : checkObject(name)
+            if (onClick) {
+              onClick()
+            }
           }}
         >
           {name}

browser/app/js/objects/ObjectsBulkActions.js

@@ -29,10 +29,10 @@ export class ObjectsBulkActions extends React.Component {
     }
   }
   handleDownload() {
-    const { checkedObjects, resetCheckedList, downloadChecked, downloadObject } = this.props
-    if (checkedObjects.length === 1) {
+    const { checkedObjects, clearChecked, downloadChecked, downloadObject } = this.props
+    if (checkedObjects.length === 1 && !checkedObjects[0].endsWith('/')) {
       downloadObject(checkedObjects[0])
-      resetCheckedList()
+      clearChecked()
     } else {
       downloadChecked()
     }
@@ -59,17 +59,16 @@ export class ObjectsBulkActions extends React.Component {
         }
       >
         <span className="la-label">
-          <i className="fa fa-check-circle" /> {checkedObjects.length} 
+          <i className="fa fa-check-circle" /> {checkedObjects.length}
           {checkedObjects.length === 1 ? " Object " : " Objects "}
           selected
         </span>
         <span className="la-actions pull-right">
-          <button 
-            id="download-checked" 
-            onClick={this.handleDownload.bind(this)}
-          >
+          <button id="download-checked" onClick={this.handleDownload.bind(this)}>
             {" "}
-            Download {checkedObjects.length === 1 ? "object" : "all as zip"}{" "}
+            Download
+            {(checkedObjects.length === 1 && !checkedObjects[0].endsWith('/')) ? 
+            " object" : " all as zip" }{" "}
           </button>
         </span>
         <span className="la-actions pull-right">
@@ -105,6 +104,7 @@ const mapStateToProps = state => {
 
 const mapDispatchToProps = dispatch => {
   return {
+    downloadObject: object => dispatch(actions.downloadObject(object)),
     downloadChecked: () => dispatch(actions.downloadCheckedObjects()),
     downloadObject: object => dispatch(actions.downloadObject(object)),
     resetCheckedList: () => dispatch(actions.resetCheckedList()),

browser/app/js/objects/__tests__/ObjectItem.test.js

@@ -28,22 +28,40 @@ describe("ObjectItem", () => {
     expect(wrapper.prop("data-type")).toBe("image")
   })
 
-  it("should call checkObject when the object is selected", () => {
-    const checkObject = jest.fn()
-    const wrapper = shallow(<ObjectItem name={"test"} checked={false} checkObject={checkObject} />)
+  it("shouldn't call onClick when the object isclicked", () => {
+    const onClick = jest.fn()
+    const wrapper = shallow(<ObjectItem name={"test"} />)
     wrapper.find("a").simulate("click", { preventDefault: jest.fn() })
+    expect(onClick).not.toHaveBeenCalled()
+  })
+
+  it("should call onClick when the folder isclicked", () => {
+    const onClick = jest.fn()
+    const wrapper = shallow(<ObjectItem name={"test/"} onClick={onClick} />)
+    wrapper.find("a").simulate("click", { preventDefault: jest.fn() })
+    expect(onClick).toHaveBeenCalled()
+  })
+
+  it("should call checkObject when the object/prefix is checked", () => {
+    const checkObject = jest.fn()
+    const wrapper = shallow(
+      <ObjectItem name={"test"} checked={false} checkObject={checkObject} />
+    )
+    wrapper.find("input[type='checkbox']").simulate("change")
     expect(checkObject).toHaveBeenCalledWith("test")
   })
 
-  it("should render highlighted row when object is selected", () => {
+  it("should render checked checkbox", () => {
     const wrapper = shallow(<ObjectItem name={"test"} checked={true} />)
-    expect(wrapper.find(".fesl-row").hasClass("fesl-row-selected")).toBeTruthy()
+    expect(wrapper.find("input[type='checkbox']").prop("checked")).toBeTruthy()
   })
 
-  it("should call uncheckObject when the object is deselected", () => {
+  it("should call uncheckObject when the object/prefix is unchecked", () => {
     const uncheckObject = jest.fn()
-    const wrapper = shallow(<ObjectItem name={"test"} checked={true} uncheckObject={uncheckObject} />)
-    wrapper.find("a").simulate("click", { preventDefault: jest.fn() })
+    const wrapper = shallow(
+      <ObjectItem name={"test"} checked={true} uncheckObject={uncheckObject} />
+    )
+    wrapper.find("input[type='checkbox']").simulate("change")
     expect(uncheckObject).toHaveBeenCalledWith("test")
   })
 })

browser/app/js/objects/__tests__/ObjectsBulkActions.test.js

@@ -20,7 +20,7 @@ import { ObjectsBulkActions } from "../ObjectsBulkActions"
 
 describe("ObjectsBulkActions", () => {
   it("should render without crashing", () => {
-    shallow(<ObjectsBulkActions checkedObjects={0} />)
+    shallow(<ObjectsBulkActions checkedObjects={[]} />)
   })
 
   it("should show actions when checkObjectsCount is more than 0", () => {
@@ -28,21 +28,33 @@ describe("ObjectsBulkActions", () => {
     expect(wrapper.hasClass("list-actions-toggled")).toBeTruthy()
   })
 
-  it("should call downloadObject for single object when download button is clicked", () => {
+  it("should call downloadObject when single object is selected and download button is clicked", () => {
     const downloadObject = jest.fn()
-    const resetCheckedList = jest.fn()
+    const clearChecked = jest.fn()
     const wrapper = shallow(
       <ObjectsBulkActions
-        checkedObjects={["test1"]}
+        checkedObjects={["test"]}
         downloadObject={downloadObject}
-        resetCheckedList={resetCheckedList}
+        clearChecked={clearChecked}
       />
     )
     wrapper.find("#download-checked").simulate("click")
     expect(downloadObject).toHaveBeenCalled()
   })
 
-  it("should call downloadChecked for multiple objects when download button is clicked", () => {
+  it("should call downloadChecked when a folder is selected and download button is clicked", () => {
+    const downloadChecked = jest.fn()
+    const wrapper = shallow(
+      <ObjectsBulkActions
+        checkedObjects={["test/"]}
+        downloadChecked={downloadChecked}
+      />
+    )
+    wrapper.find("#download-checked").simulate("click")
+    expect(downloadChecked).toHaveBeenCalled()
+  })
+
+  it("should call downloadChecked when multiple objects are selected and download button is clicked", () => {
     const downloadChecked = jest.fn()
     const wrapper = shallow(
       <ObjectsBulkActions

browser/app/less/inc/list.less

@@ -74,7 +74,7 @@ div.fesl-row {
     border-bottom: 1px solid transparent;
     cursor: default;
     .transition(background-color);
-    .transition-duration(300ms);
+    .transition-duration(500ms);
 
     @media (max-width: (@screen-xs-max - 100px)) {
         padding: 5px 20px;
@@ -87,8 +87,16 @@ div.fesl-row {
     }
 
     &:hover {
-        &:not(.fesl-row-selected) {
-            background: lighten(@text-muted-color, 22%);
+        .fis-icon {
+            &:before {
+                .opacity(0)
+            }
+        }
+
+        .fis-helper {
+            &:before {
+                .opacity(1);
+            }
         }
     }
 
@@ -129,14 +137,6 @@ div.fesl-row {
 .fesl-row-selected {
     background-color: @list-row-selected-bg;
 
-    &:nth-child(even) {
-        background-color: @list-row-selected-even-bg;
-    }
-
-    &:hover {
-        background: @list-row-selected-hover;
-    }
-
     &, .fesl-item a {
         color: darken(@text-color, 10%);
     }
@@ -161,6 +161,27 @@ div.fesl-row {
         height: 35px;
         z-index: 8;
         opacity: 0;
+        cursor: pointer;
+
+        &:checked {
+            & ~ .fis-icon {
+                background-color: #32393F;
+
+                &:before {
+                    opacity: 0;
+                }
+            }
+
+            & ~ .fis-helper {
+                &:before {
+                    .scale(0);
+                }
+
+                &:after {
+                    .scale(1);
+                }
+            }
+        }
     }
 }
 
@@ -189,6 +210,38 @@ div.fesl-row {
     }
 }
 
+.fis-helper {
+    &:before,
+    &:after {
+        position: absolute;
+        .transition(all);
+        .transition-duration(250ms);
+    }
+
+    &:before {
+        content: '';
+        width: 15px;
+        height: 15px;
+        border: 2px solid @white;
+        z-index: 7;
+        border-radius: 2px;
+        top: 10px;
+        left: 10px;
+        opacity: 0;
+    }
+
+    &:after {
+        font-family: @font-family-icon;
+        content: '\f00c';
+        top: 8px;
+        left: 9px;
+        color: @white;
+        font-size: 14px;
+        .scale(0);
+    }
+}
+
+
 /*--------------------------
     Files and Folders
 ----------------------------*/

browser/app/less/inc/variables.less

@@ -97,7 +97,5 @@
 /*-------------------------
     List
 --------------------------*/
-@list-row-selected-bg:      #fffad6;
-@list-row-selected-even-bg: #faf5d1;
-@list-row-selected-hover:   #f5f0cc;
+@list-row-selected-bg: #fbf2bf;
 @list-row-even-bg:          #fafafa;

buildscripts/checkdeps.sh

@@ -34,7 +34,7 @@ _init() {
 
 ## FIXME:
 ## In OSX, 'readlink -f' option does not exist, hence
-## we have our own readlink -f behaviour here.
+## we have our own readlink -f behavior here.
 ## Once OSX has the option, below function is good enough.
 ## 
 ## readlink() {

buildscripts/gen-ldflags.go

@@ -33,6 +33,7 @@ func genLDFlags(version string) string {
 	ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
 	ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]
 	ldflagsStr += " -X github.com/minio/minio/cmd.GOPATH=" + os.Getenv("GOPATH")
+	ldflagsStr += " -X github.com/minio/minio/cmd.GOROOT=" + os.Getenv("GOROOT")
 	return ldflagsStr
 }
 

cmd/admin-handlers.go

@@ -474,7 +474,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Validate request signature.
-	adminAPIErr := checkAdminRequestAuthType(r, globalServerConfig.GetRegion())
+	adminAPIErr := checkAdminRequestAuthType(r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
@@ -818,7 +818,7 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 
 	// Update local credentials in memory.
 	globalServerConfig.SetCredential(creds)
-	if err = globalServerConfig.Save(); err != nil {
+	if err = globalServerConfig.Save(getConfigFile()); err != nil {
 		writeErrorResponseJSON(w, ErrInternalError, r.URL)
 		return
 	}

cmd/admin-heal-ops.go

@@ -57,7 +57,7 @@ const (
 var (
 	errHealIdleTimeout      = fmt.Errorf("healing results were not consumed for too long")
 	errHealPushStopNDiscard = fmt.Errorf("heal push stopped due to heal stop signal")
-	errHealStopSignalled    = fmt.Errorf("heal stop signalled")
+	errHealStopSignalled    = fmt.Errorf("heal stop signaled")
 
 	errFnHealFromAPIErr = func(err error) error {
 		errCode := toAPIErrorCode(err)
@@ -301,7 +301,7 @@ type healSequence struct {
 	// current accumulated status of the heal sequence
 	currentStatus healSequenceStatus
 
-	// channel signalled by background routine when traversal has
+	// channel signaled by background routine when traversal has
 	// completed
 	traverseAndHealDoneCh chan error
 
@@ -441,7 +441,7 @@ func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
 	h.currentStatus.updateLock.Unlock()
 
 	// This is a "safe" point for the heal sequence to quit if
-	// signalled externally.
+	// signaled externally.
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}

cmd/admin-rpc-client.go

@@ -179,9 +179,9 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
 
 	for _, hostStr := range GetRemotePeers(endpoints) {
 		host, err := xnet.ParseHost(hostStr)
-		logger.CriticalIf(context.Background(), err)
+		logger.FatalIf(err, "Unable to parse Admin RPC Host", context.Background())
 		rpcClient, err := NewAdminRPCClient(host)
-		logger.CriticalIf(context.Background(), err)
+		logger.FatalIf(err, "Unable to initialize Admin RPC Client", context.Background())
 		adminPeerList = append(adminPeerList, adminPeer{
 			addr:      hostStr,
 			cmdRunner: rpcClient,

cmd/admin-rpc-server.go

@@ -121,7 +121,7 @@ func NewAdminRPCServer() (*xrpc.Server, error) {
 // registerAdminRPCRouter - creates and registers Admin RPC server and its router.
 func registerAdminRPCRouter(router *mux.Router) {
 	rpcServer, err := NewAdminRPCServer()
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to initialize Lock RPC Server", context.Background())
 	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
 	subrouter.Path(adminServiceSubPath).Handler(rpcServer)
 }

cmd/api-errors.go

@@ -17,10 +17,12 @@
 package cmd
 
 import (
+	"context"
 	"encoding/xml"
 	"fmt"
 	"net/http"
 
+	"github.com/coreos/etcd/client"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
@@ -191,6 +193,7 @@ const (
 	ErrHealMissingBucket
 	ErrHealAlreadyRunning
 	ErrHealOverlappingPaths
+	ErrIncorrectContinuationToken
 )
 
 // error code to APIError structure, these fields carry respective
@@ -840,7 +843,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Object storage backend is unreachable",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
-
+	ErrIncorrectContinuationToken: {
+		Code:           "InvalidArgument",
+		Description:    "The continuation token provided is incorrect",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	// Add your error structure here.
 }
 
@@ -864,6 +871,29 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminInvalidAccessKey
 	case auth.ErrInvalidSecretKeyLength:
 		apiErr = ErrAdminInvalidSecretKey
+	// SSE errors
+	case errInsecureSSERequest:
+		apiErr = ErrInsecureSSECustomerRequest
+	case errInvalidSSEAlgorithm:
+		apiErr = ErrInvalidSSECustomerAlgorithm
+	case errInvalidSSEKey:
+		apiErr = ErrInvalidSSECustomerKey
+	case errMissingSSEKey:
+		apiErr = ErrMissingSSECustomerKey
+	case errMissingSSEKeyMD5:
+		apiErr = ErrMissingSSECustomerKeyMD5
+	case errSSEKeyMD5Mismatch:
+		apiErr = ErrSSECustomerKeyMD5Mismatch
+	case errObjectTampered:
+		apiErr = ErrObjectTampered
+	case errEncryptedObject:
+		apiErr = ErrSSEEncryptedObject
+	case errInvalidSSEParameters:
+		apiErr = ErrInvalidSSECustomerParameters
+	case errSSEKeyMismatch:
+		apiErr = ErrAccessDenied // no access without correct key
+	case context.Canceled, context.DeadlineExceeded:
+		apiErr = ErrOperationTimedOut
 	}
 
 	if apiErr != ErrNone {
@@ -871,27 +901,12 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		return apiErr
 	}
 
-	switch err { // SSE errors
-	case errInsecureSSERequest:
-		return ErrInsecureSSECustomerRequest
-	case errInvalidSSEAlgorithm:
-		return ErrInvalidSSECustomerAlgorithm
-	case errInvalidSSEKey:
-		return ErrInvalidSSECustomerKey
-	case errMissingSSEKey:
-		return ErrMissingSSECustomerKey
-	case errMissingSSEKeyMD5:
-		return ErrMissingSSECustomerKeyMD5
-	case errSSEKeyMD5Mismatch:
-		return ErrSSECustomerKeyMD5Mismatch
-	case errObjectTampered:
-		return ErrObjectTampered
-	case errEncryptedObject:
-		return ErrSSEEncryptedObject
-	case errInvalidSSEParameters:
-		return ErrInvalidSSECustomerParameters
-	case errSSEKeyMismatch:
-		return ErrAccessDenied // no access without correct key
+	// etcd specific errors, a key is always a bucket for us return
+	// ErrNoSuchBucket in such a case.
+	if e, ok := err.(*client.Error); ok {
+		if e.Code == client.ErrorCodeKeyNotFound {
+			return ErrNoSuchBucket
+		}
 	}
 
 	switch err.(type) {

cmd/api-resources.go

@@ -36,7 +36,17 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
 }
 
 // Parse bucket url queries for ListObjects V2.
-func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string) {
+func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
+	// The continuation-token cannot be empty.
+	if val, ok := values["continuation-token"]; ok {
+		if len(val[0]) == 0 {
+			errCode = ErrIncorrectContinuationToken
+			return
+		}
+	}
+
 	prefix = values.Get("prefix")
 	token = values.Get("continuation-token")
 	startAfter = values.Get("start-after")

cmd/api-resources_test.go

@@ -29,6 +29,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 		fetchOwner                           bool
 		maxKeys                              int
 		encodingType                         string
+		errCode                              APIErrorCode
 	}{
 		{
 			values: url.Values{
@@ -47,6 +48,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 			fetchOwner:   true,
 			maxKeys:      100,
 			encodingType: "gzip",
+			errCode:      ErrNone,
 		},
 		{
 			values: url.Values{
@@ -64,11 +66,34 @@ func TestListObjectsV2Resources(t *testing.T) {
 			fetchOwner:   true,
 			maxKeys:      1000,
 			encodingType: "gzip",
+			errCode:      ErrNone,
+		},
+		{
+			values: url.Values{
+				"prefix":             []string{"photos/"},
+				"continuation-token": []string{""},
+				"start-after":        []string{"start-after"},
+				"delimiter":          []string{"/"},
+				"fetch-owner":        []string{"true"},
+				"encoding-type":      []string{"gzip"},
+			},
+			prefix:       "",
+			token:        "",
+			startAfter:   "",
+			delimiter:    "",
+			fetchOwner:   false,
+			maxKeys:      0,
+			encodingType: "",
+			errCode:      ErrIncorrectContinuationToken,
 		},
 	}
 
 	for i, testCase := range testCases {
-		prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType := getListObjectsV2Args(testCase.values)
+		prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(testCase.values)
+
+		if errCode != testCase.errCode {
+			t.Errorf("Test %d: Expected error code:%d, got %d", i+1, testCase.errCode, errCode)
+		}
 		if prefix != testCase.prefix {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix)
 		}

cmd/api-response.go

@@ -21,6 +21,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/pkg/handlers"
@@ -293,8 +294,10 @@ func getObjectLocation(r *http.Request, domain, bucket, object string) string {
 	}
 	// If domain is set then we need to use bucket DNS style.
 	if domain != "" {
-		u.Host = bucket + "." + domain
-		u.Path = path.Join(slashSeparator, object)
+		if strings.Contains(r.Host, domain) {
+			u.Host = bucket + "." + r.Host
+			u.Path = path.Join(slashSeparator, object)
+		}
 	}
 	return u.String()
 }

cmd/bucket-handlers-listobjects.go

@@ -70,8 +70,15 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
+	urlValues := r.URL.Query()
+
 	// Extract all the listObjectsV2 query params to their native values.
-	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _ := getListObjectsV2Args(r.URL.Query())
+	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _, errCode := getListObjectsV2Args(urlValues)
+
+	if errCode != ErrNone {
+		writeErrorResponse(w, errCode, r.URL)
+		return
+	}
 
 	// In ListObjectsV2 'continuation-token' is the marker.
 	marker := token
@@ -93,7 +100,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	}
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
-	// marshalled into S3 compatible XML header.
+	// marshaled into S3 compatible XML header.
 	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, marker, delimiter, maxKeys, fetchOwner, startAfter)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -159,7 +166,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
-	// marshalled into S3 compatible XML header.
+	// marshaled into S3 compatible XML header.
 	listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)

cmd/bucket-handlers.go

@@ -17,8 +17,10 @@
 package cmd
 
 import (
+	"context"
 	"encoding/base64"
 	"encoding/xml"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
@@ -28,13 +30,61 @@ import (
 	"strings"
 	"sync"
 
+	etcd "github.com/coreos/etcd/client"
+
 	"github.com/gorilla/mux"
+
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/sync/errgroup"
+
+	"github.com/minio/minio-go/pkg/set"
 )
 
+// Check if there are buckets on server without corresponding entry in etcd backend and
+// make entries. Here is the general flow
+// - Range over all the available buckets
+// - Check if a bucket has an entry in etcd backend
+// -- If no, make an entry
+// -- If yes, check if the IP of entry matches local IP. This means entry is for this instance.
+// -- If IP of the entry doesn't match, this means entry is for another instance. Log an error to console.
+func initFederatorBackend(objLayer ObjectLayer) {
+	b, err := objLayer.ListBuckets(context.Background())
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return
+	}
+
+	g := errgroup.WithNErrs(len(b))
+	for index := range b {
+		index := index
+		g.Go(func() error {
+			r, gerr := globalDNSConfig.Get(b[index].Name)
+			if gerr != nil {
+				if etcd.IsKeyNotFound(gerr) || gerr == dns.ErrNoEntriesFound {
+					return globalDNSConfig.Put(b[index].Name)
+				}
+				return gerr
+			}
+			if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				// There is already an entry for this bucket, with all IP addresses different. This indicates a bucket name collision. Log an error and continue.
+				return fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", b[index].Name, globalDomainIPs.ToSlice())
+			}
+			return nil
+		}, index)
+	}
+
+	for _, err := range g.Wait() {
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			return
+		}
+	}
+}
+
 // GetBucketLocationHandler - GET Bucket location.
 // -------------------------
 // This operation returns bucket location.
@@ -157,12 +207,33 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
-
-	// Invoke the list buckets.
-	bucketsInfo, err := listBuckets(ctx)
-	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
+	// If etcd, dns federation configured list buckets from etcd.
+	var bucketsInfo []BucketInfo
+	if globalDNSConfig != nil {
+		dnsBuckets, err := globalDNSConfig.List()
+		if err != nil && !etcd.IsKeyNotFound(err) && err != dns.ErrNoEntriesFound {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
+		bucketSet := set.NewStringSet()
+		for _, dnsRecord := range dnsBuckets {
+			if bucketSet.Contains(dnsRecord.Key) {
+				continue
+			}
+			bucketsInfo = append(bucketsInfo, BucketInfo{
+				Name:    strings.Trim(dnsRecord.Key, slashSeparator),
+				Created: dnsRecord.CreationDate,
+			})
+			bucketSet.Add(dnsRecord.Key)
+		}
+	} else {
+		// Invoke the list buckets.
+		var err error
+		bucketsInfo, err = listBuckets(ctx)
+		if err != nil {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
 	}
 
 	// Generate response.
@@ -211,7 +282,13 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	}
 
 	// Allocate incoming content length bytes.
-	deleteXMLBytes := make([]byte, r.ContentLength)
+	var deleteXMLBytes []byte
+	const maxBodySize = 2 * 1000 * 1024 // The max. XML contains 1000 object names (each at most 1024 bytes long) + XML overhead
+	if r.ContentLength > maxBodySize {  // Only allocated memory for at most 1000 objects
+		deleteXMLBytes = make([]byte, maxBodySize)
+	} else {
+		deleteXMLBytes = make([]byte, r.ContentLength)
+	}
 
 	// Read incoming body XML bytes.
 	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
@@ -353,12 +430,33 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+	if globalDNSConfig != nil {
+		if _, err := globalDNSConfig.Get(bucket); err != nil {
+			if etcd.IsKeyNotFound(err) || err == dns.ErrNoEntriesFound {
+				// Proceed to creating a bucket.
+				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
+					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					return
+				}
+				if err = globalDNSConfig.Put(bucket); err != nil {
+					objectAPI.DeleteBucket(ctx, bucket)
+					writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+					return
+				}
+
+				// Make sure to add Location information here only for bucket
+				w.Header().Set("Location", getObjectLocation(r, globalDomainName, bucket, ""))
+
+				writeSuccessResponseHeadersOnly(w)
+				return
+			}
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+
+		}
+		writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL)
 		return
 	}
-	defer bucketLock.Unlock()
 
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
@@ -660,6 +758,15 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		logger.LogIf(ctx, nerr.Err)
 	}
 
+	if globalDNSConfig != nil {
+		if err := globalDNSConfig.Delete(bucket); err != nil {
+			// Deleting DNS entry failed, attempt to create the bucket again.
+			objectAPI.MakeBucketWithLocation(ctx, bucket, "")
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
+	}
+
 	// Write success response.
 	writeSuccessNoContent(w)
 }

cmd/bucket-notification-handlers.go

@@ -224,11 +224,17 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
-	host, e := xnet.ParseHost(r.RemoteAddr)
-	logger.CriticalIf(ctx, e)
+	host, err := xnet.ParseHost(r.RemoteAddr)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
 
-	target, e := target.NewHTTPClientTarget(*host, w)
-	logger.CriticalIf(ctx, e)
+	target, err := target.NewHTTPClientTarget(*host, w)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
 
 	rulesMap := event.NewRulesMap(eventNames, pattern, target.ID())
 
@@ -241,10 +247,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
 	defer globalNotificationSys.RemoveRulesMap(bucketName, rulesMap)
 
-	thisAddr, e := xnet.ParseHost(GetLocalPeer(globalEndpoints))
-	logger.CriticalIf(ctx, e)
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
 
-	if err := SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
+	if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
 		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -259,7 +268,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	<-target.DoneCh
 
-	if err := RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
+	if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
 		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)

cmd/build-constants.go

@@ -22,6 +22,9 @@ var (
 	// GOPATH - GOPATH value at the time of build.
 	GOPATH = ""
 
+	// GOROOT - GOROOT value at the time of build.
+	GOROOT = ""
+
 	// Go get development tag.
 	goGetTag = "DEVELOPMENT.GOGET"
 

cmd/certs.go

@@ -86,10 +86,11 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 		return nil, nil
 	}
 
-	rootCAs, err := x509.SystemCertPool()
-	if err != nil {
-		// In some systems like Windows, system cert pool is not supported.
-		// Hence we create a new cert pool.
+	rootCAs, _ := x509.SystemCertPool()
+	if rootCAs == nil {
+		// In some systems (like Windows) system cert pool is
+		// not supported or no certificates are present on the
+		// system - so we create a new cert pool.
 		rootCAs = x509.NewCertPool()
 	}
 

cmd/common-main.go

@@ -17,16 +17,23 @@
 package cmd
 
 import (
+	"context"
 	"errors"
+	"net"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
+	etcd "github.com/coreos/etcd/client"
+
 	"github.com/minio/cli"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dns"
+
+	"github.com/minio/minio-go/pkg/set"
 )
 
 // Check for updates and print a notification message
@@ -41,12 +48,32 @@ func checkUpdate(mode string) {
 	}
 }
 
+// Initialize and load config from remote etcd or local config directory
 func initConfig() {
-	// Config file does not exist, we create it fresh and return upon success.
+	if globalEtcdClient != nil {
+		kapi := etcd.NewKeysAPI(globalEtcdClient)
+		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+		_, err := kapi.Get(ctx, getConfigFile(), nil)
+		cancel()
+		if err == nil {
+			logger.FatalIf(migrateConfig(), "Config migration failed.")
+			logger.FatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion)
+		} else {
+			if etcd.IsKeyNotFound(err) {
+				logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time.")
+				logger.Info("Created minio configuration file successfully at %v", globalEtcdClient.Endpoints())
+			} else {
+				logger.FatalIf(err, "Unable to load config version: '%s'.", serverConfigVersion)
+			}
+		}
+		return
+	}
+
 	if isFile(getConfigFile()) {
 		logger.FatalIf(migrateConfig(), "Config migration failed")
 		logger.FatalIf(loadConfig(), "Unable to load the configuration file")
 	} else {
+		// Config file does not exist, we create it fresh and return upon success.
 		logger.FatalIf(newConfig(), "Unable to initialize minio config for the first time")
 		logger.Info("Created minio configuration file successfully at " + getConfigDir())
 	}
@@ -123,8 +150,36 @@ func handleCommonEnvVars() {
 		logger.FatalIf(err, "error opening file %s", traceFile)
 	}
 
+	etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
+	if ok {
+		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
+		var err error
+		globalEtcdClient, err = etcd.New(etcd.Config{
+			Endpoints: etcdEndpoints,
+			Transport: NewCustomHTTPTransport(),
+		})
+		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
+	}
+
 	globalDomainName, globalIsEnvDomainName = os.LookupEnv("MINIO_DOMAIN")
 
+	minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
+	if ok {
+		minioEndpoints := strings.Split(minioEndpointsEnv, ",")
+		globalDomainIPs = set.NewStringSet()
+		for i, ip := range minioEndpoints {
+			if net.ParseIP(ip) == nil {
+				logger.FatalIf(errInvalidArgument, "Unable to initialize Minio server with invalid MINIO_PUBLIC_IPS[%d]: %s", i, ip)
+			}
+			globalDomainIPs.Add(ip)
+		}
+	}
+	if globalDomainName != "" && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
+		var err error
+		globalDNSConfig, err = dns.NewCoreDNS(globalDomainName, globalDomainIPs, globalMinioPort, globalEtcdClient)
+		logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainName)
+	}
+
 	if drives := os.Getenv("MINIO_CACHE_DRIVES"); drives != "" {
 		driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter))
 		if err != nil {
@@ -150,6 +205,16 @@ func handleCommonEnvVars() {
 		globalCacheExpiry = expiry
 	}
 
+	if maxUseStr := os.Getenv("MINIO_CACHE_MAXUSE"); maxUseStr != "" {
+		maxUse, err := strconv.Atoi(maxUseStr)
+		if err != nil {
+			logger.Fatal(uiErrInvalidCacheMaxUse(err), "Unable to parse MINIO_CACHE_MAXUSE value (`%s`)", maxUseStr)
+		}
+		// maxUse should be a valid percentage.
+		if maxUse > 0 && maxUse <= 100 {
+			globalCacheMaxUse = maxUse
+		}
+	}
 	// In place update is true by default if the MINIO_UPDATE is not set
 	// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
 	// in-place update is off.

cmd/config-current.go

@@ -39,9 +39,9 @@ import (
 // 6. Make changes in config-current_test.go for any test change
 
 // Config version
-const serverConfigVersion = "25"
+const serverConfigVersion = "26"
 
-type serverConfig = serverConfigV25
+type serverConfig = serverConfigV26
 
 var (
 	// globalServerConfig server config.
@@ -116,10 +116,11 @@ func (s *serverConfig) GetWorm() bool {
 }
 
 // SetCacheConfig sets the current cache config
-func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int) {
+func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int, maxuse int) {
 	s.Cache.Drives = drives
 	s.Cache.Exclude = exclude
 	s.Cache.Expiry = expiry
+	s.Cache.MaxUse = maxuse
 }
 
 // GetCacheConfig gets the current cache config
@@ -127,10 +128,19 @@ func (s *serverConfig) GetCacheConfig() CacheConfig {
 	return s.Cache
 }
 
-// Save config.
-func (s *serverConfig) Save() error {
-	// Save config file.
-	return quick.Save(getConfigFile(), s)
+// Save config file to corresponding backend
+func Save(configFile string, data interface{}) error {
+	return quick.SaveConfig(data, configFile, globalEtcdClient)
+}
+
+// Load config from backend
+func Load(configFile string, data interface{}) (quick.Config, error) {
+	return quick.LoadConfig(configFile, globalEtcdClient, data)
+}
+
+// GetVersion gets config version from backend
+func GetVersion(configFile string) (string, error) {
+	return quick.GetVersion(configFile, globalEtcdClient)
 }
 
 // Returns the string describing a difference with the given
@@ -196,6 +206,7 @@ func newServerConfig() *serverConfig {
 			Drives:  []string{},
 			Exclude: []string{},
 			Expiry:  globalCacheExpiry,
+			MaxUse:  globalCacheMaxUse,
 		},
 		Notify: notifier{},
 	}
@@ -223,6 +234,7 @@ func newServerConfig() *serverConfig {
 	srvCfg.Cache.Drives = make([]string, 0)
 	srvCfg.Cache.Exclude = make([]string, 0)
 	srvCfg.Cache.Expiry = globalCacheExpiry
+	srvCfg.Cache.MaxUse = globalCacheMaxUse
 	return srvCfg
 }
 
@@ -230,7 +242,10 @@ func newServerConfig() *serverConfig {
 // found, otherwise use default parameters
 func newConfig() error {
 	// Initialize server config.
-	srvCfg := newServerConfig()
+	srvCfg, err := newQuickConfig(newServerConfig())
+	if err != nil {
+		return err
+	}
 
 	// If env is set override the credentials from config file.
 	if globalIsEnvCreds {
@@ -258,7 +273,7 @@ func newConfig() error {
 	}
 
 	if globalIsDiskCacheEnabled {
-		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry)
+		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
 	}
 
 	// hold the mutex lock before a new config is assigned.
@@ -269,7 +284,19 @@ func newConfig() error {
 	globalServerConfigMu.Unlock()
 
 	// Save config into file.
-	return globalServerConfig.Save()
+	return Save(getConfigFile(), globalServerConfig)
+}
+
+// newQuickConfig - initialize a new server config, with an allocated
+// quick.Config interface.
+func newQuickConfig(srvCfg *serverConfig) (*serverConfig, error) {
+	qcfg, err := quick.NewConfig(srvCfg, globalEtcdClient)
+	if err != nil {
+		return nil, err
+	}
+
+	srvCfg.Config = qcfg
+	return srvCfg, nil
 }
 
 // getValidConfig - returns valid server configuration
@@ -279,7 +306,14 @@ func getValidConfig() (*serverConfig, error) {
 		Browser: true,
 	}
 
-	if _, err := quick.Load(getConfigFile(), srvCfg); err != nil {
+	var err error
+	srvCfg, err = newQuickConfig(srvCfg)
+	if err != nil {
+		return nil, err
+	}
+
+	configFile := getConfigFile()
+	if err = srvCfg.Load(configFile); err != nil {
 		return nil, err
 	}
 
@@ -327,7 +361,7 @@ func loadConfig() error {
 	}
 
 	if globalIsDiskCacheEnabled {
-		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry)
+		srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
 	}
 
 	// hold the mutex lock before a new config is assigned.
@@ -356,6 +390,7 @@ func loadConfig() error {
 		globalCacheDrives = cacheConf.Drives
 		globalCacheExcludes = cacheConf.Exclude
 		globalCacheExpiry = cacheConf.Expiry
+		globalCacheMaxUse = cacheConf.MaxUse
 	}
 	globalServerConfigMu.Unlock()
 

cmd/config-current_test.go

@@ -50,7 +50,7 @@ func TestServerConfig(t *testing.T) {
 	}
 
 	// Attempt to save.
-	if err := globalServerConfig.Save(); err != nil {
+	if err := globalServerConfig.Save(getConfigFile()); err != nil {
 		t.Fatalf("Unable to save updated config file %s", err)
 	}
 

cmd/config-migrate.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"path/filepath"
 
+	etcd "github.com/coreos/etcd/client"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
@@ -42,7 +43,7 @@ func migrateConfig() error {
 	}
 
 	// Load only config version information.
-	version, err := quick.GetVersion(getConfigFile())
+	version, err := GetVersion(getConfigFile())
 	if err != nil {
 		return err
 	}
@@ -182,6 +183,11 @@ func migrateConfig() error {
 			return err
 		}
 		fallthrough
+	case "25":
+		if err = migrateV25ToV26(); err != nil {
+			return err
+		}
+		fallthrough
 	case serverConfigVersion:
 		// No migration needed. this always points to current version.
 		err = nil
@@ -194,8 +200,8 @@ func purgeV1() error {
 	configFile := filepath.Join(getConfigDir(), "fsUsers.json")
 
 	cv1 := &configV1{}
-	_, err := quick.Load(configFile, cv1)
-	if os.IsNotExist(err) {
+	_, err := Load(configFile, cv1)
+	if os.IsNotExist(err) || etcd.IsKeyNotFound(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘1’. %v", err)
@@ -215,7 +221,7 @@ func migrateV2ToV3() error {
 	configFile := getConfigFile()
 
 	cv2 := &configV2{}
-	_, err := quick.Load(configFile, cv2)
+	_, err := Load(configFile, cv2)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -259,7 +265,7 @@ func migrateV2ToV3() error {
 	}
 	srvConfig.Logger.Syslog = slogger
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv2.Version, srvConfig.Version, err)
 	}
 
@@ -274,7 +280,7 @@ func migrateV3ToV4() error {
 	configFile := getConfigFile()
 
 	cv3 := &configV3{}
-	_, err := quick.Load(configFile, cv3)
+	_, err := Load(configFile, cv3)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -297,7 +303,7 @@ func migrateV3ToV4() error {
 	srvConfig.Logger.File = cv3.Logger.File
 	srvConfig.Logger.Syslog = cv3.Logger.Syslog
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv3.Version, srvConfig.Version, err)
 	}
 
@@ -312,7 +318,7 @@ func migrateV4ToV5() error {
 	configFile := getConfigFile()
 
 	cv4 := &configV4{}
-	_, err := quick.Load(configFile, cv4)
+	_, err := Load(configFile, cv4)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -338,7 +344,7 @@ func migrateV4ToV5() error {
 	srvConfig.Logger.ElasticSearch.Enable = false
 	srvConfig.Logger.Redis.Enable = false
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv4.Version, srvConfig.Version, err)
 	}
 
@@ -353,7 +359,7 @@ func migrateV5ToV6() error {
 	configFile := getConfigFile()
 
 	cv5 := &configV5{}
-	_, err := quick.Load(configFile, cv5)
+	_, err := Load(configFile, cv5)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -427,7 +433,7 @@ func migrateV5ToV6() error {
 		}
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv5.Version, srvConfig.Version, err)
 	}
 
@@ -442,7 +448,7 @@ func migrateV6ToV7() error {
 	configFile := getConfigFile()
 
 	cv6 := &configV6{}
-	_, err := quick.Load(configFile, cv6)
+	_, err := Load(configFile, cv6)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -483,7 +489,7 @@ func migrateV6ToV7() error {
 		srvConfig.Notify.Redis = cv6.Notify.Redis
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv6.Version, srvConfig.Version, err)
 	}
 
@@ -498,7 +504,7 @@ func migrateV7ToV8() error {
 	configFile := getConfigFile()
 
 	cv7 := &serverConfigV7{}
-	_, err := quick.Load(configFile, cv7)
+	_, err := Load(configFile, cv7)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -546,7 +552,7 @@ func migrateV7ToV8() error {
 		srvConfig.Notify.Redis = cv7.Notify.Redis
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv7.Version, srvConfig.Version, err)
 	}
 
@@ -560,7 +566,7 @@ func migrateV8ToV9() error {
 	configFile := getConfigFile()
 
 	cv8 := &serverConfigV8{}
-	_, err := quick.Load(configFile, cv8)
+	_, err := Load(configFile, cv8)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -616,7 +622,7 @@ func migrateV8ToV9() error {
 		srvConfig.Notify.PostgreSQL = cv8.Notify.PostgreSQL
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv8.Version, srvConfig.Version, err)
 	}
 
@@ -630,7 +636,7 @@ func migrateV9ToV10() error {
 	configFile := getConfigFile()
 
 	cv9 := &serverConfigV9{}
-	_, err := quick.Load(configFile, cv9)
+	_, err := Load(configFile, cv9)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -684,7 +690,7 @@ func migrateV9ToV10() error {
 		srvConfig.Notify.PostgreSQL = cv9.Notify.PostgreSQL
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv9.Version, srvConfig.Version, err)
 	}
 
@@ -698,7 +704,7 @@ func migrateV10ToV11() error {
 	configFile := getConfigFile()
 
 	cv10 := &serverConfigV10{}
-	_, err := quick.Load(configFile, cv10)
+	_, err := Load(configFile, cv10)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -755,7 +761,7 @@ func migrateV10ToV11() error {
 	srvConfig.Notify.Kafka = make(map[string]target.KafkaArgs)
 	srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv10.Version, srvConfig.Version, err)
 	}
 
@@ -769,7 +775,7 @@ func migrateV11ToV12() error {
 	configFile := getConfigFile()
 
 	cv11 := &serverConfigV11{}
-	_, err := quick.Load(configFile, cv11)
+	_, err := Load(configFile, cv11)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -853,7 +859,7 @@ func migrateV11ToV12() error {
 		}
 	}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv11.Version, srvConfig.Version, err)
 	}
 
@@ -866,7 +872,7 @@ func migrateV12ToV13() error {
 	configFile := getConfigFile()
 
 	cv12 := &serverConfigV12{}
-	_, err := quick.Load(configFile, cv12)
+	_, err := Load(configFile, cv12)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -933,7 +939,7 @@ func migrateV12ToV13() error {
 	srvConfig.Notify.Webhook = make(map[string]target.WebhookArgs)
 	srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv12.Version, srvConfig.Version, err)
 	}
 
@@ -946,7 +952,7 @@ func migrateV13ToV14() error {
 	configFile := getConfigFile()
 
 	cv13 := &serverConfigV13{}
-	_, err := quick.Load(configFile, cv13)
+	_, err := Load(configFile, cv13)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1018,7 +1024,7 @@ func migrateV13ToV14() error {
 	// Set the new browser parameter to true by default
 	srvConfig.Browser = true
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv13.Version, srvConfig.Version, err)
 	}
 
@@ -1031,7 +1037,7 @@ func migrateV14ToV15() error {
 	configFile := getConfigFile()
 
 	cv14 := &serverConfigV14{}
-	_, err := quick.Load(configFile, cv14)
+	_, err := Load(configFile, cv14)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1107,7 +1113,7 @@ func migrateV14ToV15() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv14.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv14.Version, srvConfig.Version, err)
 	}
 
@@ -1121,7 +1127,7 @@ func migrateV15ToV16() error {
 	configFile := getConfigFile()
 
 	cv15 := &serverConfigV15{}
-	_, err := quick.Load(configFile, cv15)
+	_, err := Load(configFile, cv15)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1197,7 +1203,7 @@ func migrateV15ToV16() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv15.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv15.Version, srvConfig.Version, err)
 	}
 
@@ -1211,7 +1217,7 @@ func migrateV16ToV17() error {
 	configFile := getConfigFile()
 
 	cv16 := &serverConfigV16{}
-	_, err := quick.Load(configFile, cv16)
+	_, err := Load(configFile, cv16)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1318,7 +1324,7 @@ func migrateV16ToV17() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv16.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv16.Version, srvConfig.Version, err)
 	}
 
@@ -1332,7 +1338,7 @@ func migrateV17ToV18() error {
 	configFile := getConfigFile()
 
 	cv17 := &serverConfigV17{}
-	_, err := quick.Load(configFile, cv17)
+	_, err := Load(configFile, cv17)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1422,7 +1428,7 @@ func migrateV17ToV18() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv17.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv17.Version, srvConfig.Version, err)
 	}
 
@@ -1434,7 +1440,7 @@ func migrateV18ToV19() error {
 	configFile := getConfigFile()
 
 	cv18 := &serverConfigV18{}
-	_, err := quick.Load(configFile, cv18)
+	_, err := Load(configFile, cv18)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1528,7 +1534,7 @@ func migrateV18ToV19() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv18.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv18.Version, srvConfig.Version, err)
 	}
 
@@ -1540,7 +1546,7 @@ func migrateV19ToV20() error {
 	configFile := getConfigFile()
 
 	cv19 := &serverConfigV19{}
-	_, err := quick.Load(configFile, cv19)
+	_, err := Load(configFile, cv19)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1633,7 +1639,7 @@ func migrateV19ToV20() error {
 	// Load browser config from existing config in the file.
 	srvConfig.Browser = cv19.Browser
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv19.Version, srvConfig.Version, err)
 	}
 
@@ -1645,7 +1651,7 @@ func migrateV20ToV21() error {
 	configFile := getConfigFile()
 
 	cv20 := &serverConfigV20{}
-	_, err := quick.Load(configFile, cv20)
+	_, err := Load(configFile, cv20)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1737,7 +1743,7 @@ func migrateV20ToV21() error {
 	// Load domain config from existing config in the file.
 	srvConfig.Domain = cv20.Domain
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv20.Version, srvConfig.Version, err)
 	}
 
@@ -1749,7 +1755,7 @@ func migrateV21ToV22() error {
 	configFile := getConfigFile()
 
 	cv21 := &serverConfigV21{}
-	_, err := quick.Load(configFile, cv21)
+	_, err := Load(configFile, cv21)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1841,7 +1847,7 @@ func migrateV21ToV22() error {
 	// Load domain config from existing config in the file.
 	srvConfig.Domain = cv21.Domain
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv21.Version, srvConfig.Version, err)
 	}
 
@@ -1853,7 +1859,7 @@ func migrateV22ToV23() error {
 	configFile := getConfigFile()
 
 	cv22 := &serverConfigV22{}
-	_, err := quick.Load(configFile, cv22)
+	_, err := Load(configFile, cv22)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -1954,7 +1960,7 @@ func migrateV22ToV23() error {
 	srvConfig.Cache.Exclude = []string{}
 	srvConfig.Cache.Expiry = globalCacheExpiry
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv22.Version, srvConfig.Version, err)
 	}
 
@@ -1966,7 +1972,7 @@ func migrateV23ToV24() error {
 	configFile := getConfigFile()
 
 	cv23 := &serverConfigV23{}
-	_, err := quick.Load(configFile, cv23)
+	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv23)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -2067,7 +2073,7 @@ func migrateV23ToV24() error {
 	srvConfig.Cache.Exclude = cv23.Cache.Exclude
 	srvConfig.Cache.Expiry = cv23.Cache.Expiry
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv23.Version, srvConfig.Version, err)
 	}
 
@@ -2079,7 +2085,7 @@ func migrateV24ToV25() error {
 	configFile := getConfigFile()
 
 	cv24 := &serverConfigV24{}
-	_, err := quick.Load(configFile, cv24)
+	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv24)
 	if os.IsNotExist(err) {
 		return nil
 	} else if err != nil {
@@ -2185,10 +2191,129 @@ func migrateV24ToV25() error {
 	srvConfig.Cache.Exclude = cv24.Cache.Exclude
 	srvConfig.Cache.Expiry = cv24.Cache.Expiry
 
-	if err = quick.Save(configFile, srvConfig); err != nil {
+	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv24.Version, srvConfig.Version, err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, cv24.Version, srvConfig.Version)
 	return nil
 }
+
+func migrateV25ToV26() error {
+	configFile := getConfigFile()
+
+	cv25 := &serverConfigV25{}
+	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv25)
+	if os.IsNotExist(err) {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config version ‘25’. %v", err)
+	}
+	if cv25.Version != "25" {
+		return nil
+	}
+
+	// Copy over fields from V25 into V26 config struct
+	srvConfig := &serverConfigV26{
+		Notify: notifier{},
+	}
+	srvConfig.Version = "26"
+	srvConfig.Credential = cv25.Credential
+	srvConfig.Region = cv25.Region
+	if srvConfig.Region == "" {
+		// Region needs to be set for AWS Signature Version 4.
+		srvConfig.Region = globalMinioDefaultRegion
+	}
+
+	if len(cv25.Notify.AMQP) == 0 {
+		srvConfig.Notify.AMQP = make(map[string]target.AMQPArgs)
+		srvConfig.Notify.AMQP["1"] = target.AMQPArgs{}
+	} else {
+		srvConfig.Notify.AMQP = cv25.Notify.AMQP
+	}
+	if len(cv25.Notify.Elasticsearch) == 0 {
+		srvConfig.Notify.Elasticsearch = make(map[string]target.ElasticsearchArgs)
+		srvConfig.Notify.Elasticsearch["1"] = target.ElasticsearchArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Elasticsearch = cv25.Notify.Elasticsearch
+	}
+	if len(cv25.Notify.Redis) == 0 {
+		srvConfig.Notify.Redis = make(map[string]target.RedisArgs)
+		srvConfig.Notify.Redis["1"] = target.RedisArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.Redis = cv25.Notify.Redis
+	}
+	if len(cv25.Notify.PostgreSQL) == 0 {
+		srvConfig.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
+		srvConfig.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.PostgreSQL = cv25.Notify.PostgreSQL
+	}
+	if len(cv25.Notify.Kafka) == 0 {
+		srvConfig.Notify.Kafka = make(map[string]target.KafkaArgs)
+		srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
+	} else {
+		srvConfig.Notify.Kafka = cv25.Notify.Kafka
+	}
+	if len(cv25.Notify.NATS) == 0 {
+		srvConfig.Notify.NATS = make(map[string]target.NATSArgs)
+		srvConfig.Notify.NATS["1"] = target.NATSArgs{}
+	} else {
+		srvConfig.Notify.NATS = cv25.Notify.NATS
+	}
+	if len(cv25.Notify.Webhook) == 0 {
+		srvConfig.Notify.Webhook = make(map[string]target.WebhookArgs)
+		srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
+	} else {
+		srvConfig.Notify.Webhook = cv25.Notify.Webhook
+	}
+	if len(cv25.Notify.MySQL) == 0 {
+		srvConfig.Notify.MySQL = make(map[string]target.MySQLArgs)
+		srvConfig.Notify.MySQL["1"] = target.MySQLArgs{
+			Format: event.NamespaceFormat,
+		}
+	} else {
+		srvConfig.Notify.MySQL = cv25.Notify.MySQL
+	}
+
+	if len(cv25.Notify.MQTT) == 0 {
+		srvConfig.Notify.MQTT = make(map[string]target.MQTTArgs)
+		srvConfig.Notify.MQTT["1"] = target.MQTTArgs{}
+	} else {
+		srvConfig.Notify.MQTT = cv25.Notify.MQTT
+	}
+
+	// Load browser config from existing config in the file.
+	srvConfig.Browser = cv25.Browser
+
+	// Load worm config from existing config in the file.
+	srvConfig.Worm = cv25.Worm
+
+	// Load domain config from existing config in the file.
+	srvConfig.Domain = cv25.Domain
+
+	// Load storage class config from existing storage class config in the file.
+	srvConfig.StorageClass.RRS = cv25.StorageClass.RRS
+	srvConfig.StorageClass.Standard = cv25.StorageClass.Standard
+
+	// Load cache config from existing cache config in the file.
+	srvConfig.Cache.Drives = cv25.Cache.Drives
+	srvConfig.Cache.Exclude = cv25.Cache.Exclude
+	srvConfig.Cache.Expiry = cv25.Cache.Expiry
+
+	// Add predefined value to new server config.
+	srvConfig.Cache.MaxUse = globalCacheMaxUse
+
+	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv25.Version, srvConfig.Version, err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, cv25.Version, srvConfig.Version)
+	return nil
+}

cmd/config-versions.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/quick"
 )
 
 /////////////////// Config V1 ///////////////////
@@ -633,6 +634,35 @@ type serverConfigV24 struct {
 // IMPORTANT NOTE: When updating this struct make sure that
 // serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV25 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BoolFlag         `json:"browser"`
+	Worm       BoolFlag         `json:"worm"`
+	Domain     string           `json:"domain"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+}
+
+// serverConfigV26 is just like version '25', stores additionally
+// cache max use value in 'CacheConfig'.
+//
+// IMPORTANT NOTE: When updating this struct make sure that
+// serverConfig.ConfigDiff() is updated as necessary.
+type serverConfigV26 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
 	Version string `json:"version"`
 
 	// S3 API configuration.

cmd/crypto/doc.go

@@ -0,0 +1,116 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package crypto implements AWS S3 related cryptographic building blocks
+// for implementing Server-Side-Encryption (SSE-S3) and Server-Side-Encryption
+// with customer provided keys (SSE-C).
+//
+// All objects are encrypted with an unique and randomly generated 'ObjectKey'.
+// The ObjectKey itself is never stored in plaintext. Instead it is only stored
+// in a sealed from. The sealed 'ObjectKey' is created by encrypting the 'ObjectKey'
+// with an unique key-encryption-key. Given the correct key-encryption-key the
+// sealed 'ObjectKey' can be unsealed and the object can be decrypted.
+//
+//
+// ## SSE-C
+//
+// SSE-C computes the key-encryption-key from the client-provided key, an
+// initialization vector (IV) and the bucket/object path.
+//
+// 1. Encrypt:
+//       Input: ClientKey, bucket, object, metadata, object_data
+//       -              IV := Random({0,1}²⁵⁶)
+//       -       ObjectKey := SHA256(ClientKey || Random({0,1}²⁵⁶))
+//       -       KeyEncKey := HMAC-SHA256(ClientKey, IV || bucket || object)
+//       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
+//       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
+//       -        metadata <- IV
+//       -        metadata <- SealedKey
+//       Output: enc_object_data, metadata
+//
+// 2. Decrypt:
+//       Input: ClientKey, bucket, object, metadata, enc_object_data
+//       -          IV <- metadata
+//       -   SealedKey <- metadata
+//       -   KeyEncKey := HMAC-SHA256(ClientKey, IV || bucket || object)
+//       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
+//       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
+//       Output: object_data
+//
+//
+// ## SSE-S3
+//
+// SSE-S3 can use either a master key or a KMS as root-of-trust.
+// The en/decryption slightly depens upon which root-of-trust is used.
+//
+// ### SSE-S3 and single master key
+//
+// The master key is used to derive unique object- and key-encryption-keys.
+// SSE-S3 with a single master key works as SSE-C where the master key is
+// used as the client-provided key.
+//
+// 1. Encrypt:
+//       Input: MasterKey, bucket, object, metadata, object_data
+//       -              IV := Random({0,1}²⁵⁶)
+//       -       ObjectKey := SHA256(MasterKey || Random({0,1}²⁵⁶))
+//       -       KeyEncKey := HMAC-SHA256(MasterKey, IV || bucket || object)
+//       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
+//       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
+//       -        metadata <- IV
+//       -        metadata <- SealedKey
+//       Output: enc_object_data, metadata
+//
+// 2. Decrypt:
+//       Input: MasterKey, bucket, object, metadata, enc_object_data
+//       -          IV <- metadata
+//       -   SealedKey <- metadata
+//       -   KeyEncKey := HMAC-SHA256(MasterKey, IV || bucket || object)
+//       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
+//       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
+//       Output: object_data
+//
+//
+// ### SSE-S3 and KMS
+//
+// SSE-S3 requires that the KMS provides two functions:
+//       1.       Generate(KeyID) -> (Key, EncKey)
+//       2. Unseal(KeyID, EncKey) -> Key
+//
+// 1. Encrypt:
+//       Input: KeyID, bucket, object, metadata, object_data
+//       -     Key, EncKey := Generate(KeyID)
+//       -              IV := Random({0,1}²⁵⁶)
+//       -       ObjectKey := SHA256(Key, Random({0,1}²⁵⁶))
+//       -       KeyEncKey := HMAC-SHA256(Key, IV || bucket || object)
+//       -       SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
+//       - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
+//       -        metadata <- IV
+//       -        metadata <- KeyID
+//       -        metadata <- EncKey
+//       -        metadata <- SealedKey
+//       Output: enc_object_data, metadata
+//
+// 2. Decrypt:
+//       Input: bucket, object, metadata, enc_object_data
+//       -      KeyID  <- metadata
+//       -      EncKey <- metadata
+//       -          IV <- metadata
+//       -   SealedKey <- metadata
+//       -         Key := Unseal(KeyID, EncKey)
+//       -   KeyEncKey := HMAC-SHA256(Key, IV || bucket || object)
+//       -   ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
+//       - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
+//       Output: object_data
+//
+package crypto

cmd/crypto/error.go

@@ -0,0 +1,23 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import "errors"
+
+var (
+	// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
+	// is not supported.
+	ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
+)

cmd/crypto/header.go

@@ -0,0 +1,49 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"net/http"
+)
+
+// SSEHeader is the general AWS SSE HTTP header key.
+const SSEHeader = "X-Amz-Server-Side-Encryption"
+
+// SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
+// For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
+// For SSE-C  see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
+const SSEAlgorithmAES256 = "AES256"
+
+// S3 represents AWS SSE-S3. It provides functionality to handle
+// SSE-S3 requests.
+var S3 = s3{}
+
+type s3 struct{}
+
+// IsRequested returns true if the HTTP headers indicates that
+// the S3 client requests SSE-S3.
+func (s3) IsRequested(h http.Header) bool {
+	_, ok := h[SSEHeader]
+	return ok
+}
+
+// Parse parses the SSE-S3 related HTTP headers and checks
+// whether they contain valid values.
+func (s3) Parse(h http.Header) (err error) {
+	if h.Get(SSEHeader) != SSEAlgorithmAES256 {
+		err = ErrInvalidEncryptionMethod
+	}
+	return
+}

cmd/crypto/header_test.go

@@ -0,0 +1,56 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"net/http"
+	"testing"
+)
+
+var isRequestedTests = []struct {
+	Header   http.Header
+	Expected bool
+}{
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true},  // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true}, // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true},        // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false},  // 3
+}
+
+func TestS3IsRequested(t *testing.T) {
+	for i, test := range isRequestedTests {
+		if got := S3.IsRequested(test.Header); got != test.Expected {
+			t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
+		}
+	}
+}
+
+var parseTests = []struct {
+	Header      http.Header
+	ExpectedErr error
+}{
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, ExpectedErr: nil},                         // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 1
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, ExpectedErr: ErrInvalidEncryptionMethod},        // 2
+	{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, ExpectedErr: ErrInvalidEncryptionMethod},   // 3
+}
+
+func TestS3Parse(t *testing.T) {
+	for i, test := range parseTests {
+		if err := S3.Parse(test.Header); err != test.ExpectedErr {
+			t.Errorf("Test %d: Wanted '%v' but got '%v'", i, test.ExpectedErr, err)
+		}
+	}
+}

cmd/crypto/key.go

@@ -0,0 +1,93 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"context"
+	"crypto/hmac"
+	"crypto/rand"
+	"encoding/binary"
+	"errors"
+	"io"
+	"path/filepath"
+
+	"github.com/minio/minio/cmd/logger"
+	sha256 "github.com/minio/sha256-simd"
+	"github.com/minio/sio"
+)
+
+// ObjectKey is a 256 bit secret key used to encrypt the object.
+// It must never be stored in plaintext.
+type ObjectKey [32]byte
+
+// GenerateKey generates a unique ObjectKey from a 256 bit external key
+// and a source of randomness. If random is nil the default PRNG of system
+// (crypto/rand) is used.
+func GenerateKey(extKey [32]byte, random io.Reader) (key ObjectKey) {
+	if random == nil {
+		random = rand.Reader
+	}
+	var nonce [32]byte
+	if _, err := io.ReadFull(random, nonce[:]); err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to read enough randomness from the system"))
+	}
+	sha := sha256.New()
+	sha.Write(extKey[:])
+	sha.Write(nonce[:])
+	sha.Sum(key[:0])
+	return
+}
+
+// Seal encrypts the ObjectKey using the 256 bit external key and IV. The sealed
+// key is also cryptographically bound to the object's path (bucket/object).
+func (key ObjectKey) Seal(extKey, iv [32]byte, bucket, object string) []byte {
+	var sealedKey bytes.Buffer
+	mac := hmac.New(sha256.New, extKey[:])
+	mac.Write(iv[:])
+	mac.Write([]byte(filepath.Join(bucket, object)))
+
+	if n, err := sio.Encrypt(&sealedKey, bytes.NewReader(key[:]), sio.Config{Key: mac.Sum(nil)}); n != 64 || err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to generate sealed key"))
+	}
+	return sealedKey.Bytes()
+}
+
+// Unseal decrypts a sealed key using the 256 bit external key and IV. Since the sealed key
+// is cryptographically bound to the object's path the same bucket/object as during sealing
+// must be provided. On success the ObjectKey contains the decrypted sealed key.
+func (key *ObjectKey) Unseal(sealedKey []byte, extKey, iv [32]byte, bucket, object string) error {
+	var unsealedKey bytes.Buffer
+	mac := hmac.New(sha256.New, extKey[:])
+	mac.Write(iv[:])
+	mac.Write([]byte(filepath.Join(bucket, object)))
+
+	if n, err := sio.Decrypt(&unsealedKey, bytes.NewReader(sealedKey), sio.Config{Key: mac.Sum(nil)}); n != 32 || err != nil {
+		return err // TODO(aead): upgrade sio to use sio.Error
+	}
+	copy(key[:], unsealedKey.Bytes())
+	return nil
+}
+
+// DerivePartKey derives an unique 256 bit key from an ObjectKey and the part index.
+func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
+	var bin [4]byte
+	binary.LittleEndian.PutUint32(bin[:], id)
+
+	mac := hmac.New(sha256.New, key[:])
+	mac.Write(bin[:])
+	mac.Sum(partKey[:0])
+	return
+}

cmd/crypto/key_test.go

@@ -0,0 +1,127 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/hex"
+	"io"
+	"testing"
+)
+
+var shortRandom = func(limit int64) io.Reader { return io.LimitReader(rand.Reader, limit) }
+
+func recoverTest(i int, shouldPass bool, t *testing.T) {
+	if err := recover(); err == nil && !shouldPass {
+		t.Errorf("Test %d should fail but passed successfully", i)
+	} else if err != nil && shouldPass {
+		t.Errorf("Test %d should pass but failed: %v", i, err)
+	}
+}
+
+var generateKeyTests = []struct {
+	ExtKey     [32]byte
+	Random     io.Reader
+	ShouldPass bool
+}{
+	{ExtKey: [32]byte{}, Random: nil, ShouldPass: true},             // 0
+	{ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true},     // 1
+	{ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true}, // 2
+	// {ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3 See: https://github.com/minio/minio/issues/6064
+}
+
+func TestGenerateKey(t *testing.T) {
+	for i, test := range generateKeyTests {
+		func() {
+			defer recoverTest(i, test.ShouldPass, t)
+			key := GenerateKey(test.ExtKey, test.Random)
+			if [32]byte(key) == [32]byte{} {
+				t.Errorf("Test %d: generated key is zero key", i) // check that we generate random and unique key
+			}
+		}()
+	}
+}
+
+var sealUnsealKeyTests = []struct {
+	SealExtKey, SealIV     [32]byte
+	SealBucket, SealObject string
+
+	UnsealExtKey, UnsealIV     [32]byte
+	UnsealBucket, UnsealObject string
+
+	ShouldPass bool
+}{
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "object",
+		ShouldPass: true,
+	}, // 0
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{1}, UnsealIV: [32]byte{0}, UnsealBucket: "bucket", UnsealObject: "object",
+		ShouldPass: false,
+	}, // 1
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{1}, UnsealBucket: "bucket", UnsealObject: "object",
+		ShouldPass: false,
+	}, // 2
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "Bucket", UnsealObject: "object",
+		ShouldPass: false,
+	}, // 3
+	{
+		SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
+		UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "Object",
+		ShouldPass: false,
+	}, // 4
+}
+
+func TestSealUnsealKey(t *testing.T) {
+	for i, test := range sealUnsealKeyTests {
+		key := GenerateKey(test.SealExtKey, rand.Reader)
+		sealedKey := key.Seal(test.SealExtKey, test.SealIV, test.SealBucket, test.SealObject)
+		if err := key.Unseal(sealedKey, test.UnsealExtKey, test.UnsealIV, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass {
+			t.Errorf("Test %d should fail but passed successfully", i)
+		} else if err != nil && test.ShouldPass {
+			t.Errorf("Test %d should pass put failed: %v", i, err)
+		}
+	}
+}
+
+var derivePartKeyTest = []struct {
+	PartID  uint32
+	PartKey string
+}{
+	{PartID: 0, PartKey: "aa7855e13839dd767cd5da7c1ff5036540c9264b7a803029315e55375287b4af"},
+	{PartID: 1, PartKey: "a3e7181c6eed030fd52f79537c56c4d07da92e56d374ff1dd2043350785b37d8"},
+	{PartID: 10000, PartKey: "f86e65c396ed52d204ee44bd1a0bbd86eb8b01b7354e67a3b3ae0e34dd5bd115"},
+}
+
+func TestDerivePartKey(t *testing.T) {
+	var key ObjectKey
+	for i, test := range derivePartKeyTest {
+		expectedPartKey, err := hex.DecodeString(test.PartKey)
+		if err != nil {
+			t.Fatalf("Test %d failed to decode expected part-key: %v", i, err)
+		}
+		partKey := key.DerivePartKey(test.PartID)
+		if !bytes.Equal(partKey[:], expectedPartKey[:]) {
+			t.Errorf("Test %d derives wrong part-key: got '%s' want: '%s'", i, hex.EncodeToString(partKey[:]), test.PartKey)
+		}
+	}
+}

cmd/crypto/sse.go

@@ -0,0 +1,60 @@
+// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"context"
+	"errors"
+	"io"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/ioutil"
+	"github.com/minio/sio"
+)
+
+const (
+	// S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3.
+	S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
+	// S3KMSKeyID is the metadata key referencing the KMS key-id used to
+	// generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS.
+	S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
+	// S3KMSSealedKey is the metadata key referencing the encrypted key generated
+	// by KMS. It is only used for SSE-S3 + KMS.
+	S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
+)
+
+// EncryptSinglePart encrypts an io.Reader which must be the
+// the body of a single-part PUT request.
+func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {
+	r, err := sio.EncryptReader(r, sio.Config{MinVersion: sio.Version20, Key: key[:]})
+	if err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to encrypt io.Reader using object key"))
+	}
+	return r
+}
+
+// DecryptSinglePart decrypts an io.Writer which must an object
+// uploaded with the single-part PUT API. The offset and length
+// specify the requested range.
+func DecryptSinglePart(w io.Writer, offset, length int64, key ObjectKey) io.WriteCloser {
+	const PayloadSize = 1 << 16 // DARE 2.0
+	w = ioutil.LimitedWriter(w, offset%PayloadSize, length)
+
+	decWriter, err := sio.DecryptWriter(w, sio.Config{Key: key[:]})
+	if err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to decrypt io.Writer using object key"))
+	}
+	return decWriter
+}

cmd/disk-cache-config.go

@@ -25,6 +25,7 @@ import (
 type CacheConfig struct {
 	Drives  []string `json:"drives"`
 	Expiry  int      `json:"expiry"`
+	MaxUse  int      `json:"maxuse"`
 	Exclude []string `json:"exclude"`
 }
 

cmd/disk-cache.go

@@ -44,7 +44,6 @@ const (
 	// disk cache needs to have cacheSizeMultiplier * object size space free for a cache entry to be created.
 	cacheSizeMultiplier  = 100
 	cacheTrashDir        = "trash"
-	cacheMaxDiskUsagePct = 80 // in %
 	cacheCleanupInterval = 10 // in minutes
 )
 
@@ -239,7 +238,7 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 			pipeWriter.CloseWithError(err)
 			return
 		}
-		pipeWriter.Close() // Close writer explicitly signalling we wrote all data.
+		pipeWriter.Close() // Close writer explicitly signaling we wrote all data.
 	}()
 	err = dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(objInfo))
 	if err != nil {
@@ -839,7 +838,7 @@ func newCache(config CacheConfig) (*diskCache, error) {
 		if err := checkAtimeSupport(dir); err != nil {
 			return nil, errors.New("Atime support required for disk caching")
 		}
-		cache, err := newCacheFSObjects(dir, config.Expiry, cacheMaxDiskUsagePct)
+		cache, err := newCacheFSObjects(dir, config.Expiry, config.MaxUse)
 		if err != nil {
 			return nil, err
 		}

cmd/disk-cache_test.go

@@ -28,10 +28,10 @@ import (
 )
 
 // Initialize cache FS objects.
-func initCacheFSObjects(disk string, t *testing.T) (*cacheFSObjects, error) {
+func initCacheFSObjects(disk string, cacheMaxUse int, t *testing.T) (*cacheFSObjects, error) {
 	newTestConfig(globalMinioDefaultRegion)
 	var err error
-	obj, err := newCacheFSObjects(disk, globalCacheExpiry, 100)
+	obj, err := newCacheFSObjects(disk, globalCacheExpiry, cacheMaxUse)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -39,10 +39,10 @@ func initCacheFSObjects(disk string, t *testing.T) (*cacheFSObjects, error) {
 }
 
 // inits diskCache struct for nDisks
-func initDiskCaches(drives []string, t *testing.T) (*diskCache, error) {
+func initDiskCaches(drives []string, cacheMaxUse int, t *testing.T) (*diskCache, error) {
 	var cfs []*cacheFSObjects
 	for _, d := range drives {
-		obj, err := initCacheFSObjects(d, t)
+		obj, err := initCacheFSObjects(d, cacheMaxUse, t)
 		if err != nil {
 			return nil, err
 		}
@@ -59,7 +59,46 @@ func TestGetCacheFS(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		d, err := initDiskCaches(fsDirs, t)
+		d, err := initDiskCaches(fsDirs, 100, t)
+		if err != nil {
+			t.Fatal(err)
+		}
+		bucketName := "testbucket"
+		objectName := "testobject"
+		ctx := context.Background()
+		// find cache drive where object would be hashed
+		index := d.hashIndex(bucketName, objectName)
+		// turn off drive by setting online status to false
+		d.cfs[index].online = false
+		cfs, err := d.getCacheFS(ctx, bucketName, objectName)
+		if n == 1 && err == errDiskNotFound {
+			continue
+		}
+		if err != nil {
+			t.Fatal(err)
+		}
+		i := -1
+		for j, f := range d.cfs {
+			if f == cfs {
+				i = j
+				break
+			}
+		}
+		if i != (index+1)%n {
+			t.Fatalf("expected next cache location to be picked")
+		}
+	}
+}
+
+// test whether a drive being offline causes
+// getCacheFS to fetch next online drive
+func TestGetCacheFSMaxUse(t *testing.T) {
+	for n := 1; n < 10; n++ {
+		fsDirs, err := getRandomDisks(n)
+		if err != nil {
+			t.Fatal(err)
+		}
+		d, err := initDiskCaches(fsDirs, globalCacheMaxUse, t)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -141,7 +180,7 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	d, err := initDiskCaches(fsDirs, t)
+	d, err := initDiskCaches(fsDirs, 100, t)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -209,6 +248,87 @@ func TestDiskCache(t *testing.T) {
 	}
 }
 
+// Test diskCache with upper bound on max cache use.
+func TestDiskCacheMaxUse(t *testing.T) {
+	fsDirs, err := getRandomDisks(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	d, err := initDiskCaches(fsDirs, globalCacheMaxUse, t)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cache := d.cfs[0]
+	ctx := context.Background()
+	bucketName := "testbucket"
+	objectName := "testobject"
+	content := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	etag := "061208c10af71a30c6dcd6cf5d89f0fe"
+	contentType := "application/zip"
+	size := len(content)
+
+	httpMeta := make(map[string]string)
+	httpMeta["etag"] = etag
+	httpMeta["content-type"] = contentType
+
+	objInfo := ObjectInfo{}
+	objInfo.Bucket = bucketName
+	objInfo.Name = objectName
+	objInfo.Size = int64(size)
+	objInfo.ContentType = contentType
+	objInfo.ETag = etag
+	objInfo.UserDefined = httpMeta
+
+	byteReader := bytes.NewReader([]byte(content))
+	hashReader, err := hash.NewReader(byteReader, int64(size), "", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !cache.diskAvailable(int64(size)) {
+		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
+		if err != errDiskFull {
+			t.Fatal("Cache max-use limit violated.")
+		}
+	} else {
+		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
+		if err != nil {
+			t.Fatal(err)
+		}
+		cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !cache.Exists(ctx, bucketName, objectName) {
+			t.Fatal("Expected object to exist on cache")
+		}
+		if cachedObjInfo.ETag != objInfo.ETag {
+			t.Fatal("Expected ETag to match")
+		}
+		if cachedObjInfo.Size != objInfo.Size {
+			t.Fatal("Size mismatch")
+		}
+		if cachedObjInfo.ContentType != objInfo.ContentType {
+			t.Fatal("Cached content-type does not match")
+		}
+		writer := bytes.NewBuffer(nil)
+		err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if ccontent := writer.Bytes(); !bytes.Equal([]byte(content), ccontent) {
+			t.Errorf("wrong cached file content")
+		}
+		err = cache.Delete(ctx, bucketName, objectName)
+		if err != nil {
+			t.Errorf("object missing from cache")
+		}
+		online := cache.IsOnline()
+		if !online {
+			t.Errorf("expected cache drive to be online")
+		}
+	}
+}
+
 func TestIsCacheExcludeDirective(t *testing.T) {
 	testCases := []struct {
 		cacheControlOpt string

cmd/endpoint.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"os"
 	"path"
 	"path/filepath"
 	"runtime"
@@ -442,6 +443,8 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		return serverAddr, endpoints, setupType, err
 	}
 
+	updateDomainIPs(uniqueArgs)
+
 	setupType = DistXLSetupType
 	return serverAddr, endpoints, setupType, nil
 }
@@ -493,3 +496,22 @@ func GetRemotePeers(endpoints EndpointList) []string {
 
 	return peerSet.ToSlice()
 }
+
+// In federated and distributed setup, update IP addresses of the hosts passed in command line
+// if MINIO_PUBLIC_IPS are not set manually
+func updateDomainIPs(endPoints set.StringSet) {
+	_, dok := os.LookupEnv("MINIO_DOMAIN")
+	_, eok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
+	_, iok := os.LookupEnv("MINIO_PUBLIC_IPS")
+	if dok && eok && !iok {
+		globalDomainIPs = set.NewStringSet()
+		for e := range endPoints {
+			host, _, _ := net.SplitHostPort(e)
+			ipList, _ := getHostIP4(host)
+			remoteIPList := ipList.FuncMatch(func(ip string, matchString string) bool {
+				return !strings.HasPrefix(ip, "127.")
+			}, "")
+			globalDomainIPs.Add(remoteIPList.ToSlice()[0])
+		}
+	}
+}

cmd/fs-v1-helpers.go

@@ -178,7 +178,6 @@ func fsStatVolume(ctx context.Context, volume string) (os.FileInfo, error) {
 	}
 
 	if !fi.IsDir() {
-		logger.LogIf(ctx, errVolumeAccessDenied)
 		return nil, errVolumeAccessDenied
 	}
 
@@ -200,7 +199,7 @@ func osErrToFSFileErr(err error) error {
 		return errFileAccessDenied
 	}
 	if isSysErrNotDir(err) {
-		return errFileAccessDenied
+		return errFileNotFound
 	}
 	if isSysErrPathNotFound(err) {
 		return errFileNotFound
@@ -219,8 +218,7 @@ func fsStatDir(ctx context.Context, statDir string) (os.FileInfo, error) {
 		return nil, err
 	}
 	if !fi.IsDir() {
-		logger.LogIf(ctx, errFileAccessDenied)
-		return nil, errFileAccessDenied
+		return nil, errFileNotFound
 	}
 	return fi, nil
 }
@@ -245,8 +243,7 @@ func fsStatFile(ctx context.Context, statFile string) (os.FileInfo, error) {
 		return nil, err
 	}
 	if fi.IsDir() {
-		logger.LogIf(ctx, errFileAccessDenied)
-		return nil, errFileAccessDenied
+		return nil, errFileNotFound
 	}
 	return fi, nil
 }

cmd/fs-v1-helpers_test.go

@@ -136,7 +136,7 @@ func TestFSStats(t *testing.T) {
 			srcFSPath:   path,
 			srcVol:      "success-vol",
 			srcPath:     "path",
-			expectedErr: errFileAccessDenied,
+			expectedErr: errFileNotFound,
 		},
 		// Test case - 6.
 		// Test case with src path segment > 255.

cmd/fs-v1-multipart.go

@@ -264,7 +264,7 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
 			}
 			return
 		}
-		// Close writer explicitly signalling we wrote all data.
+		// Close writer explicitly signaling we wrote all data.
 		if gerr := srcInfo.Writer.Close(); gerr != nil {
 			logger.LogIf(ctx, gerr)
 			return

cmd/fs-v1.go

@@ -34,6 +34,7 @@ import (
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/mimedb"
+	"github.com/minio/minio/pkg/mountinfo"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -44,8 +45,6 @@ var defaultEtag = "00000000000000000000000000000000-1"
 type FSObjects struct {
 	// Disk usage metrics
 	totalUsed uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
-	// Disk usage running routine
-	usageRunning int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
 
 	// Path to be exported over S3 API.
 	fsPath string
@@ -64,6 +63,8 @@ type FSObjects struct {
 	// ListObjects pool management.
 	listPool *treeWalkPool
 
+	diskMount bool
+
 	appendFileMap   map[string]*fsAppendFile
 	appendFileMapMu sync.Mutex
 
@@ -109,6 +110,9 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 
 	var err error
 	if fsPath, err = getValidPath(fsPath); err != nil {
+		if err == errMinDiskSize {
+			return nil, err
+		}
 		return nil, uiErrUnableToWriteInBackend(err)
 	}
 
@@ -138,6 +142,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		nsMutex:       newNSLock(false),
 		listPool:      newTreeWalkPool(globalLookupTimeout),
 		appendFileMap: make(map[string]*fsAppendFile),
+		diskMount:     mountinfo.IsLikelyMountPoint(fsPath),
 	}
 
 	// Once the filesystem has initialized hold the read lock for
@@ -156,7 +161,10 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		return nil, uiErrUnableToReadFromBackend(err).Msg("Unable to initialize policy system")
 	}
 
-	go fs.diskUsage(globalServiceDoneCh)
+	if !fs.diskMount {
+		go fs.diskUsage(globalServiceDoneCh)
+	}
+
 	go fs.cleanupStaleMultipartUploads(ctx, globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh)
 
 	// Return successfully initialized object layer.
@@ -173,32 +181,40 @@ func (fs *FSObjects) Shutdown(ctx context.Context) error {
 
 // diskUsage returns du information for the posix path, in a continuous routine.
 func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
-	ticker := time.NewTicker(globalUsageCheckInterval)
-	defer ticker.Stop()
-
 	usageFn := func(ctx context.Context, entry string) error {
-		var fi os.FileInfo
-		var err error
-		if hasSuffix(entry, slashSeparator) {
-			fi, err = fsStatDir(ctx, entry)
-		} else {
-			fi, err = fsStatFile(ctx, entry)
+		if globalHTTPServer != nil {
+			// Wait at max 1 minute for an inprogress request
+			// before proceeding to count the usage.
+			waitCount := 60
+			// Any requests in progress, delay the usage.
+			for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
+				waitCount--
+				time.Sleep(1 * time.Second)
+			}
 		}
-		if err != nil {
-			return err
+
+		select {
+		case <-doneCh:
+			return errWalkAbort
+		default:
+			var fi os.FileInfo
+			var err error
+			if hasSuffix(entry, slashSeparator) {
+				fi, err = fsStatDir(ctx, entry)
+			} else {
+				fi, err = fsStatFile(ctx, entry)
+			}
+			if err != nil {
+				return err
+			}
+			atomic.AddUint64(&fs.totalUsed, uint64(fi.Size()))
 		}
-		atomic.AddUint64(&fs.totalUsed, uint64(fi.Size()))
 		return nil
 	}
 
-	// Check if disk usage routine is running, if yes then return.
-	if atomic.LoadInt32(&fs.usageRunning) == 1 {
-		return
-	}
-	atomic.StoreInt32(&fs.usageRunning, 1)
-	defer atomic.StoreInt32(&fs.usageRunning, 0)
-
-	if err := getDiskUsage(context.Background(), fs.fsPath, usageFn); err != nil {
+	// Return this routine upon errWalkAbort, continue for any other error on purpose
+	// so that we can start the routine freshly in another 12 hours.
+	if err := getDiskUsage(context.Background(), fs.fsPath, usageFn); err == errWalkAbort {
 		return
 	}
 
@@ -206,15 +222,20 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
 		select {
 		case <-doneCh:
 			return
-		case <-ticker.C:
-			// Check if disk usage routine is running, if yes let it finish.
-			if atomic.LoadInt32(&fs.usageRunning) == 1 {
-				continue
-			}
-			atomic.StoreInt32(&fs.usageRunning, 1)
-
+		case <-time.After(globalUsageCheckInterval):
 			var usage uint64
 			usageFn = func(ctx context.Context, entry string) error {
+				if globalHTTPServer != nil {
+					// Wait at max 1 minute for an inprogress request
+					// before proceeding to count the usage.
+					waitCount := 60
+					// Any requests in progress, delay the usage.
+					for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
+						waitCount--
+						time.Sleep(1 * time.Second)
+					}
+				}
+
 				var fi os.FileInfo
 				var err error
 				if hasSuffix(entry, slashSeparator) {
@@ -230,11 +251,8 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
 			}
 
 			if err := getDiskUsage(context.Background(), fs.fsPath, usageFn); err != nil {
-				atomic.StoreInt32(&fs.usageRunning, 0)
 				continue
 			}
-
-			atomic.StoreInt32(&fs.usageRunning, 0)
 			atomic.StoreUint64(&fs.totalUsed, usage)
 		}
 	}
@@ -242,8 +260,16 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
 
 // StorageInfo - returns underlying storage statistics.
 func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo {
+	di, err := getDiskInfo(fs.fsPath)
+	if err != nil {
+		return StorageInfo{}
+	}
+	used := di.Total - di.Free
+	if !fs.diskMount {
+		used = atomic.LoadUint64(&fs.totalUsed)
+	}
 	storageInfo := StorageInfo{
-		Used: atomic.LoadUint64(&fs.totalUsed),
+		Used: used,
 	}
 	storageInfo.Backend.Type = FS
 	return storageInfo
@@ -479,7 +505,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 			}
 			return
 		}
-		// Close writer explicitly signalling we wrote all data.
+		// Close writer explicitly signaling we wrote all data.
 		if gerr := srcInfo.Writer.Close(); gerr != nil {
 			logger.LogIf(ctx, gerr)
 			return
@@ -695,7 +721,7 @@ func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object s
 	}
 
 	if _, err := fs.statBucketDir(ctx, bucket); err != nil {
-		return oi, toObjectErr(err, bucket)
+		return oi, err
 	}
 
 	if strings.HasSuffix(object, slashSeparator) && !fs.isObjectDir(bucket, object) {
@@ -1214,7 +1240,7 @@ func (fs *FSObjects) SetBucketPolicy(ctx context.Context, bucket string, policy
 
 // GetBucketPolicy will get policy on bucket
 func (fs *FSObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
-	return GetPolicyConfig(fs, bucket)
+	return getPolicyConfig(fs, bucket)
 }
 
 // DeleteBucketPolicy deletes all policies on bucket

cmd/gateway-common.go

@@ -154,6 +154,7 @@ func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
 		UserDefined:     userDefined,
 		ContentType:     oi.ContentType,
 		ContentEncoding: oi.Metadata.Get("Content-Encoding"),
+		StorageClass:    oi.StorageClass,
 	}
 }
 

cmd/gateway-main.go

@@ -34,7 +34,7 @@ import (
 )
 
 func init() {
-	logger.Init(GOPATH)
+	logger.Init(GOPATH, GOROOT)
 	logger.RegisterUIError(fmtError)
 }
 
@@ -197,7 +197,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		getCert = globalTLSCerts.GetCertificate
 	}
 
-	globalHTTPServer = xhttp.NewServer([]string{gatewayAddr}, registerHandlers(router, globalHandlers...), getCert)
+	globalHTTPServer = xhttp.NewServer([]string{gatewayAddr}, criticalErrorHandler{registerHandlers(router, globalHandlers...)}, getCert)
 	globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
 	globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes
 	go func() {
@@ -215,6 +215,13 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.FatalIf(err, "Unable to initialize gateway backend")
 	}
 
+	if gw.Name() != "nas" {
+		// Initialize policy sys for all gateways. NAS gateway already
+		// initializes policy sys internally, avoid double initialization.
+		// Additionally also don't block the initialization of gateway.
+		go globalPolicySys.Init(newObject)
+	}
+
 	// Once endpoints are finalized, initialize the new object api.
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = newObject

cmd/gateway/azure/gateway-azure.go

@@ -82,6 +82,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
 EXAMPLES:
   1. Start minio gateway server for Azure Blob Storage backend.
@@ -100,6 +101,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}}
 `
 
@@ -238,6 +240,8 @@ func s3MetaToAzureProperties(ctx context.Context, s3Metadata map[string]string)
 			props.ContentMD5 = v
 		case k == "Content-Type":
 			props.ContentType = v
+		case k == "Content-Language":
+			props.ContentLanguage = v
 		}
 	}
 	return blobMeta, props, nil
@@ -292,6 +296,9 @@ func azurePropertiesToS3Meta(meta storage.BlobMetadata, props storage.BlobProper
 	if props.ContentType != "" {
 		s3Metadata["Content-Type"] = props.ContentType
 	}
+	if props.ContentLanguage != "" {
+		s3Metadata["Content-Language"] = props.ContentLanguage
+	}
 	return s3Metadata
 }
 

cmd/gateway/b2/gateway-b2.go

@@ -72,6 +72,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
 EXAMPLES:
   1. Start minio gateway server for B2 backend.
@@ -85,6 +86,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}}
 `
 	minio.RegisterGatewayCommand(cli.Command{

cmd/gateway/gcs/gateway-gcs.go

@@ -118,6 +118,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
   GCS credentials file:
      GOOGLE_APPLICATION_CREDENTIALS: Path to credentials.json
@@ -137,6 +138,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}} mygcsprojectid
 `
 

cmd/gateway/manta/gateway-manta.go

@@ -78,6 +78,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
 EXAMPLES:
   1. Start minio gateway server for Manta Object Storage backend.
@@ -102,6 +103,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}}
 `
 

cmd/gateway/nas/gateway-nas.go

@@ -17,12 +17,9 @@
 package nas
 
 import (
-	"context"
-
 	"github.com/minio/cli"
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/policy"
 )
 
 const (
@@ -57,6 +54,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
 EXAMPLES:
   1. Start minio gateway server for NAS backend.
@@ -70,6 +68,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}} /shared/nasvol
 `
 
@@ -126,8 +125,3 @@ type nasObjects struct {
 func (l *nasObjects) IsNotificationSupported() bool {
 	return false
 }
-
-// GetBucketPolicy will get policy on bucket
-func (l *nasObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
-	return minio.GetPolicyConfig(l, bucket)
-}

cmd/gateway/oss/gateway-oss.go

@@ -74,6 +74,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
 EXAMPLES:
   1. Start minio gateway server for Aliyun OSS backend.
@@ -92,6 +93,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}}
 `
 

cmd/gateway/s3/gateway-s3.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -65,6 +65,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
 EXAMPLES:
   1. Start minio gateway server for AWS S3 backend.
@@ -83,6 +84,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}}
 `
 
@@ -97,12 +99,16 @@ EXAMPLES:
 
 // Handler for 'minio gateway s3' command line.
 func s3GatewayMain(ctx *cli.Context) {
-	// Validate gateway arguments.
-	host := ctx.Args().First()
-	// Validate gateway arguments.
-	logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
+	args := ctx.Args()
+	if !ctx.Args().Present() {
+		args = cli.Args{"https://s3.amazonaws.com"}
+	}
 
-	minio.StartGateway(ctx, &S3{host})
+	// Validate gateway arguments.
+	logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), args.First()), "Invalid argument")
+
+	// Start the gateway..
+	minio.StartGateway(ctx, &S3{args.First()})
 }
 
 // S3 implements Gateway.
@@ -115,34 +121,46 @@ func (g *S3) Name() string {
 	return s3Backend
 }
 
-// NewGatewayLayer returns s3 ObjectLayer.
-func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
-	var err error
-	var endpoint string
-	var secure = true
+// newS3 - Initializes a new client by auto probing S3 server signature.
+func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
+	if url == "" {
+		url = "https://s3.amazonaws.com"
+	}
 
-	// Validate host parameters.
-	if g.host != "" {
-		// Override default params if the host is provided
-		endpoint, secure, err = minio.ParseGatewayEndpoint(g.host)
+	// Override default params if the host is provided
+	endpoint, secure, err := minio.ParseGatewayEndpoint(url)
+	if err != nil {
+		return nil, err
+	}
+
+	clnt, err := miniogo.NewV4(endpoint, accessKey, secretKey, secure)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err = clnt.BucketExists("probe-bucket-sign"); err != nil {
+		clnt, err = miniogo.NewV2(endpoint, accessKey, secretKey, secure)
 		if err != nil {
 			return nil, err
 		}
+		if _, err = clnt.BucketExists("probe-bucket-sign"); err != nil {
+			return nil, err
+		}
 	}
 
-	// Default endpoint parameters
-	if endpoint == "" {
-		endpoint = "s3.amazonaws.com"
-	}
+	return &miniogo.Core{Client: clnt}, nil
+}
 
-	// Initialize minio client object.
-	client, err := miniogo.NewCore(endpoint, creds.AccessKey, creds.SecretKey, secure)
+// NewGatewayLayer returns s3 ObjectLayer.
+func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
+	// Probe S3 signature with input credentials.
+	clnt, err := newS3(g.host, creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		return nil, err
 	}
 
 	return &s3Objects{
-		Client: client,
+		Client: clnt,
 	}, nil
 }
 
@@ -255,7 +273,7 @@ func (l *s3Objects) ListObjects(ctx context.Context, bucket string, prefix strin
 
 // ListObjectsV2 lists all blobs in S3 bucket filtered by prefix
 func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, e error) {
-	result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys)
+	result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys, startAfter)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return loi, minio.ErrorRespToObjectError(err, bucket)

cmd/gateway/sia/gateway-sia.go

@@ -79,6 +79,7 @@ ENVIRONMENT VARIABLES: (Default values in parenthesis)
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
   SIA_TEMP_DIR:        The name of the local Sia temporary storage directory. (.sia_temp)
   SIA_API_PASSWORD:    API password for Sia daemon. (default is empty)
@@ -91,6 +92,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}}
 `
 

cmd/generic-handlers.go

@@ -19,13 +19,19 @@ package cmd
 import (
 	"bufio"
 	"context"
+	"fmt"
 	"net"
 	"net/http"
 	"strings"
 	"time"
 
+	"github.com/minio/minio-go/pkg/set"
+
+	etcd "github.com/coreos/etcd/client"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/dns"
+	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/sys"
 	"github.com/rs/cors"
 	"golang.org/x/time/rate"
@@ -611,14 +617,81 @@ func (h pathValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	h.handler.ServeHTTP(w, r)
 }
 
+// To forward the path style requests on a bucket to the right
+// configured server, bucket to IP configuration is obtained
+// from centralized etcd configuration service.
+type bucketForwardingHandler struct {
+	fwd     *handlers.Forwarder
+	handler http.Handler
+}
+
+func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if globalDNSConfig == nil || globalDomainName == "" || guessIsBrowserReq(r) || guessIsHealthCheckReq(r) || guessIsMetricsReq(r) || guessIsRPCReq(r) {
+		f.handler.ServeHTTP(w, r)
+		return
+	}
+	bucket, object := urlPath2BucketObjectName(r.URL.Path)
+	// ListBucket requests should be handled at current endpoint as
+	// all buckets data can be fetched from here.
+	if r.Method == http.MethodGet && bucket == "" && object == "" {
+		f.handler.ServeHTTP(w, r)
+		return
+	}
+
+	// MakeBucket requests should be handled at current endpoint
+	if r.Method == http.MethodPut && bucket != "" && object == "" {
+		f.handler.ServeHTTP(w, r)
+		return
+	}
+
+	// CopyObject requests should be handled at current endpoint as path style
+	// requests have target bucket and object in URI and source details are in
+	// header fields
+	if r.Method == http.MethodPut && r.Header.Get("X-Amz-Copy-Source") != "" {
+		f.handler.ServeHTTP(w, r)
+		return
+	}
+	sr, err := globalDNSConfig.Get(bucket)
+	if err != nil {
+		if etcd.IsKeyNotFound(err) || err == dns.ErrNoEntriesFound {
+			writeErrorResponse(w, ErrNoSuchBucket, r.URL)
+		} else {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		}
+		return
+	}
+	if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
+		host, port := getRandomHostPort(sr)
+		r.URL.Scheme = "http"
+		if globalIsSSL {
+			r.URL.Scheme = "https"
+		}
+		r.URL.Host = fmt.Sprintf("%s:%d", host, port)
+		f.fwd.ServeHTTP(w, r)
+		return
+	}
+	f.handler.ServeHTTP(w, r)
+}
+
+// setBucketForwardingHandler middleware forwards the path style requests
+// on a bucket to the right bucket location, bucket to IP configuration
+// is obtained from centralized etcd configuration service.
+func setBucketForwardingHandler(h http.Handler) http.Handler {
+	fwd := handlers.NewForwarder(&handlers.Forwarder{
+		PassHost:     true,
+		RoundTripper: NewCustomHTTPTransport(),
+	})
+	return bucketForwardingHandler{fwd, h}
+}
+
 // setRateLimitHandler middleware limits the throughput to h using a
 // rate.Limiter token bucket configured with maxOpenFileLimit and
 // burst set to 1. The request will idle for up to 1*time.Second.
 // If the limiter detects the deadline will be exceeded, the request is
-// cancelled immediately.
+// canceled immediately.
 func setRateLimitHandler(h http.Handler) http.Handler {
 	_, maxLimit, err := sys.GetMaxOpenFileLimit()
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to get maximum open file limit", context.Background())
 	// Burst value is set to 1 to allow only maxOpenFileLimit
 	// requests to happen at once.
 	l := rate.NewLimiter(rate.Limit(maxLimit), 1)
@@ -665,3 +738,20 @@ func (s securityHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
 	header.Set("Content-Security-Policy", "block-all-mixed-content") // prevent mixed (HTTP / HTTPS content)
 	s.handler.ServeHTTP(w, r)
 }
+
+// criticalErrorHandler handles critical server failures caused by
+// `panic(logger.ErrCritical)` as done by `logger.CriticalIf`.
+//
+// It should be always the first / highest HTTP handler.
+type criticalErrorHandler struct{ handler http.Handler }
+
+func (h criticalErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	defer func() {
+		if err := recover(); err == logger.ErrCritical { // handle
+			writeErrorResponse(w, ErrInternalError, r.URL)
+		} else if err != nil {
+			panic(err) // forward other panic calls
+		}
+	}()
+	h.handler.ServeHTTP(w, r)
+}

cmd/globals.go

@@ -22,11 +22,15 @@ import (
 	"runtime"
 	"time"
 
+	"github.com/minio/minio-go/pkg/set"
+
+	etcd "github.com/coreos/etcd/client"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/fatih/color"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/certs"
+	"github.com/minio/minio/pkg/dns"
 )
 
 // minio configuration related constants.
@@ -159,7 +163,8 @@ var (
 	globalPublicCerts []*x509.Certificate
 
 	globalIsEnvDomainName bool
-	globalDomainName      string // Root domain for virtual host style requests
+	globalDomainName      string        // Root domain for virtual host style requests
+	globalDomainIPs       set.StringSet // Root domain IP address(s) for a distributed Minio deployment
 
 	globalListingTimeout   = newDynamicTimeout( /*30*/ 600*time.Second /*5*/, 600*time.Second) // timeout for listing related ops
 	globalObjectTimeout    = newDynamicTimeout( /*1*/ 10*time.Minute /*10*/, 600*time.Second)  // timeout for Object API related ops
@@ -174,17 +179,23 @@ var (
 	// Set to store standard storage class
 	globalStandardStorageClass storageClass
 
-	globalIsEnvWORM   bool
+	globalIsEnvWORM bool
+	// Is worm enabled
 	globalWORMEnabled bool
 
 	// Is Disk Caching set up
 	globalIsDiskCacheEnabled bool
+
 	// Disk cache drives
 	globalCacheDrives []string
+
 	// Disk cache excludes
 	globalCacheExcludes []string
+
 	// Disk cache expiry
 	globalCacheExpiry = 90
+	// Max allowed disk cache percentage
+	globalCacheMaxUse = 80
 
 	// RPC V1 - Initial version
 	// RPC V2 - format.json XL version changed to 2
@@ -192,12 +203,18 @@ var (
 	// Current RPC version
 	globalRPCAPIVersion = RPCVersion{3, 0, 0}
 
-	// Add new variable global values here.
+	// Allocated etcd endpoint for config and bucket DNS.
+	globalEtcdClient etcd.Client
+
+	// Allocated DNS config wrapper over etcd client.
+	globalDNSConfig dns.Config
 
 	// Default usage check interval value.
 	globalDefaultUsageCheckInterval = 12 * time.Hour // 12 hours
 	// Usage check interval value.
 	globalUsageCheckInterval = globalDefaultUsageCheckInterval
+
+	// Add new variable global values here.
 )
 
 // global colors.

cmd/http/server.go

@@ -59,7 +59,12 @@ type Server struct {
 	listenerMutex          *sync.Mutex   // to guard 'listener' field.
 	listener               *httpListener // HTTP listener for all 'Addrs' field.
 	inShutdown             uint32        // indicates whether the server is in shutdown or not
-	requestCount           int32         // counter holds no. of request in process.
+	requestCount           int32         // counter holds no. of request in progress.
+}
+
+// GetRequestCount - returns number of request in progress.
+func (srv *Server) GetRequestCount() int32 {
+	return atomic.LoadInt32(&srv.requestCount)
 }
 
 // Start - start HTTP server

cmd/lock-rpc-server.go

@@ -124,7 +124,10 @@ func (l *lockRPCReceiver) lockMaintenance(interval time.Duration) {
 	for _, nlrip := range nlripLongLived {
 		// Initialize client based on the long live locks.
 		host, err := xnet.ParseHost(nlrip.lri.node)
-		logger.CriticalIf(context.Background(), err)
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			continue
+		}
 		c, err := NewLockRPCClient(host)
 		if err != nil {
 			logger.LogIf(context.Background(), err)
@@ -183,7 +186,7 @@ func NewLockRPCServer() (*xrpc.Server, error) {
 // Register distributed NS lock handlers.
 func registerDistNSLockRouter(router *mux.Router) {
 	rpcServer, err := NewLockRPCServer()
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to initialize Lock RPC Server", context.Background())
 
 	// Start lock maintenance from all lock servers.
 	go startLockMaintenance(globalLockServer)

cmd/logger/logger.go

@@ -52,6 +52,25 @@ var matchingFuncNames = [...]string{
 	"http.HandlerFunc.ServeHTTP",
 	"cmd.serverMain",
 	"cmd.StartGateway",
+	"cmd.(*webAPIHandlers).ListBuckets",
+	"cmd.(*webAPIHandlers).MakeBucket",
+	"cmd.(*webAPIHandlers).DeleteBucket",
+	"cmd.(*webAPIHandlers).ListObjects",
+	"cmd.(*webAPIHandlers).RemoveObject",
+	"cmd.(*webAPIHandlers).Login",
+	"cmd.(*webAPIHandlers).GenerateAuth",
+	"cmd.(*webAPIHandlers).SetAuth",
+	"cmd.(*webAPIHandlers).GetAuth",
+	"cmd.(*webAPIHandlers).CreateURLToken",
+	"cmd.(*webAPIHandlers).Upload",
+	"cmd.(*webAPIHandlers).Download",
+	"cmd.(*webAPIHandlers).DownloadZip",
+	"cmd.(*webAPIHandlers).GetBucketPolicy",
+	"cmd.(*webAPIHandlers).ListAllBucketPolicies",
+	"cmd.(*webAPIHandlers).SetBucketPolicy",
+	"cmd.(*webAPIHandlers).PresignedGet",
+	"cmd.(*webAPIHandlers).ServerInfo",
+	"cmd.(*webAPIHandlers).StorageInfo",
 	// add more here ..
 }
 
@@ -142,22 +161,24 @@ func RegisterUIError(f func(string, error, bool) string) {
 // and GOROOT directories. Also append github.com/minio/minio
 // This is done to clean up the filename, when stack trace is
 // displayed when an error happens.
-func Init(goPath string) {
+func Init(goPath string, goRoot string) {
 
 	var goPathList []string
+	var goRootList []string
 	var defaultgoPathList []string
+	var defaultgoRootList []string
+	pathSeperator := ":"
 	// Add all possible GOPATH paths into trimStrings
 	// Split GOPATH depending on the OS type
 	if runtime.GOOS == "windows" {
-		goPathList = strings.Split(goPath, ";")
-		defaultgoPathList = strings.Split(build.Default.GOPATH, ";")
-	} else {
-		// All other types of OSs
-		goPathList = strings.Split(goPath, ":")
-		defaultgoPathList = strings.Split(build.Default.GOPATH, ":")
-
+		pathSeperator = ";"
 	}
 
+	goPathList = strings.Split(goPath, pathSeperator)
+	goRootList = strings.Split(goRoot, pathSeperator)
+	defaultgoPathList = strings.Split(build.Default.GOPATH, pathSeperator)
+	defaultgoRootList = strings.Split(build.Default.GOROOT, pathSeperator)
+
 	// Add trim string "{GOROOT}/src/" into trimStrings
 	trimStrings = []string{filepath.Join(runtime.GOROOT(), "src") + string(filepath.Separator)}
 
@@ -167,10 +188,21 @@ func Init(goPath string) {
 		trimStrings = append(trimStrings, filepath.Join(goPathString, "src")+string(filepath.Separator))
 	}
 
+	for _, goRootString := range goRootList {
+		trimStrings = append(trimStrings, filepath.Join(goRootString, "src")+string(filepath.Separator))
+	}
+
 	for _, defaultgoPathString := range defaultgoPathList {
 		trimStrings = append(trimStrings, filepath.Join(defaultgoPathString, "src")+string(filepath.Separator))
 	}
 
+	for _, defaultgoRootString := range defaultgoRootList {
+		trimStrings = append(trimStrings, filepath.Join(defaultgoRootString, "src")+string(filepath.Separator))
+	}
+
+	// Remove duplicate entries.
+	trimStrings = uniqueEntries(trimStrings)
+
 	// Add "github.com/minio/minio" as the last to cover
 	// paths like "{GOROOT}/src/github.com/minio/minio"
 	// and "{GOPATH}/src/github.com/minio/minio"
@@ -200,7 +232,7 @@ func getTrace(traceLevel int) []string {
 	var trace []string
 	pc, file, lineNumber, ok := runtime.Caller(traceLevel)
 
-	for ok {
+	for ok && file != "" {
 		// Clean up the common prefixes
 		file = trimTrace(file)
 		// Get the function name
@@ -330,13 +362,15 @@ func LogIf(ctx context.Context, err error) {
 	fmt.Println(output)
 }
 
-// CriticalIf :
-// Like LogIf with exit
-// It'll be called for fatal error conditions during run-time
+// ErrCritical is the value panic'd whenever CriticalIf is called.
+var ErrCritical struct{}
+
+// CriticalIf logs the provided error on the console. It fails the
+// current go-routine by causing a `panic(ErrCritical)`.
 func CriticalIf(ctx context.Context, err error) {
 	if err != nil {
 		LogIf(ctx, err)
-		os.Exit(1)
+		panic(ErrCritical)
 	}
 }
 

cmd/logger/utils.go

@@ -50,3 +50,16 @@ func ansiSaveAttributes() {
 func ansiRestoreAttributes() {
 	ansiEscape("8")
 }
+
+func uniqueEntries(paths []string) []string {
+	found := map[string]bool{}
+	unqiue := []string{}
+
+	for v := range paths {
+		if _, ok := found[paths[v]]; !ok {
+			found[paths[v]] = true
+			unqiue = append(unqiue, paths[v])
+		}
+	}
+	return unqiue
+}

cmd/namespace-lock.go

@@ -86,9 +86,9 @@ func newDsyncNodes(endpoints EndpointList) (clnts []dsync.NetLocker, myNode int)
 			locker = &(receiver.ll)
 		} else {
 			host, err := xnet.ParseHost(endpoint.Host)
-			logger.CriticalIf(context.Background(), err)
+			logger.FatalIf(err, "Unable to parse Lock RPC Host", context.Background())
 			locker, err = NewLockRPCClient(host)
-			logger.CriticalIf(context.Background(), err)
+			logger.FatalIf(err, "Unable to initialize Lock RPC Client", context.Background())
 		}
 
 		clnts = append(clnts, locker)

cmd/namespace-lock_test.go

@@ -226,7 +226,7 @@ func TestNamespaceForceUnlockTest(t *testing.T) {
 
 	select {
 	case <-ch:
-		// Signalled so all is fine.
+		// Signaled so all is fine.
 		break
 
 	case <-time.After(100 * time.Millisecond):

cmd/object-api-utils.go

@@ -20,12 +20,15 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
+	"math/rand"
 	"path"
 	"runtime"
 	"strings"
+	"time"
 	"unicode/utf8"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/skyrings/skyring-common/tools/uuid"
 )
 
@@ -277,6 +280,22 @@ func isMinioReservedBucket(bucketName string) bool {
 	return bucketName == minioReservedBucket
 }
 
+// returns a slice of hosts by reading a slice of DNS records
+func getHostsSlice(records []dns.SrvRecord) []string {
+	var hosts []string
+	for _, r := range records {
+		hosts = append(hosts, r.Host)
+	}
+	return hosts
+}
+
+// returns a random host (and corresponding port) from a slice of DNS records
+func getRandomHostPort(records []dns.SrvRecord) (string, int) {
+	rand.Seed(time.Now().Unix())
+	srvRecord := records[rand.Intn(len(records))]
+	return srvRecord.Host, srvRecord.Port
+}
+
 // byBucketName is a collection satisfying sort.Interface.
 type byBucketName []BucketInfo
 

cmd/object-handlers.go

@@ -32,7 +32,9 @@ import (
 	"strconv"
 
 	"github.com/gorilla/mux"
+	miniogo "github.com/minio/minio-go"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/ioutil"
@@ -536,13 +538,73 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	// Copy source object to destination, if source and destination
-	// object is same then only metadata is updated.
-	objInfo, err := objectAPI.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo)
-	if err != nil {
-		pipeWriter.CloseWithError(err)
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
+	var objInfo ObjectInfo
+
+	// Checks if a remote putobject call is needed for CopyObject operation
+	// 1. If source and destination bucket names are same, it means no call needed to etcd to get destination info
+	// 2. If destination bucket doesn't exist locally, only then a etcd call is needed
+	var isRemoteCallRequired = func(ctx context.Context, src, dst string, objAPI ObjectLayer) bool {
+		if src == dst {
+			return false
+		}
+		_, berr := objAPI.GetBucketInfo(ctx, dst)
+		return berr == toObjectErr(errVolumeNotFound, dst)
+	}
+
+	// Returns a minio-go Client configured to access remote host described by destDNSRecord
+	// Applicable only in a federated deployment
+	var getRemoteInstanceClient = func(host string, port int) (*miniogo.Core, error) {
+		// In a federated deployment, all the instances share config files and hence expected to have same
+		// credentials. So, access current instances creds and use it to create client for remote instance
+		endpoint := net.JoinHostPort(host, strconv.Itoa(port))
+		accessKey := globalServerConfig.Credential.AccessKey
+		secretKey := globalServerConfig.Credential.SecretKey
+		return miniogo.NewCore(endpoint, accessKey, secretKey, globalIsSSL)
+	}
+
+	if isRemoteCallRequired(ctx, srcBucket, dstBucket, objectAPI) {
+		if globalDNSConfig == nil {
+			writeErrorResponse(w, ErrNoSuchBucket, r.URL)
+			return
+		}
+		var dstRecords []dns.SrvRecord
+		if dstRecords, err = globalDNSConfig.Get(dstBucket); err == nil {
+			go func() {
+				if gerr := objectAPI.GetObject(ctx, srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag); gerr != nil {
+					pipeWriter.CloseWithError(gerr)
+					writeErrorResponse(w, ErrInternalError, r.URL)
+					return
+				}
+				// Close writer explicitly to indicate data has been written
+				srcInfo.Writer.Close()
+			}()
+
+			// Send PutObject request to appropriate instance (in federated deployment)
+			host, port := getRandomHostPort(dstRecords)
+			client, rerr := getRemoteInstanceClient(host, port)
+			if rerr != nil {
+				pipeWriter.CloseWithError(rerr)
+				writeErrorResponse(w, ErrInternalError, r.URL)
+				return
+			}
+			remoteObjInfo, rerr := client.PutObject(dstBucket, dstObject, srcInfo.Reader, srcInfo.Size, "", "", srcInfo.UserDefined)
+			if rerr != nil {
+				pipeWriter.CloseWithError(rerr)
+				writeErrorResponse(w, ErrInternalError, r.URL)
+				return
+			}
+			objInfo.ETag = remoteObjInfo.ETag
+			objInfo.ModTime = remoteObjInfo.LastModified
+		}
+	} else {
+		// Copy source object to destination, if source and destination
+		// object is same then only metadata is updated.
+		objInfo, err = objectAPI.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo)
+		if err != nil {
+			pipeWriter.CloseWithError(err)
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+			return
+		}
 	}
 
 	pipeReader.Close()

cmd/peer-rpc-client.go

@@ -166,9 +166,9 @@ func makeRemoteRPCClients(endpoints EndpointList) map[xnet.Host]*PeerRPCClient {
 	peerRPCClientMap := make(map[xnet.Host]*PeerRPCClient)
 	for _, hostStr := range GetRemotePeers(endpoints) {
 		host, err := xnet.ParseHost(hostStr)
-		logger.CriticalIf(context.Background(), err)
+		logger.FatalIf(err, "Unable to parse peer RPC Host", context.Background())
 		rpcClient, err := NewPeerRPCClient(host)
-		logger.CriticalIf(context.Background(), err)
+		logger.FatalIf(err, "Unable to parse peer RPC Client", context.Background())
 		peerRPCClientMap[*host] = rpcClient
 	}
 

cmd/peer-rpc-server.go

@@ -178,7 +178,7 @@ func (receiver *peerRPCReceiver) SetCredentials(args *SetCredentialsArgs, reply
 	prevCred := globalServerConfig.SetCredential(args.Credentials)
 
 	// Save credentials to config file
-	if err := globalServerConfig.Save(); err != nil {
+	if err := globalServerConfig.Save(getConfigFile()); err != nil {
 		// As saving configurstion failed, restore previous credential in memory.
 		globalServerConfig.SetCredential(prevCred)
 
@@ -201,7 +201,7 @@ func NewPeerRPCServer() (*xrpc.Server, error) {
 // registerPeerRPCRouter - creates and registers Peer RPC server and its router.
 func registerPeerRPCRouter(router *mux.Router) {
 	rpcServer, err := NewPeerRPCServer()
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to initialize peer RPC Server", context.Background())
 	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
 	subrouter.Path(peerServiceSubPath).Handler(rpcServer)
 }

cmd/policy.go

@@ -99,7 +99,7 @@ func (sys *PolicySys) refresh(objAPI ObjectLayer) error {
 	}
 	sys.removeDeletedBuckets(buckets)
 	for _, bucket := range buckets {
-		config, err := GetPolicyConfig(objAPI, bucket.Name)
+		config, err := objAPI.GetBucketPolicy(context.Background(), bucket.Name)
 		if err != nil {
 			if _, ok := err.(BucketPolicyNotFound); ok {
 				sys.Remove(bucket.Name)
@@ -187,8 +187,8 @@ func getConditionValues(request *http.Request, locationConstraint string) map[st
 	return args
 }
 
-// GetPolicyConfig - get policy config for given bucket name.
-func GetPolicyConfig(objAPI ObjectLayer, bucketName string) (*policy.Policy, error) {
+// getPolicyConfig - get policy config for given bucket name.
+func getPolicyConfig(objAPI ObjectLayer, bucketName string) (*policy.Policy, error) {
 	// Construct path to policy.json for the given bucket.
 	configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
 

cmd/posix.go

@@ -34,6 +34,7 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/mountinfo"
 )
 
 const (
@@ -60,15 +61,15 @@ func isValidVolname(volname string) bool {
 // posix - implements StorageAPI interface.
 type posix struct {
 	// Disk usage metrics
-	totalUsed uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
-	// Disk usage running routine
-	usageRunning int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
-	ioErrCount   int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
+	totalUsed  uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
+	ioErrCount int32  // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
 
 	diskPath  string
 	pool      sync.Pool
 	connected bool
 
+	diskMount bool // indicates if the path is an actual mount.
+
 	// Disk usage metrics
 	stopUsageCh chan struct{}
 }
@@ -135,10 +136,10 @@ func getValidPath(path string) (string, error) {
 	if err != nil {
 		return path, err
 	}
+	defer os.Remove(pathJoin(path, ".writable-check.tmp"))
 	file.Close()
 
-	err = os.Remove(pathJoin(path, ".writable-check.tmp"))
-	return path, err
+	return path, nil
 }
 
 // isDirEmpty - returns whether given directory is empty or not.
@@ -183,9 +184,12 @@ func newPosix(path string) (*posix, error) {
 			},
 		},
 		stopUsageCh: make(chan struct{}),
+		diskMount:   mountinfo.IsLikelyMountPoint(path),
 	}
 
-	go p.diskUsage(globalServiceDoneCh)
+	if !p.diskMount {
+		go p.diskUsage(globalServiceDoneCh)
+	}
 
 	// Success.
 	return p, nil
@@ -218,7 +222,7 @@ func checkDiskMinTotal(di disk.Info) (err error) {
 	// used for journalling, inodes etc.
 	totalDiskSpace := float64(di.Total) * 0.95
 	if int64(totalDiskSpace) <= diskMinTotalSpace {
-		return errDiskFull
+		return errMinDiskSize
 	}
 	return nil
 }
@@ -293,10 +297,14 @@ func (s *posix) DiskInfo() (info DiskInfo, err error) {
 	if err != nil {
 		return info, err
 	}
+	used := di.Total - di.Free
+	if !s.diskMount {
+		used = atomic.LoadUint64(&s.totalUsed)
+	}
 	return DiskInfo{
 		Total: di.Total,
 		Free:  di.Free,
-		Used:  atomic.LoadUint64(&s.totalUsed),
+		Used:  used,
 	}, nil
 
 }
@@ -336,7 +344,20 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
 	defer ticker.Stop()
 
 	usageFn := func(ctx context.Context, entry string) error {
+		if globalHTTPServer != nil {
+			// Wait at max 1 minute for an inprogress request
+			// before proceeding to count the usage.
+			waitCount := 60
+			// Any requests in progress, delay the usage.
+			for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
+				waitCount--
+				time.Sleep(1 * time.Second)
+			}
+		}
+
 		select {
+		case <-doneCh:
+			return errWalkAbort
 		case <-s.stopUsageCh:
 			return errWalkAbort
 		default:
@@ -349,14 +370,9 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
 		}
 	}
 
-	// Check if disk usage routine is running, if yes then return.
-	if atomic.LoadInt32(&s.usageRunning) == 1 {
-		return
-	}
-	atomic.StoreInt32(&s.usageRunning, 1)
-	defer atomic.StoreInt32(&s.usageRunning, 0)
-
-	if err := getDiskUsage(context.Background(), s.diskPath, usageFn); err != nil {
+	// Return this routine upon errWalkAbort, continue for any other error on purpose
+	// so that we can start the routine freshly in another 12 hours.
+	if err := getDiskUsage(context.Background(), s.diskPath, usageFn); err == errWalkAbort {
 		return
 	}
 
@@ -366,16 +382,20 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
 			return
 		case <-doneCh:
 			return
-		case <-ticker.C:
-			// Check if disk usage routine is running, if yes let it
-			// finish, before starting a new one.
-			if atomic.LoadInt32(&s.usageRunning) == 1 {
-				continue
-			}
-			atomic.StoreInt32(&s.usageRunning, 1)
-
+		case <-time.After(globalUsageCheckInterval):
 			var usage uint64
 			usageFn = func(ctx context.Context, entry string) error {
+				if globalHTTPServer != nil {
+					// Wait at max 1 minute for an inprogress request
+					// before proceeding to count the usage.
+					waitCount := 60
+					// Any requests in progress, delay the usage.
+					for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
+						waitCount--
+						time.Sleep(1 * time.Second)
+					}
+				}
+
 				select {
 				case <-s.stopUsageCh:
 					return errWalkAbort
@@ -390,11 +410,9 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
 			}
 
 			if err := getDiskUsage(context.Background(), s.diskPath, usageFn); err != nil {
-				atomic.StoreInt32(&s.usageRunning, 0)
 				continue
 			}
 
-			atomic.StoreInt32(&s.usageRunning, 0)
 			atomic.StoreUint64(&s.totalUsed, usage)
 		}
 	}

cmd/posix_test.go

@@ -470,7 +470,7 @@ func TestPosixMakeVol(t *testing.T) {
 	}
 }
 
-// TestPosixDeleteVol - Validates the expected behaviour of posix.DeleteVol for various cases.
+// TestPosixDeleteVol - Validates the expected behavior of posix.DeleteVol for various cases.
 func TestPosixDeleteVol(t *testing.T) {
 	// create posix test setup
 	posixStorage, path, err := newPosixTestSetup()
@@ -1948,7 +1948,7 @@ func TestCheckDiskTotalMin(t *testing.T) {
 				FSType: "XFS",
 				Files:  9999,
 			},
-			err: errDiskFull,
+			err: errMinDiskSize,
 		},
 	}
 

cmd/posix_windows_test.go

@@ -69,7 +69,7 @@ func TestUNCPaths(t *testing.T) {
 	}
 }
 
-// Test to validate posix behaviour on windows when a non-final path component is a file.
+// Test to validate posix behavior on windows when a non-final path component is a file.
 func TestUNCPathENOTDIR(t *testing.T) {
 	var err error
 	// Instantiate posix object to manage a disk

cmd/routers.go

@@ -49,6 +49,8 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
 var globalHandlers = []HandlerFunc{
 	// set HTTP security headers such as Content-Security-Policy.
 	addSecurityHeaders,
+	// Forward path style requests to actual host in a bucket federated setup.
+	setBucketForwardingHandler,
 	// Ratelimit the incoming requests using a token bucket algorithm
 	setRateLimitHandler,
 	// Validate all the incoming paths.

cmd/server-main.go

@@ -33,7 +33,7 @@ import (
 )
 
 func init() {
-	logger.Init(GOPATH)
+	logger.Init(GOPATH, GOROOT)
 	logger.RegisterUIError(fmtError)
 }
 
@@ -82,6 +82,7 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
   DOMAIN:
      MINIO_DOMAIN: To enable virtual-host-style requests, set this value to Minio host domain name.
@@ -89,6 +90,11 @@ ENVIRONMENT VARIABLES:
   WORM:
      MINIO_WORM: To turn on Write-Once-Read-Many in server, set this value to "on".
 
+  BUCKET-DNS:
+     MINIO_DOMAIN:    To enable bucket DNS requests, set this value to Minio host domain name.
+     MINIO_PUBLIC_IPS: To enable bucket DNS requests, set this value to list of Minio host public IP(s) delimited by ",".
+     MINIO_ETCD_ENDPOINTS: To enable bucket DNS requests, set this value to list of etcd endpoints delimited by ",".
+
 EXAMPLES:
   1. Start minio server on "/home/shared" directory.
      $ {{.HelpName}} /home/shared
@@ -113,6 +119,7 @@ EXAMPLES:
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
      $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}} /home/shared
 `,
 }
@@ -281,7 +288,7 @@ func serverMain(ctx *cli.Context) {
 		getCert = globalTLSCerts.GetCertificate
 	}
 
-	globalHTTPServer = xhttp.NewServer([]string{globalMinioAddr}, handler, getCert)
+	globalHTTPServer = xhttp.NewServer([]string{globalMinioAddr}, criticalErrorHandler{handler}, getCert)
 	globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
 	globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes
 	go func() {
@@ -303,6 +310,11 @@ func serverMain(ctx *cli.Context) {
 	globalObjectAPI = newObject
 	globalObjLayerMutex.Unlock()
 
+	// Populate existing buckets to the etcd backend
+	if globalDNSConfig != nil {
+		initFederatorBackend(newObject)
+	}
+
 	// Prints the formatted startup message once object layer is initialized.
 	apiEndpoints := getAPIEndpoints(globalMinioAddr)
 	printStartupMessage(apiEndpoints)

cmd/server_test.go

@@ -2420,7 +2420,7 @@ func (s *TestSuiteCommon) TestBucketMultipartList(c *check) {
 	// unmarshalling works from a client perspective, specifically
 	// while unmarshalling time.Time type for 'Initiated' field.
 	// time.Time does not honor xml marshaler, it means that we need
-	// to encode/format it before giving it to xml marshalling.
+	// to encode/format it before giving it to xml marshaling.
 
 	// This below check adds client side verification to see if its
 	// truly parseable.

cmd/signature-v4-utils.go

@@ -39,6 +39,9 @@ func skipContentSha256Cksum(r *http.Request) bool {
 
 	if isRequestPresignedSignatureV4(r) {
 		v, ok = r.URL.Query()["X-Amz-Content-Sha256"]
+		if !ok {
+			v, ok = r.Header["X-Amz-Content-Sha256"]
+		}
 	} else {
 		v, ok = r.Header["X-Amz-Content-Sha256"]
 	}
@@ -62,6 +65,9 @@ func getContentSha256Cksum(r *http.Request) string {
 		// will default to 'UNSIGNED-PAYLOAD'.
 		defaultSha256Cksum = unsignedPayload
 		v, ok = r.URL.Query()["X-Amz-Content-Sha256"]
+		if !ok {
+			v, ok = r.Header["X-Amz-Content-Sha256"]
+		}
 	} else {
 		// X-Amz-Content-Sha256, if not set in signed requests, checksum
 		// will default to sha256([]byte("")).

cmd/storage-errors.go

@@ -76,6 +76,9 @@ var errBitrotHashAlgoInvalid = errors.New("bit-rot hash algorithm is invalid")
 // errCrossDeviceLink - rename across devices not allowed.
 var errCrossDeviceLink = errors.New("Rename across devices not allowed, please fix your backend configuration")
 
+// errMinDiskSize - cannot create volume or files when disk size is less than threshold.
+var errMinDiskSize = errors.New("The disk size is less than the minimum threshold")
+
 // hashMisMatchError - represents a bit-rot hash verification failure
 // error.
 type hashMismatchError struct {

cmd/storage-rpc-client.go

@@ -316,9 +316,9 @@ func NewStorageRPCClient(host *xnet.Host, endpointPath string) (*StorageRPCClien
 // Initialize new storage rpc client.
 func newStorageRPC(endpoint Endpoint) *StorageRPCClient {
 	host, err := xnet.ParseHost(endpoint.Host)
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to parse storage RPC Host", context.Background())
 	rpcClient, err := NewStorageRPCClient(host, endpoint.Path)
-	logger.CriticalIf(context.Background(), err)
+	logger.FatalIf(err, "Unable to initialize storage RPC client", context.Background())
 	rpcClient.connected = rpcClient.Call(storageServiceName+".Connect", &AuthArgs{}, &VoidReply{}) == nil
 	return rpcClient
 }

cmd/test-utils_test.go

@@ -605,7 +605,7 @@ func newTestConfig(bucketLocation string) (rootPath string, err error) {
 	globalServerConfig.SetRegion(bucketLocation)
 
 	// Save config.
-	if err = globalServerConfig.Save(); err != nil {
+	if err = globalServerConfig.Save(getConfigFile()); err != nil {
 		return "", err
 	}
 

cmd/typed-errors.go

@@ -67,3 +67,6 @@ var errNotFirstDisk = errors.New("Not first disk")
 
 // error returned by first disk waiting to initialize other servers.
 var errFirstDiskWait = errors.New("Waiting on other disks")
+
+// error returned when a bucket already exists
+var errBucketAlreadyExists = errors.New("Your previous request to create the named bucket succeeded and you already own it")

cmd/ui-errors.go

@@ -53,6 +53,12 @@ var (
 		"MINIO_CACHE_EXPIRY: Valid cache expiry duration is in days.",
 	)
 
+	uiErrInvalidCacheMaxUse = newUIErrFn(
+		"Invalid cache max-use value",
+		"Please check the passed value",
+		"MINIO_CACHE_MAXUSE: Valid cache max-use value between 0-100.",
+	)
+
 	uiErrInvalidCredentials = newUIErrFn(
 		"Invalid credentials",
 		"Please provide correct credentials",

cmd/update-main.go

@@ -458,7 +458,7 @@ func getUpdateInfo(timeout time.Duration, mode string) (updateMsg string, sha256
 
 func doUpdate(sha256Hex string, latestReleaseTime time.Time, ok bool) (successMsg string, err error) {
 	if !ok {
-		successMsg = greenColorSprintf("Minio update to version RELEASE.%s cancelled.",
+		successMsg = greenColorSprintf("Minio update to version RELEASE.%s canceled.",
 			latestReleaseTime.Format(minioReleaseTagTimeLayout))
 		return successMsg, nil
 	}

cmd/utils.go

@@ -258,7 +258,7 @@ func ToS3ETag(etag string) string {
 // used while communicating with the cloud backends.
 // This sets the value for MaxIdleConnsPerHost from 2 (go default)
 // to 100.
-func NewCustomHTTPTransport() http.RoundTripper {
+func NewCustomHTTPTransport() *http.Transport {
 	return &http.Transport{
 		Proxy: http.ProxyFromEnvironment,
 		DialContext: (&net.Dialer{

cmd/web-handlers.go

@@ -31,6 +31,7 @@ import (
 	"strings"
 	"time"
 
+	etcd "github.com/coreos/etcd/client"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
 	"github.com/gorilla/rpc/v2/json2"
@@ -39,6 +40,7 @@ import (
 	"github.com/minio/minio/browser"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/policy"
@@ -135,6 +137,26 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep
 		return toJSONError(errInvalidBucketName)
 	}
 
+	if globalDNSConfig != nil {
+		if _, err := globalDNSConfig.Get(args.BucketName); err != nil {
+			if etcd.IsKeyNotFound(err) || err == dns.ErrNoEntriesFound {
+				// Proceed to creating a bucket.
+				if err = objectAPI.MakeBucketWithLocation(context.Background(), args.BucketName, globalServerConfig.GetRegion()); err != nil {
+					return toJSONError(err)
+				}
+				if err = globalDNSConfig.Put(args.BucketName); err != nil {
+					objectAPI.DeleteBucket(context.Background(), args.BucketName)
+					return toJSONError(err)
+				}
+
+				reply.UIVersion = browser.UIVersion
+				return nil
+			}
+			return toJSONError(err)
+		}
+		return toJSONError(errBucketAlreadyExists)
+	}
+
 	if err := objectAPI.MakeBucketWithLocation(context.Background(), args.BucketName, globalServerConfig.GetRegion()); err != nil {
 		return toJSONError(err, args.BucketName)
 	}
@@ -176,6 +198,14 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs,
 		logger.LogIf(ctx, nerr.Err)
 	}
 
+	if globalDNSConfig != nil {
+		if err := globalDNSConfig.Delete(args.BucketName); err != nil {
+			// Deleting DNS entry failed, attempt to create the bucket again.
+			objectAPI.MakeBucketWithLocation(ctx, args.BucketName, "")
+			return toJSONError(err)
+		}
+	}
+
 	reply.UIVersion = browser.UIVersion
 	return nil
 }
@@ -208,16 +238,32 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re
 	if authErr != nil {
 		return toJSONError(authErr)
 	}
-	buckets, err := listBuckets(context.Background())
-	if err != nil {
-		return toJSONError(err)
-	}
-	for _, bucket := range buckets {
-		reply.Buckets = append(reply.Buckets, WebBucketInfo{
-			Name:         bucket.Name,
-			CreationDate: bucket.Created,
-		})
+	// If etcd, dns federation configured list buckets from etcd.
+	if globalDNSConfig != nil {
+		dnsBuckets, err := globalDNSConfig.List()
+		if err != nil {
+			return toJSONError(err)
+		}
+		for _, dnsRecord := range dnsBuckets {
+			bucketName := strings.Trim(dnsRecord.Key, "/")
+			reply.Buckets = append(reply.Buckets, WebBucketInfo{
+				Name:         bucketName,
+				CreationDate: dnsRecord.CreationDate,
+			})
+		}
+	} else {
+		buckets, err := listBuckets(context.Background())
+		if err != nil {
+			return toJSONError(err)
+		}
+		for _, bucket := range buckets {
+			reply.Buckets = append(reply.Buckets, WebBucketInfo{
+				Name:         bucket.Name,
+				CreationDate: bucket.Created,
+			})
+		}
 	}
+
 	reply.UIVersion = browser.UIVersion
 	return nil
 }
@@ -412,11 +458,6 @@ type LoginRep struct {
 func (web *webAPIHandlers) Login(r *http.Request, args *LoginArgs, reply *LoginRep) error {
 	token, err := authenticateWeb(args.Username, args.Password)
 	if err != nil {
-		// Make sure to log errors related to browser login,
-		// for security and auditing reasons.
-		reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", r.RemoteAddr)
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogIf(ctx, err)
 		return toJSONError(err)
 	}
 
@@ -437,7 +478,9 @@ func (web webAPIHandlers) GenerateAuth(r *http.Request, args *WebGenericArgs, re
 		return toJSONError(errAuthentication)
 	}
 	cred, err := auth.GetNewCredentials()
-	logger.CriticalIf(context.Background(), err)
+	if err != nil {
+		return toJSONError(err)
+	}
 	reply.AccessKey = cred.AccessKey
 	reply.SecretKey = cred.SecretKey
 	reply.UIVersion = browser.UIVersion
@@ -480,9 +523,9 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se
 	// Update credentials in memory
 	prevCred := globalServerConfig.SetCredential(creds)
 
-	// Save credentials to config file
-	if err := globalServerConfig.Save(); err != nil {
-		// As saving configurstion failed, restore previous credential in memory.
+	// Persist updated credentials.
+	if err = globalServerConfig.Save(getConfigFile()); err != nil {
+		// Save the current creds when failed to update.
 		globalServerConfig.SetCredential(prevCred)
 		logger.LogIf(context.Background(), err)
 		return toJSONError(err)

cmd/xl-sets.go

@@ -476,7 +476,7 @@ func (s *xlSets) SetBucketPolicy(ctx context.Context, bucket string, policy *pol
 
 // GetBucketPolicy will return a policy on a bucket
 func (s *xlSets) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
-	return GetPolicyConfig(s, bucket)
+	return getPolicyConfig(s, bucket)
 }
 
 // DeleteBucketPolicy deletes all policies on bucket
@@ -614,7 +614,7 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke
 			}
 			return
 		}
-		// Close writer explicitly signalling we wrote all data.
+		// Close writer explicitly signaling we wrote all data.
 		if gerr := srcInfo.Writer.Close(); gerr != nil {
 			logger.LogIf(ctx, gerr)
 			return

cmd/xl-v1-bucket.go

@@ -284,7 +284,7 @@ func (xl xlObjects) SetBucketPolicy(ctx context.Context, bucket string, policy *
 
 // GetBucketPolicy will get policy on bucket
 func (xl xlObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
-	return GetPolicyConfig(xl, bucket)
+	return getPolicyConfig(xl, bucket)
 }
 
 // DeleteBucketPolicy deletes all policies on bucket

cmd/xl-v1-metadata.go

@@ -463,7 +463,7 @@ func writeXLMetadata(ctx context.Context, disk StorageAPI, bucket, prefix string
 		logger.LogIf(ctx, err)
 		return err
 	}
-	// Persist marshalled data.
+	// Persist marshaled data.
 	err = disk.AppendFile(bucket, jsonFile, metadataBytes)
 	logger.LogIf(ctx, err)
 	return err

cmd/xl-v1-multipart.go

@@ -277,7 +277,7 @@ func (xl xlObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, ds
 			}
 			return
 		}
-		// Close writer explicitly signalling we wrote all data.
+		// Close writer explicitly signaling we wrote all data.
 		if gerr := srcInfo.Writer.Close(); gerr != nil {
 			logger.LogIf(ctx, gerr)
 			return
@@ -375,8 +375,17 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	}
 
 	// Fetch buffer for I/O, returns from the pool if not allocates a new one and returns.
-	buffer := xl.bp.Get()
-	defer xl.bp.Put(buffer)
+	var buffer []byte
+	switch size := data.Size(); {
+	case size == 0:
+		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
+	case size < blockSizeV1:
+		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
+		buffer = make([]byte, size, 2*size)
+	default:
+		buffer = xl.bp.Get()
+		defer xl.bp.Put(buffer)
+	}
 
 	file, err := storage.CreateFile(ctx, data, minioMetaTmpBucket, tmpPartPath, buffer, DefaultBitrotAlgorithm, writeQuorum)
 	if err != nil {
@@ -473,7 +482,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 // to indicate where the listing should begin from.
 //
 // Implements S3 compatible ListObjectParts API. The resulting
-// ListPartsInfo structure is marshalled directly into XML and
+// ListPartsInfo structure is marshaled directly into XML and
 // replied back to the client.
 func (xl xlObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int) (result ListPartsInfo, e error) {
 	if err := checkListPartsArgs(ctx, bucket, object, xl); err != nil {

cmd/xl-v1-object.go

@@ -142,7 +142,7 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc
 			pipeWriter.CloseWithError(toObjectErr(gerr, srcBucket, srcObject))
 			return
 		}
-		pipeWriter.Close() // Close writer explicitly signalling we wrote all data.
+		pipeWriter.Close() // Close writer explicitly signaling we wrote all data.
 	}()
 
 	hashReader, err := hash.NewReader(pipeReader, length, "", "")
@@ -611,8 +611,17 @@ func (xl xlObjects) putObject(ctx context.Context, bucket string, object string,
 	}
 
 	// Fetch buffer for I/O, returns from the pool if not allocates a new one and returns.
-	buffer := xl.bp.Get()
-	defer xl.bp.Put(buffer)
+	var buffer []byte
+	switch size := data.Size(); {
+	case size == 0:
+		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
+	case size < blockSizeV1:
+		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
+		buffer = make([]byte, size, 2*size)
+	default:
+		buffer = xl.bp.Get()
+		defer xl.bp.Put(buffer)
+	}
 
 	// Read data and split into parts - similar to multipart mechanism
 	for partIdx := 1; ; partIdx++ {

dockerscripts/healthcheck.sh

@@ -48,7 +48,7 @@ healthcheck_main () {
             http_response=$(curl -s -k -o /dev/null -w "%{http_code}" ${scheme}${address}${resource})
         fi
 
-        # If http_repsonse is 200 - server is up.
+        # If http_response is 200 - server is up.
         [ "$http_response" = "200" ]
     fi
 }

docs/bucket/notifications/README.md

@@ -20,7 +20,7 @@ Bucket events can be published to the following targets:
 
 ## Prerequisites
 
-* Install and configure Minio Server from [here](http://docs.minio.io/docs/minio-quickstart-guide).
+* Install and configure Minio Server from [here](https://docs.minio.io/docs/minio-quickstart-guide).
 * Install and configure Minio Client from [here](https://docs.minio.io/docs/minio-client-quickstart-guide).
 
 <a name="AMQP"></a>

docs/config/README.md

@@ -10,7 +10,7 @@ minio server --config-dir /etc/minio /data
 ```
 
 ### Certificate Directory
-TLS certificates are stored under ``${HOME}/.minio/certs`` directory. You need to place certificates here to enable `HTTPS` based access. Read more about [How to secure access to Minio server with TLS](http://docs.minio.io/docs/how-to-secure-access-to-minio-server-with-tls).
+TLS certificates are stored under ``${HOME}/.minio/certs`` directory. You need to place certificates here to enable `HTTPS` based access. Read more about [How to secure access to Minio server with TLS](https://docs.minio.io/docs/how-to-secure-access-to-minio-server-with-tls).
 
 Following is the directory structure for Minio server with TLS certificates.
 
@@ -115,15 +115,15 @@ By default, parity for objects with standard storage class is set to `N/2`, and
 |Field|Type|Description|
 |:---|:---|:---|
 |``notify``| |Notify enables bucket notification events for lambda computing via the following targets.|
-|``notify.amqp``| |[Configure to publish Minio events via AMQP target.](http://docs.minio.io/docs/minio-bucket-notification-guide#AMQP)|
-|``notify.nats``| |[Configure to publish Minio events via NATS target.](http://docs.minio.io/docs/minio-bucket-notification-guide#NATS)|
-|``notify.elasticsearch``| |[Configure to publish Minio events via Elasticsearch target.](http://docs.minio.io/docs/minio-bucket-notification-guide#Elasticsearch)|
-|``notify.redis``| |[Configure to publish Minio events via Redis target.](http://docs.minio.io/docs/minio-bucket-notification-guide#Redis)|
-|``notify.postgresql``| |[Configure to publish Minio events via PostgreSQL target.](http://docs.minio.io/docs/minio-bucket-notification-guide#PostgreSQL)|
-|``notify.kafka``| |[Configure to publish Minio events via Apache Kafka target.](http://docs.minio.io/docs/minio-bucket-notification-guide#apache-kafka)|
-|``notify.webhook``| |[Configure to publish Minio events via Webhooks target.](http://docs.minio.io/docs/minio-bucket-notification-guide#webhooks)|
+|``notify.amqp``| |[Configure to publish Minio events via AMQP target.](https://docs.minio.io/docs/minio-bucket-notification-guide#AMQP)|
+|``notify.nats``| |[Configure to publish Minio events via NATS target.](https://docs.minio.io/docs/minio-bucket-notification-guide#NATS)|
+|``notify.elasticsearch``| |[Configure to publish Minio events via Elasticsearch target.](https://docs.minio.io/docs/minio-bucket-notification-guide#Elasticsearch)|
+|``notify.redis``| |[Configure to publish Minio events via Redis target.](https://docs.minio.io/docs/minio-bucket-notification-guide#Redis)|
+|``notify.postgresql``| |[Configure to publish Minio events via PostgreSQL target.](https://docs.minio.io/docs/minio-bucket-notification-guide#PostgreSQL)|
+|``notify.kafka``| |[Configure to publish Minio events via Apache Kafka target.](https://docs.minio.io/docs/minio-bucket-notification-guide#apache-kafka)|
+|``notify.webhook``| |[Configure to publish Minio events via Webhooks target.](https://docs.minio.io/docs/minio-bucket-notification-guide#webhooks)|
 |``notify.mysql``| |[Configure to publish Minio events via MySql target.](https://docs.minio.io/docs/minio-bucket-notification-guide#MySQL)|
-|``notify.mqtt``| |[Configure to publish Minio events via MQTT target.](http://docs.minio.io/docs/minio-bucket-notification-guide#MQTT)|
+|``notify.mqtt``| |[Configure to publish Minio events via MQTT target.](https://docs.minio.io/docs/minio-bucket-notification-guide#MQTT)|
 
 ## Explore Further
 * [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide)

docs/disk-caching/DESIGN.md

@@ -12,6 +12,7 @@ minio server -h
      MINIO_CACHE_DRIVES: List of mounted cache drives or directories delimited by ";"
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";"
      MINIO_CACHE_EXPIRY: Cache expiry duration in days
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 ...
 ...
 
@@ -21,6 +22,7 @@ minio server -h
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3"
      $ export MINIO_CACHE_EXCLUDE="mybucket/*;*.pdf"
      $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
      $ minio server /home/shared
 ```
 

docs/disk-caching/README.md

@@ -8,7 +8,7 @@ Disk caching feature here refers to the use of caching disks to store content cl
 ## Get started
 
 ### 1. Prerequisites
-Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio).
+Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide).
 
 ### 2. Run Minio with cache
 Disk caching can be enabled by updating the `cache` config settings for Minio server. Config `cache` settings takes the mounted drive(s) or directory paths, cache expiry duration (in days) and any wildcard patterns to exclude from being cached.
@@ -27,6 +27,7 @@ The cache settings may also be set through environment variables. When set, envi
 export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3"
 export MINIO_CACHE_EXPIRY=90
 export MINIO_CACHE_EXCLUDE="*.pdf;mybucket/*"
+export MINIO_CACHE_MAXUSE=80
 minio server /export{1...24}
 ```
 

docs/erasure/README.md

@@ -18,7 +18,7 @@ Erasure code protects data from multiple drives failure, unlike RAID or replicat
 
 ## What is Bit Rot protection?
 
-Bit Rot, also known as data rot or silent data corruption is a data loss issue faced by disk drives today. Data on the drive may silently get corrupted without signalling an error has occurred, making bit rot more dangerous than a permanent hard drive failure.
+Bit Rot, also known as data rot or silent data corruption is a data loss issue faced by disk drives today. Data on the drive may silently get corrupted without signaling an error has occurred, making bit rot more dangerous than a permanent hard drive failure.
 
 Minio's erasure coded backend uses high speed [HighwayHash](https://blog.minio.io/highwayhash-fast-hashing-at-over-10-gb-s-per-core-in-golang-fee938b5218a) checksums to protect against Bit Rot.
 

docs/erasure/storage-class/README.md

@@ -11,8 +11,8 @@ saving the object in specific number of data and parity disks.
 
 ## Storage usage
 
-The selection of varying data and parity drives has a direct impact on the drive space usage. With storage class, you can optimise for high
-redundancy or better drive space utilisation.
+The selection of varying data and parity drives has a direct impact on the drive space usage. With storage class, you can optimize for high
+redundancy or better drive space utilization.
 
 To get an idea of how various combinations of data and parity drives affect the storage usage, let’s take an example of a 100 MiB file stored
 on 16 drive Minio deployment. If you use eight data and eight parity drives, the file space usage will be approximately twice, i.e. 100 MiB

docs/federation/lookup/Corefile.example

@@ -0,0 +1,11 @@
+. {
+  etcd churchofminio.com {
+      endpoint http://localhost:2379 http://localhost:4001
+      upstream /etc/resolv.conf
+  }
+  debug
+  prometheus
+  cache 160 mydomain.com
+  loadbalance
+  proxy . /etc/resolv.conf
+}

docs/federation/lookup/README.md

@@ -0,0 +1,84 @@
+# Federation Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
+This document explains how to configure Minio with `Bucket lookup from DNS` style federation.
+
+## Get started
+
+### 1. Prerequisites
+Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide).
+
+### 2. Run Minio in federated mode
+Bucket lookup from DNS federation requires two dependencies
+
+- etcd (for config, bucket SRV records)
+- coredns (for DNS management based on populated bucket SRV records, optional)
+
+## Architecture
+
+![bucket-lookup](https://github.com/minio/minio/blob/master/docs/federation/lookup/bucket-lookup.png?raw=true)
+
+### Environment variables
+
+#### MINIO_ETCD_ENDPOINTS
+
+This is comma separated list of etcd servers that you want to use as the Minio federation back-end. This should
+be same across the federated deployment, i.e. all the Minio instances within a federated deployment should use same
+etcd back-end.
+
+#### MINIO_DOMAIN
+
+This is the top level domain name used for the federated setup. This domain name should ideally resolve to a load-balancer
+running in front of all the federated Minio instances. The domain name is used to create sub domain entries to etcd. For
+example, if the domain is set to `domain.com`, the buckets `bucket1`, `bucket2` will be accessible as `bucket1.domain.com`
+and `bucket2.domain.com`.
+
+#### MINIO_PUBLIC_IPS
+
+This is comma separated list of IP addresses to which buckets created on this Minio instance will resolve to. For example,
+a bucket `bucket1` created on current Minio instance will be accessible as `bucket1.domain.com`, and the DNS entry for
+`bucket1.domain.com` will point to IP address set in `MINIO_PUBLIC_IPS`.
+
+*Note*
+
+- This field is mandatory for standalone and erasure code Minio server deployments, to enable federated mode.
+- This field is optional for distributed deployments. If you don't set this field in a federated setup, we use the IP addresses of
+hosts passed to the Minio server startup and use them for DNS entries.
+
+### Run Multiple Clusters
+
+> cluster1
+
+```sh
+export MINIO_ETCD_ENDPOINTS="http://remote-etcd1:2379,http://remote-etcd2:4001"
+export MINIO_DOMAIN=domain.com
+export MINIO_PUBLIC_IPS=44.35.2.1,44.35.2.2,44.35.2.3,44.35.2.4
+minio server http://rack{1...4}.host{1...4}.domain.com/mnt/export{1...32}
+```
+
+> cluster2
+
+```sh
+export MINIO_ETCD_ENDPOINTS="http://remote-etcd1:2379,http://remote-etcd2:4001"
+export MINIO_DOMAIN=domain.com
+export MINIO_PUBLIC_IPS=44.35.1.1,44.35.1.2,44.35.1.3,44.35.1.4
+minio server http://rack{5...8}.host{5...8}.domain.com/mnt/export{1...32}
+```
+
+In this configuration you can see `MINIO_ETCD_ENDPOINTS` points to the etcd backend which manages Minio's
+`config.json` and bucket DNS SRV records. `MINIO_DOMAIN` indicates the domain suffix for the bucket which
+will be used to resolve bucket through DNS. For example if you have a bucket such as `mybucket`, the
+client can use now `mybucket.domain.com` to directly resolve itself to the right cluster. `MINIO_PUBLIC_IPS`
+points to the public IP address where each cluster might be accessible, this is unique for each cluster.
+
+NOTE: `mybucket` only exists on one cluster either `cluster1` or `cluster2` this is random and
+is decided by how `domain.com` gets resolved, if there is a round-robin DNS on `domain.com` then
+it is randomized which cluster might provision the bucket.
+
+### 3. Test your setup
+To test this setup, access the Minio server via browser or [`mc`](https://docs.minio.io/docs/minio-client-quickstart-guide). You’ll see the uploaded files are accessible from the all the Minio endpoints.
+
+# Explore Further
+- [Use `mc` with Minio Server](https://docs.minio.io/docs/minio-client-quickstart-guide)
+- [Use `aws-cli` with Minio Server](https://docs.minio.io/docs/aws-cli-with-minio)
+- [Use `s3cmd` with Minio Server](https://docs.minio.io/docs/s3cmd-with-minio)
+- [Use `minio-go` SDK with Minio Server](https://docs.minio.io/docs/golang-client-quickstart-guide)
+- [The Minio documentation website](https://docs.minio.io)

docs/large-bucket/README.md

@@ -8,7 +8,7 @@ We call it large-bucket as it allows a single Minio bucket to expand over multip
 If you're aware of distributed Minio setup, the installation and configuration remains the same. Only new addition to the input syntax is `...` convention to abbreviate the drive arguments. Remote drives in a distributed setup are encoded as HTTP(s) URIs which can be similarly abbreviated as well.
 
 ### 1. Prerequisites
-Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio).
+Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide).
 
 ### 2. Run Minio on many drives
 We'll see examples on how to do this in the following sections.

docs/minio-limits.md

@@ -36,19 +36,19 @@ We found the following APIs to be redundant or less useful outside of AWS S3. If
 
 #### List of Amazon S3 Bucket API's not supported on Minio
 
-- BucketACL (Use [bucket policies](http://docs.minio.io/docs/minio-client-complete-guide#policy) instead)
+- BucketACL (Use [bucket policies](https://docs.minio.io/docs/minio-client-complete-guide#policy) instead)
 - BucketCORS (CORS enabled by default on all buckets for all HTTP verbs)
 - BucketLifecycle (Not required for Minio erasure coded backend)
-- BucketReplication (Use [`mc mirror`](http://docs.minio.io/docs/minio-client-complete-guide#mirror) instead)
+- BucketReplication (Use [`mc mirror`](https://docs.minio.io/docs/minio-client-complete-guide#mirror) instead)
 - BucketVersions, BucketVersioning (Use [`s3git`](https://github.com/s3git/s3git))
 - BucketWebsite (Use [`caddy`](https://github.com/mholt/caddy) or [`nginx`](https://www.nginx.com/resources/wiki/))
-- BucketAnalytics, BucketMetrics, BucketLogging (Use [bucket notification](http://docs.minio.io/docs/minio-client-complete-guide#events) APIs)
+- BucketAnalytics, BucketMetrics, BucketLogging (Use [bucket notification](https://docs.minio.io/docs/minio-client-complete-guide#events) APIs)
 - BucketRequestPayment
 - BucketTagging
 
 #### List of Amazon S3 Object API's not supported on Minio
 
-- ObjectACL (Use [bucket policies](http://docs.minio.io/docs/minio-client-complete-guide#policy) instead)
+- ObjectACL (Use [bucket policies](https://docs.minio.io/docs/minio-client-complete-guide#policy) instead)
 - ObjectTorrent
 - ObjectVersions