[sec] Match ${aws:username} exactly instead of prefix match (#7791)

This PR fixes a security issue where an IAM user based
on his policy is granted more privileges than restricted
by the users IAM policy.

This is due to an issue of prefix based Matcher() function
which was incorrectly matching prefix based on resource
prefixes instead of exact match.

.gitignore

@@ -24,3 +24,4 @@ stage/
 .sia_temp/
 config.json
 healthcheck
+check-user

Dockerfile

@@ -10,7 +10,8 @@ RUN  \
      apk add --no-cache git && \
      git clone https://github.com/minio/minio && cd minio && \
      go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
-     cd dockerscripts; go build -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go
+     cd dockerscripts; go build -tags kqueue -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go && \
+     go build -tags kqueue -ldflags "-s -w" -o /usr/bin/check-user check-user.go
 
 FROM alpine:3.9
 
@@ -22,10 +23,11 @@ EXPOSE 9000
 
 COPY --from=0 /go/bin/minio /usr/bin/minio
 COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
+COPY --from=0 /usr/bin/check-user /usr/bin/check-user
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \
-     apk add --no-cache ca-certificates 'curl>7.61.0' && \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]

Dockerfile.dev

@@ -2,7 +2,7 @@ FROM alpine:3.9
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck /usr/bin/
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck dockerscripts/check-user /usr/bin/
 COPY minio /usr/bin/
 
 ENV MINIO_UPDATE off
@@ -10,11 +10,12 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key
 
 RUN \
-     apk add --no-cache ca-certificates 'curl>7.61.0' && \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      chmod +x /usr/bin/minio  && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck
+     chmod +x /usr/bin/healthcheck && \
+     chmod +x /usr/bin/check-user
 
 EXPOSE 9000
 

Dockerfile.release

@@ -7,13 +7,15 @@ ENV GO111MODULE on
 RUN  \
      apk add --no-cache git && \
      git clone https://github.com/minio/minio && cd minio/dockerscripts && \
-     go build -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go
+     go build -tags kqueue -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go && \
+     go build -tags kqueue -ldflags "-s -w" -o /usr/bin/check-user check-user.go
 
 FROM alpine:3.9
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
+COPY --from=0 /usr/bin/check-user /usr/bin/check-user
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 ENV MINIO_UPDATE off
@@ -21,12 +23,13 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key
 
 RUN \
-     apk add --no-cache ca-certificates 'curl>7.61.0' && \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      curl https://dl.min.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
      chmod +x /usr/bin/minio  && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck
+     chmod +x /usr/bin/healthcheck && \
+     chmod +x /usr/bin/check-user
 
 EXPOSE 9000
 

Makefile

@@ -2,6 +2,12 @@ PWD := $(shell pwd)
 GOPATH := $(shell go env GOPATH)
 LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
 
+GOOS := $(shell go env GOOS)
+GOOSALT ?= 'linux'
+ifeq ($(GOOS),'darwin')
+  GOOSALT = 'mac'
+endif
+
 TAG ?= $(USER)
 BUILD_LDFLAGS := '$(LDFLAGS)'
 
@@ -14,8 +20,8 @@ checks:
 getdeps:
 	@mkdir -p ${GOPATH}/bin
 	@which golint 1>/dev/null || (echo "Installing golint" && go get -u golang.org/x/lint/golint)
-	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet -O ${GOPATH}/bin/staticcheck https://github.com/dominikh/go-tools/releases/download/2019.1/staticcheck_linux_amd64 && chmod +x ${GOPATH}/bin/staticcheck)
-	@which misspell 1>/dev/null || (echo "Installing misspell" && wget --quiet https://github.com/client9/misspell/releases/download/v0.3.4/misspell_0.3.4_linux_64bit.tar.gz && tar xf misspell_0.3.4_linux_64bit.tar.gz && mv misspell ${GOPATH}/bin/misspell && chmod +x ${GOPATH}/bin/misspell && rm -f misspell_0.3.4_linux_64bit.tar.gz)
+	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet -O ${GOPATH}/bin/staticcheck https://github.com/dominikh/go-tools/releases/download/2019.1/staticcheck_${GOOS}_amd64 && chmod +x ${GOPATH}/bin/staticcheck)
+	@which misspell 1>/dev/null || (echo "Installing misspell" && wget --quiet https://github.com/client9/misspell/releases/download/v0.3.4/misspell_0.3.4_${GOOSALT}_64bit.tar.gz && tar xf misspell_0.3.4_${GOOSALT}_64bit.tar.gz && mv misspell ${GOPATH}/bin/misspell && chmod +x ${GOPATH}/bin/misspell && rm -f misspell_0.3.4_${GOOSALT}_64bit.tar.gz)
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
@@ -67,6 +73,7 @@ build: checks
 	@echo "Building minio binary to './minio'"
 	@GO111MODULE=on GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
 	@GO111MODULE=on GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/dockerscripts/healthcheck $(PWD)/dockerscripts/healthcheck.go 1>/dev/null
+	@GO111MODULE=on GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/dockerscripts/check-user $(PWD)/dockerscripts/check-user.go 1>/dev/null
 
 docker: build
 	@docker build -t $(TAG) . -f Dockerfile.dev

SECURITY.md

@@ -0,0 +1,41 @@
+# Security Policy
+
+## Supported Versions
+
+We always provide security updates for the [latest release](https://github.com/minio/minio/releases/latest).
+Whenever there is a security update you just need to upgrade to the latest version.
+
+## Reporting a Vulnerability
+
+All security bugs in [minio/minio](https://github,com/minio/minio) (or other minio/* repositories)
+should be reported by email to security@min.io. Your email will be acknowledged within 48 hours, 
+and you'll receive a more detailed response to your email within 72 hours indicating the next steps 
+in handling your report. 
+
+Please, provide a detailed explanation of the issue. In particular, outline the type of the security 
+issue (DoS, authentication bypass, information disclouse, ...) and the assumptions you're making (e.g. do 
+you need access credentials for a successful exploit).
+
+If you have not received a reply to your email within 48 hours or you have not heard from the security team
+for the past five days please contact the security team directly:
+   - Primary security coordinator: aead@min.io
+   - Secondary coordinator: harsha@min.io
+   - If you receive no response: dev@min.io
+
+### Disclosure Process
+
+MinIO uses the following disclosure process:
+
+1. Once the security report is received one member of the security team tries to verify and reproduce
+   the issue and determines the impact it has.
+2. A member of the security team will respond and either confrim or reject the security report.
+   If the report is rejected the repsonse explains why.
+3. Code is audited to find any potential similar problems.
+4. Fixes are prepared for the latest release. 
+5. On the date that the fixes are applied a security advisory will be published on https://blog.min.io.
+   Please inform us in your report email whether MinIO should mention your contribution w.r.t. fixing
+   the security issue. By default MinIO will **not** publish this information to protect your privacy.
+   
+This process can take some time, especially when coordination is required with maintainers of other projects.
+Every effort will be made to handle the bug in as timely a manner as possible, however it's important that we 
+follow the process described above to ensure that disclosures are handled consistently. 

browser/README.md

@@ -4,10 +4,11 @@
 
 ## Installation
 
-### Install yarn
+### Install node
 ```sh
-curl -o- -L https://yarnpkg.com/install.sh | bash
-yarn
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+exec -l $SHELL
+nvm install stable
 ```
 
 ### Install `go-bindata` and `go-bindata-assetfs`
@@ -15,8 +16,8 @@ yarn
 If you do not have a working Golang environment, please follow [Install Golang](https://docs.min.io/docs/how-to-install-golang)
 
 ```sh
-go get github.com/jteeuwen/go-bindata/...
-go get github.com/elazarl/go-bindata-assetfs/...
+go get github.com/go-bindata/go-bindata/go-bindata
+go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs
 ```
 
 ## Generating Assets
@@ -24,7 +25,7 @@ go get github.com/elazarl/go-bindata-assetfs/...
 ### Generate ui-assets.go
 
 ```sh
-yarn release
+npm run release
 ```
 
 This generates ui-assets.go in the current directory. Now do `make` in the parent directory to build the minio binary with the newly generated ``ui-assets.go``
@@ -32,7 +33,7 @@ This generates ui-assets.go in the current directory. Now do `make` in the paren
 ### Run MinIO Browser with live reload
 
 ```sh
-yarn dev
+npm run dev
 ```
 
 Open [http://localhost:8080/minio/](http://localhost:8080/minio/) in your browser to play with the application
@@ -66,7 +67,7 @@ index 3ccdaba..9496c56 100644
 ```
 
 ```sh
-yarn dev
+npm run dev
 ```
 
 Open [http://localhost:8888/minio/](http://localhost:8888/minio/) in your browser to play with the application

browser/app/js/browser/ChangePasswordModal.js

@@ -18,157 +18,237 @@ import React from "react"
 import { connect } from "react-redux"
 import web from "../web"
 import * as alertActions from "../alert/actions"
+import { getRandomAccessKey, getRandomSecretKey } from "../utils"
+import jwtDecode from "jwt-decode"
+import classNames from "classnames"
 
-import {
-  Tooltip,
-  Modal,
-  ModalBody,
-  ModalHeader,
-  OverlayTrigger
-} from "react-bootstrap"
+import { Modal, ModalBody, ModalHeader } from "react-bootstrap"
 import InputGroup from "./InputGroup"
+import { ACCESS_KEY_MIN_LENGTH, SECRET_KEY_MIN_LENGTH } from "../constants"
 
 export class ChangePasswordModal extends React.Component {
   constructor(props) {
     super(props)
     this.state = {
-      accessKey: "",
-      secretKey: "",
-      keysReadOnly: false
+      currentAccessKey: "",
+      currentSecretKey: "",
+      currentSecretKeyVisible: false,
+      newAccessKey: "",
+      newSecretKey: "",
+      newSecretKeyVisible: false
     }
   }
-  // When its shown, it loads the access key and secret key.
+  // When its shown, it loads the access key from JWT token
   componentWillMount() {
-    const { serverInfo } = this.props
-
-    // Check environment variables first.
-    if (serverInfo.info.isEnvCreds || serverInfo.info.isWorm) {
-      this.setState({
-        accessKey: "xxxxxxxxx",
-        secretKey: "xxxxxxxxx",
-        keysReadOnly: true
-      })
-    } else {
-      web.GetAuth().then(data => {
-        this.setState({
-          accessKey: data.accessKey,
-          secretKey: data.secretKey
-        })
-      })
-    }
-  }
-
-  // Handle field changes from inside the modal.
-  accessKeyChange(e) {
+    const token = jwtDecode(web.GetToken())
     this.setState({
-      accessKey: e.target.value
-    })
-  }
-
-  secretKeyChange(e) {
-    this.setState({
-      secretKey: e.target.value
-    })
-  }
-
-  secretKeyVisible(secretKeyVisible) {
-    this.setState({
-      secretKeyVisible
+      currentAccessKey: token.sub,
+      newAccessKey: token.sub
     })
   }
 
   // Save the auth params and set them.
   setAuth(e) {
     const { showAlert } = this.props
-    const accessKey = this.state.accessKey
-    const secretKey = this.state.secretKey
-    web
-      .SetAuth({
-        accessKey,
-        secretKey
-      })
-      .then(data => {
-        showAlert({
-          type: "success",
-          message: "Changed credentials"
+
+    if (this.canUpdateCredentials()) {
+      const currentAccessKey = this.state.currentAccessKey
+      const currentSecretKey = this.state.currentSecretKey
+      const newAccessKey = this.state.newAccessKey
+      const newSecretKey = this.state.newSecretKey
+      web
+        .SetAuth({
+          currentAccessKey,
+          currentSecretKey,
+          newAccessKey,
+          newSecretKey
         })
-      })
-      .catch(err => {
-        showAlert({
-          type: "danger",
-          message: err.message
+        .then(data => {
+          showAlert({
+            type: "success",
+            message: "Credentials updated successfully."
+          })
         })
-      })
+        .catch(err => {
+          showAlert({
+            type: "danger",
+            message: err.message
+          })
+        })
+    }
   }
 
   generateAuth(e) {
-    web.GenerateAuth().then(data => {
+    const { serverInfo } = this.props
+    // Generate random access key only for root user
+    if (!serverInfo.userInfo.isIAMUser) {
       this.setState({
-        accessKey: data.accessKey,
-        secretKey: data.secretKey,
-        secretKeyVisible: true
+        newAccessKey: getRandomAccessKey()
       })
+    }
+
+    this.setState({
+      newSecretKey: getRandomSecretKey(),
+      newSecretKeyVisible: true
     })
   }
 
+  canChangePassword() {
+    const { serverInfo } = this.props
+    // Password change is not allowed in WORM mode
+    if (serverInfo.info.isWorm) {
+      return false
+    }
+
+    // When credentials are set on ENV, password change not allowed for owner
+    if (serverInfo.info.isEnvCreds && !serverInfo.userInfo.isIAMUser) {
+      return false
+    }
+    return true
+  }
+
+  canUpdateCredentials() {
+    return (
+      this.state.currentAccessKey.length > 0 &&
+      this.state.currentSecretKey.length > 0 &&
+      this.state.newAccessKey.length >= ACCESS_KEY_MIN_LENGTH &&
+      this.state.newSecretKey.length >= SECRET_KEY_MIN_LENGTH
+    )
+  }
+
   render() {
-    const { hideChangePassword } = this.props
+    const { hideChangePassword, serverInfo } = this.props
+    const allowChangePassword = this.canChangePassword()
+
+    if (!allowChangePassword) {
+      return (
+        <Modal bsSize="sm" animation={false} show={true}>
+          <ModalHeader>Change Password</ModalHeader>
+          <ModalBody>
+            Credentials of this user cannot be updated through MinIO Browser.
+          </ModalBody>
+          <div className="modal-footer">
+            <button
+              id="cancel-change-password"
+              className="btn btn-link"
+              onClick={hideChangePassword}
+            >
+              Close
+            </button>
+          </div>
+        </Modal>
+      )
+    }
+
     return (
       <Modal bsSize="sm" animation={false} show={true}>
         <ModalHeader>Change Password</ModalHeader>
         <ModalBody className="m-t-20">
-          <InputGroup
-            value={this.state.accessKey}
-            onChange={this.accessKeyChange.bind(this)}
-            id="accessKey"
-            label="Access Key"
-            name="accesskey"
-            type="text"
-            spellCheck="false"
-            required="required"
-            autoComplete="false"
-            align="ig-left"
-            readonly={this.state.keysReadOnly}
-          />
-          <i
-            onClick={this.secretKeyVisible.bind(
-              this,
-              !this.state.secretKeyVisible
+          <div className="has-toggle-password">
+            <InputGroup
+              value={this.state.currentAccessKey}
+              id="currentAccessKey"
+              label="Current Access Key"
+              name="currentAccesskey"
+              type="text"
+              spellCheck="false"
+              required="required"
+              autoComplete="false"
+              align="ig-left"
+              readonly={true}
+            />
+
+            <i
+              onClick={() => {
+                this.setState({
+                  currentSecretKeyVisible: !this.state.currentSecretKeyVisible
+                })
+              }}
+              className={
+                "toggle-password fa fa-eye " +
+                (this.state.currentSecretKeyVisible ? "toggled" : "")
+              }
+            />
+            <InputGroup
+              value={this.state.currentSecretKey}
+              onChange={e => {
+                this.setState({ currentSecretKey: e.target.value })
+              }}
+              id="currentSecretKey"
+              label="Current Secret Key"
+              name="currentSecretKey"
+              type={this.state.currentSecretKeyVisible ? "text" : "password"}
+              spellCheck="false"
+              required="required"
+              autoComplete="false"
+              align="ig-left"
+            />
+          </div>
+
+          <div className="has-toggle-password m-t-30">
+            {!serverInfo.userInfo.isIAMUser && (
+              <InputGroup
+                value={this.state.newAccessKey}
+                id="newAccessKey"
+                label={"New Access Key"}
+                name="newAccesskey"
+                type="text"
+                spellCheck="false"
+                required="required"
+                autoComplete="false"
+                align="ig-left"
+                onChange={e => {
+                  this.setState({ newAccessKey: e.target.value })
+                }}
+                readonly={serverInfo.userInfo.isIAMUser}
+              />
             )}
-            className={
-              "toggle-password fa fa-eye " +
-              (this.state.secretKeyVisible ? "toggled" : "")
-            }
-          />
-          <InputGroup
-            value={this.state.secretKey}
-            onChange={this.secretKeyChange.bind(this)}
-            id="secretKey"
-            label="Secret Key"
-            name="accesskey"
-            type={this.state.secretKeyVisible ? "text" : "password"}
-            spellCheck="false"
-            required="required"
-            autoComplete="false"
-            align="ig-left"
-            readonly={this.state.keysReadOnly}
-          />
+
+            <i
+              onClick={() => {
+                this.setState({
+                  newSecretKeyVisible: !this.state.newSecretKeyVisible
+                })
+              }}
+              className={
+                "toggle-password fa fa-eye " +
+                (this.state.newSecretKeyVisible ? "toggled" : "")
+              }
+            />
+            <InputGroup
+              value={this.state.newSecretKey}
+              onChange={e => {
+                this.setState({ newSecretKey: e.target.value })
+              }}
+              id="newSecretKey"
+              label="New Secret Key"
+              name="newSecretKey"
+              type={this.state.newSecretKeyVisible ? "text" : "password"}
+              spellCheck="false"
+              required="required"
+              autoComplete="false"
+              align="ig-left"
+              onChange={e => {
+                this.setState({ newSecretKey: e.target.value })
+              }}
+            />
+          </div>
         </ModalBody>
         <div className="modal-footer">
           <button
             id="generate-keys"
-            className={
-              "btn btn-primary " + (this.state.keysReadOnly ? "hidden" : "")
-            }
+            className={"btn btn-primary"}
             onClick={this.generateAuth.bind(this)}
           >
             Generate
           </button>
           <button
             id="update-keys"
-            className={
-              "btn btn-success " + (this.state.keysReadOnly ? "hidden" : "")
-            }
+            className={classNames({
+              btn: true,
+              "btn-success": this.canUpdateCredentials()
+            })}
+            disabled={!this.canUpdateCredentials()}
             onClick={this.setAuth.bind(this)}
           >
             Update
@@ -198,4 +278,7 @@ const mapDispatchToProps = dispatch => {
   }
 }
 
-export default connect(mapStateToProps, mapDispatchToProps)(ChangePasswordModal)
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps
+)(ChangePasswordModal)

browser/app/js/browser/__tests__/ChangePasswordModal.test.js

@@ -17,21 +17,38 @@
 import React from "react"
 import { shallow, mount } from "enzyme"
 import { ChangePasswordModal } from "../ChangePasswordModal"
+import jwtDecode from "jwt-decode"
+
+jest.mock("jwt-decode")
+
+jwtDecode.mockImplementation(() => ({ sub: "minio" }))
 
 jest.mock("../../web", () => ({
-  GetAuth: jest.fn(() => {
-    return Promise.resolve({ accessKey: "test1", secretKey: "test2" })
-  }),
   GenerateAuth: jest.fn(() => {
     return Promise.resolve({ accessKey: "gen1", secretKey: "gen2" })
   }),
-  SetAuth: jest.fn(({ accessKey, secretKey }) => {
-    if (accessKey == "test3" && secretKey == "test4") {
-      return Promise.resolve({})
-    } else {
-      return Promise.reject({ message: "Error" })
+  SetAuth: jest.fn(
+    ({ currentAccessKey, currentSecretKey, newAccessKey, newSecretKey }) => {
+      if (
+        currentAccessKey == "minio" &&
+        currentSecretKey == "minio123" &&
+        newAccessKey == "test" &&
+        newSecretKey == "test1234"
+      ) {
+        return Promise.resolve({})
+      } else {
+        return Promise.reject({
+          message: "Error"
+        })
+      }
     }
-  })
+  ),
+  GetToken: jest.fn(() => "")
+}))
+
+jest.mock("../../utils", () => ({
+  getRandomAccessKey: () => "raccesskey",
+  getRandomSecretKey: () => "rsecretkey"
 }))
 
 describe("ChangePasswordModal", () => {
@@ -40,57 +57,112 @@ describe("ChangePasswordModal", () => {
     memory: "test",
     platform: "test",
     runtime: "test",
-    info: { isEnvCreds: false }
+    info: { isEnvCreds: false },
+    userInfo: { isIAMUser: false }
   }
 
   it("should render without crashing", () => {
     shallow(<ChangePasswordModal serverInfo={serverInfo} />)
   })
 
-  it("should get the keys when its rendered", () => {
-    const wrapper = shallow(<ChangePasswordModal serverInfo={serverInfo} />)
-    setImmediate(() => {
-      expect(wrapper.state("accessKey")).toBe("test1")
-      expect(wrapper.state("secretKey")).toBe("test2")
-    })
+  it("should not allow changing password when isWorm is true", () => {
+    const newServerInfo = { ...serverInfo, info: { isWorm: true } }
+    const wrapper = shallow(<ChangePasswordModal serverInfo={newServerInfo} />)
+    expect(
+      wrapper
+        .find("ModalBody")
+        .childAt(0)
+        .text()
+    ).toBe("Credentials of this user cannot be updated through MinIO Browser.")
   })
 
-  it("should show readonly keys when isEnvCreds is true", () => {
-    const newServerInfo = { ...serverInfo, info: { isEnvCreds: true } }
+  it("should not allow changing password when isEnvCreds is true and not IAM user", () => {
+    const newServerInfo = {
+      ...serverInfo,
+      info: { isEnvCreds: true },
+      userInfo: { isIAMUser: false }
+    }
     const wrapper = shallow(<ChangePasswordModal serverInfo={newServerInfo} />)
-    expect(wrapper.state("accessKey")).toBe("xxxxxxxxx")
-    expect(wrapper.state("secretKey")).toBe("xxxxxxxxx")
-    expect(wrapper.find("#accessKey").prop("readonly")).toBeTruthy()
-    expect(wrapper.find("#secretKey").prop("readonly")).toBeTruthy()
-    expect(wrapper.find("#generate-keys").hasClass("hidden")).toBeTruthy()
-    expect(wrapper.find("#update-keys").hasClass("hidden")).toBeTruthy()
+    expect(
+      wrapper
+        .find("ModalBody")
+        .childAt(0)
+        .text()
+    ).toBe("Credentials of this user cannot be updated through MinIO Browser.")
   })
 
   it("should generate accessKey and secretKey when Generate buttons is clicked", () => {
     const wrapper = shallow(<ChangePasswordModal serverInfo={serverInfo} />)
     wrapper.find("#generate-keys").simulate("click")
     setImmediate(() => {
-      expect(wrapper.state("accessKey")).toBe("gen1")
-      expect(wrapper.state("secretKey")).toBe("gen2")
+      expect(wrapper.state("newAccessKey")).toBe("raccesskey")
+      expect(wrapper.state("newSecretKey")).toBe("rsecretkey")
     })
   })
 
+  it("should not generate accessKey for IAM User", () => {
+    const newServerInfo = {
+      ...serverInfo,
+      userInfo: { isIAMUser: true }
+    }
+    const wrapper = shallow(<ChangePasswordModal serverInfo={newServerInfo} />)
+    wrapper.find("#generate-keys").simulate("click")
+    setImmediate(() => {
+      expect(wrapper.state("newAccessKey")).toBe("minio")
+      expect(wrapper.state("newSecretKey")).toBe("rsecretkey")
+    })
+  })
+
+  it("should not show new accessKey field for IAM User", () => {
+    const newServerInfo = {
+      ...serverInfo,
+      userInfo: { isIAMUser: true }
+    }
+    const wrapper = shallow(<ChangePasswordModal serverInfo={newServerInfo} />)
+    expect(wrapper.find("#newAccesskey").exists()).toBeFalsy()
+  })
+
+  it("should disble Update button for invalid accessKey or secretKey", () => {
+    const showAlert = jest.fn()
+    const wrapper = shallow(
+      <ChangePasswordModal serverInfo={serverInfo} showAlert={showAlert} />
+    )
+    wrapper
+      .find("#currentAccessKey")
+      .simulate("change", { target: { value: "minio" } })
+    wrapper
+      .find("#currentSecretKey")
+      .simulate("change", { target: { value: "minio123" } })
+    wrapper.find("#newAccessKey").simulate("change", { target: { value: "t" } })
+    wrapper
+      .find("#newSecretKey")
+      .simulate("change", { target: { value: "t1" } })
+    expect(wrapper.find("#update-keys").prop("disabled")).toBeTruthy()
+  })
+
   it("should update accessKey and secretKey when Update button is clicked", () => {
     const showAlert = jest.fn()
     const wrapper = shallow(
       <ChangePasswordModal serverInfo={serverInfo} showAlert={showAlert} />
     )
     wrapper
-      .find("#accessKey")
-      .simulate("change", { target: { value: "test3" } })
+      .find("#currentAccessKey")
+      .simulate("change", { target: { value: "minio" } })
     wrapper
-      .find("#secretKey")
-      .simulate("change", { target: { value: "test4" } })
+      .find("#currentSecretKey")
+      .simulate("change", { target: { value: "minio123" } })
+    wrapper
+      .find("#newAccessKey")
+      .simulate("change", { target: { value: "test" } })
+    wrapper
+      .find("#newSecretKey")
+      .simulate("change", { target: { value: "test1234" } })
+    expect(wrapper.find("#update-keys").prop("disabled")).toBeFalsy()
     wrapper.find("#update-keys").simulate("click")
     setImmediate(() => {
       expect(showAlert).toHaveBeenCalledWith({
         type: "success",
-        message: "Changed credentials"
+        message: "Credentials updated successfully."
       })
     })
   })

browser/app/js/browser/actions.js

@@ -34,7 +34,7 @@ export const fetchStorageInfo = () => {
     return web.StorageInfo().then(res => {
       const storageInfo = {
         total: res.storageInfo.Total,
-        used:  res.storageInfo.Used
+        used: res.storageInfo.Used
       }
       dispatch(setStorageInfo(storageInfo))
     })
@@ -54,7 +54,8 @@ export const fetchServerInfo = () => {
         memory: res.MinioMemory,
         platform: res.MinioPlatform,
         runtime: res.MinioRuntime,
-        info: res.MinioGlobalInfo
+        info: res.MinioGlobalInfo,
+        userInfo: res.MinioUserInfo
       }
       dispatch(setServerInfo(serverInfo))
     })

browser/app/js/constants.js

@@ -28,3 +28,6 @@ export const NONE = "none"
 export const SHARE_OBJECT_EXPIRY_DAYS = 5
 export const SHARE_OBJECT_EXPIRY_HOURS = 0
 export const SHARE_OBJECT_EXPIRY_MINUTES = 0
+
+export const ACCESS_KEY_MIN_LENGTH = 3
+export const SECRET_KEY_MIN_LENGTH = 8

browser/app/js/utils.js

@@ -14,11 +14,11 @@
  * limitations under the License.
  */
 
-import { minioBrowserPrefix } from './constants.js'
+import { minioBrowserPrefix } from "./constants.js"
 
 export const sortObjectsByName = (objects, order) => {
-  let folders = objects.filter(object => object.name.endsWith('/'))
-  let files = objects.filter(object => !object.name.endsWith('/'))
+  let folders = objects.filter(object => object.name.endsWith("/"))
+  let files = objects.filter(object => !object.name.endsWith("/"))
   folders = folders.sort((a, b) => {
     if (a.name.toLowerCase() < b.name.toLowerCase()) return -1
     if (a.name.toLowerCase() > b.name.toLowerCase()) return 1
@@ -37,32 +37,34 @@ export const sortObjectsByName = (objects, order) => {
 }
 
 export const sortObjectsBySize = (objects, order) => {
-  let folders = objects.filter(object => object.name.endsWith('/'))
-  let files = objects.filter(object => !object.name.endsWith('/'))
+  let folders = objects.filter(object => object.name.endsWith("/"))
+  let files = objects.filter(object => !object.name.endsWith("/"))
   files = files.sort((a, b) => a.size - b.size)
-  if (order)
-    files = files.reverse()
+  if (order) files = files.reverse()
   return [...folders, ...files]
 }
 
 export const sortObjectsByDate = (objects, order) => {
-  let folders = objects.filter(object => object.name.endsWith('/'))
-  let files = objects.filter(object => !object.name.endsWith('/'))
-  files = files.sort((a, b) => new Date(a.lastModified).getTime() - new Date(b.lastModified).getTime())
-  if (order)
-    files = files.reverse()
+  let folders = objects.filter(object => object.name.endsWith("/"))
+  let files = objects.filter(object => !object.name.endsWith("/"))
+  files = files.sort(
+    (a, b) =>
+      new Date(a.lastModified).getTime() - new Date(b.lastModified).getTime()
+  )
+  if (order) files = files.reverse()
   return [...folders, ...files]
 }
 
-export const pathSlice = (path) => {
-  path = path.replace(minioBrowserPrefix, '')
-  let prefix = ''
-  let bucket = ''
-  if (!path) return {
+export const pathSlice = path => {
+  path = path.replace(minioBrowserPrefix, "")
+  let prefix = ""
+  let bucket = ""
+  if (!path)
+    return {
       bucket,
       prefix
-  }
-  let objectIndex = path.indexOf('/', 1)
+    }
+  let objectIndex = path.indexOf("/", 1)
   if (objectIndex == -1) {
     bucket = path.slice(1)
     return {
@@ -79,7 +81,29 @@ export const pathSlice = (path) => {
 }
 
 export const pathJoin = (bucket, prefix) => {
-  if (!prefix)
-    prefix = ''
-  return minioBrowserPrefix + '/' + bucket + '/' + prefix
+  if (!prefix) prefix = ""
+  return minioBrowserPrefix + "/" + bucket + "/" + prefix
+}
+
+export const getRandomAccessKey = () => {
+  const alphaNumericTable = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+  let arr = new Uint8Array(20)
+  window.crypto.getRandomValues(arr)
+  const random = Array.prototype.map.call(arr, v => {
+    const i = v % alphaNumericTable.length
+    return alphaNumericTable.charAt(i)
+  })
+  return random.join("")
+}
+
+export const getRandomSecretKey = () => {
+  let arr = new Uint8Array(40)
+  window.crypto.getRandomValues(arr)
+  const binStr = Array.prototype.map
+    .call(arr, v => {
+      return String.fromCharCode(v)
+    })
+    .join("")
+  const base64Str = btoa(binStr)
+  return base64Str.replace(/\//g, "+").substr(0, 40)
 }

browser/app/js/web.js

@@ -72,6 +72,9 @@ class Web {
   Logout() {
     storage.removeItem('token')
   }
+  GetToken() {
+    return storage.getItem('token')
+  }
   ServerInfo() {
     return this.makeCall('ServerInfo')
   }
@@ -99,12 +102,6 @@ class Web {
   RemoveObject(args) {
     return this.makeCall('RemoveObject', args)
   }
-  GetAuth() {
-    return this.makeCall('GetAuth')
-  }
-  GenerateAuth() {
-    return this.makeCall('GenerateAuth')
-  }
   SetAuth(args) {
     return this.makeCall('SetAuth', args)
       .then(res => {

browser/app/less/inc/modal.less

@@ -190,8 +190,8 @@
 ----------------------------*/
 .toggle-password {
     position: absolute;
-    bottom: 30px;
-    right: 35px;
+    bottom: 0   ;
+    right: 0;
     width: 30px;
     height: 30px;
     border: 1px solid #eee;
@@ -206,6 +206,10 @@
         background: #eee;
     }
 }
+
+.has-toggle-password {
+    position: relative;
+}
 //--------------------------
 
 

browser/package.json

@@ -29,8 +29,8 @@
   "homepage": "https://github.com/minio/minio",
   "devDependencies": {
     "async": "^1.5.2",
-    "babel-cli": "^6.14.0",
-    "babel-core": "^6.14.0",
+    "babel-cli": "^6.26.0",
+    "babel-core": "^6.26.3",
     "babel-jest": "^22.1.0",
     "babel-loader": "^7.1.2",
     "babel-plugin-syntax-object-rest-spread": "^6.13.0",
@@ -38,10 +38,10 @@
     "babel-polyfill": "^6.23.0",
     "babel-preset-es2015": "^6.14.0",
     "babel-preset-react": "^6.11.1",
-    "babel-register": "^6.14.0",
+    "babel-register": "^6.26.0",
     "copy-webpack-plugin": "^4.6.0",
     "css-loader": "^0.23.1",
-    "enzyme": "^3.3.0",
+    "enzyme": "^3.10.0",
     "enzyme-adapter-react-16": "^1.1.1",
     "esformatter": "^0.10.0",
     "esformatter-jsx": "^7.4.1",
@@ -68,11 +68,12 @@
     "humanize": "0.0.9",
     "identity-obj-proxy": "^3.0.0",
     "json-loader": "^0.5.4",
+    "jwt-decode": "^2.2.0",
     "local-storage-fallback": "^4.0.2",
     "material-design-iconic-font": "^2.2.0",
     "mime-db": "^1.25.0",
     "mime-types": "^2.1.13",
-    "moment": "^2.15.1",
+    "moment": "^2.24.0",
     "react": "^16.2.0",
     "react-addons-test-utils": "^0.14.8",
     "react-bootstrap": "^0.32.0",

browser/yarn.lock

@@ -251,22 +251,10 @@ ajv@^6.1.0:
     json-schema-traverse "^0.4.1"
     uri-js "^4.2.2"
 
-align-text@^0.1.1, align-text@^0.1.3:
-  version "0.1.4"
-  resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117"
-  dependencies:
-    kind-of "^3.0.2"
-    longest "^1.0.1"
-    repeat-string "^1.5.2"
-
 alphanum-sort@^1.0.1, alphanum-sort@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/alphanum-sort/-/alphanum-sort-1.0.2.tgz#97a1119649b211ad33691d9f9f486a8ec9fbe0a3"
 
-amdefine@>=0.0.4:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/amdefine/-/amdefine-1.0.1.tgz#4a5282ac164729e93619bcfd3ad151f817ce91f5"
-
 ansi-colors@^3.0.0:
   version "3.2.3"
   resolved "https://registry.yarnpkg.com/ansi-colors/-/ansi-colors-3.2.3.tgz#57d35b8686e851e2cc04c403f1c00203976a1813"
@@ -378,6 +366,11 @@ array-equal@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/array-equal/-/array-equal-1.0.0.tgz#8c2a5ef2472fd9ea742b04c77a75093ba2757c93"
 
+array-filter@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/array-filter/-/array-filter-1.0.0.tgz#baf79e62e6ef4c2a4c0b831232daffec251f9d83"
+  integrity sha1-uveeYubvTCpMC4MSMtr/7CUfnYM=
+
 array-flatten@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2"
@@ -410,6 +403,15 @@ array-unique@^0.3.2:
   resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428"
   integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=
 
+array.prototype.flat@^1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.2.1.tgz#812db8f02cad24d3fab65dd67eabe3b8903494a4"
+  integrity sha512-rVqIs330nLJvfC7JqYvEWwqVr5QjYF1ib02i3YJtR/fICO6527Tjpc/e4Mvmxh3GIePPreRXMdaGyC99YphWEw==
+  dependencies:
+    define-properties "^1.1.2"
+    es-abstract "^1.10.0"
+    function-bind "^1.1.1"
+
 arrify@^1.0.0, arrify@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d"
@@ -467,7 +469,7 @@ async-limiter@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.0.tgz#78faed8c3d074ab81f22b4e985d79e8738f720f8"
 
-async@^1.4.0, async@^1.5.2:
+async@^1.5.2:
   version "1.5.2"
   resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a"
 
@@ -520,24 +522,25 @@ aws4@^1.6.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e"
 
-babel-cli@^6.14.0:
-  version "6.24.1"
-  resolved "https://registry.yarnpkg.com/babel-cli/-/babel-cli-6.24.1.tgz#207cd705bba61489b2ea41b5312341cf6aca2283"
+babel-cli@^6.26.0:
+  version "6.26.0"
+  resolved "https://registry.yarnpkg.com/babel-cli/-/babel-cli-6.26.0.tgz#502ab54874d7db88ad00b887a06383ce03d002f1"
+  integrity sha1-UCq1SHTX24itALiHoGODzgPQAvE=
   dependencies:
-    babel-core "^6.24.1"
-    babel-polyfill "^6.23.0"
-    babel-register "^6.24.1"
-    babel-runtime "^6.22.0"
-    commander "^2.8.1"
-    convert-source-map "^1.1.0"
+    babel-core "^6.26.0"
+    babel-polyfill "^6.26.0"
+    babel-register "^6.26.0"
+    babel-runtime "^6.26.0"
+    commander "^2.11.0"
+    convert-source-map "^1.5.0"
     fs-readdir-recursive "^1.0.0"
-    glob "^7.0.0"
-    lodash "^4.2.0"
-    output-file-sync "^1.1.0"
-    path-is-absolute "^1.0.0"
+    glob "^7.1.2"
+    lodash "^4.17.4"
+    output-file-sync "^1.1.2"
+    path-is-absolute "^1.0.1"
     slash "^1.0.0"
-    source-map "^0.5.0"
-    v8flags "^2.0.10"
+    source-map "^0.5.6"
+    v8flags "^2.1.1"
   optionalDependencies:
     chokidar "^1.6.1"
 
@@ -581,29 +584,30 @@ babel-core@^6.0.0, babel-core@^6.26.0:
     slash "^1.0.0"
     source-map "^0.5.6"
 
-babel-core@^6.14.0, babel-core@^6.24.1:
-  version "6.25.0"
-  resolved "https://registry.yarnpkg.com/babel-core/-/babel-core-6.25.0.tgz#7dd42b0463c742e9d5296deb3ec67a9322dad729"
+babel-core@^6.26.3:
+  version "6.26.3"
+  resolved "https://registry.yarnpkg.com/babel-core/-/babel-core-6.26.3.tgz#b2e2f09e342d0f0c88e2f02e067794125e75c207"
+  integrity sha512-6jyFLuDmeidKmUEb3NM+/yawG0M2bDZ9Z1qbZP59cyHLz8kYGKYwpJP0UwUKKUiTRNvxfLesJnTedqczP7cTDA==
   dependencies:
-    babel-code-frame "^6.22.0"
-    babel-generator "^6.25.0"
+    babel-code-frame "^6.26.0"
+    babel-generator "^6.26.0"
     babel-helpers "^6.24.1"
     babel-messages "^6.23.0"
-    babel-register "^6.24.1"
-    babel-runtime "^6.22.0"
-    babel-template "^6.25.0"
-    babel-traverse "^6.25.0"
-    babel-types "^6.25.0"
-    babylon "^6.17.2"
-    convert-source-map "^1.1.0"
-    debug "^2.1.1"
-    json5 "^0.5.0"
-    lodash "^4.2.0"
-    minimatch "^3.0.2"
-    path-is-absolute "^1.0.0"
-    private "^0.1.6"
+    babel-register "^6.26.0"
+    babel-runtime "^6.26.0"
+    babel-template "^6.26.0"
+    babel-traverse "^6.26.0"
+    babel-types "^6.26.0"
+    babylon "^6.18.0"
+    convert-source-map "^1.5.1"
+    debug "^2.6.9"
+    json5 "^0.5.1"
+    lodash "^4.17.4"
+    minimatch "^3.0.4"
+    path-is-absolute "^1.0.1"
+    private "^0.1.8"
     slash "^1.0.0"
-    source-map "^0.5.0"
+    source-map "^0.5.7"
 
 babel-generator@^6.18.0, babel-generator@^6.26.0:
   version "6.26.0"
@@ -618,19 +622,6 @@ babel-generator@^6.18.0, babel-generator@^6.26.0:
     source-map "^0.5.6"
     trim-right "^1.0.1"
 
-babel-generator@^6.25.0:
-  version "6.25.0"
-  resolved "https://registry.yarnpkg.com/babel-generator/-/babel-generator-6.25.0.tgz#33a1af70d5f2890aeb465a4a7793c1df6a9ea9fc"
-  dependencies:
-    babel-messages "^6.23.0"
-    babel-runtime "^6.22.0"
-    babel-types "^6.25.0"
-    detect-indent "^4.0.0"
-    jsesc "^1.3.0"
-    lodash "^4.2.0"
-    source-map "^0.5.0"
-    trim-right "^1.0.1"
-
 babel-helper-builder-react-jsx@^6.24.1:
   version "6.24.1"
   resolved "https://registry.yarnpkg.com/babel-helper-builder-react-jsx/-/babel-helper-builder-react-jsx-6.24.1.tgz#0ad7917e33c8d751e646daca4e77cc19377d2cbc"
@@ -997,6 +988,15 @@ babel-polyfill@6.23.0, babel-polyfill@^6.23.0:
     core-js "^2.4.0"
     regenerator-runtime "^0.10.0"
 
+babel-polyfill@^6.26.0:
+  version "6.26.0"
+  resolved "https://registry.yarnpkg.com/babel-polyfill/-/babel-polyfill-6.26.0.tgz#379937abc67d7895970adc621f284cd966cf2153"
+  integrity sha1-N5k3q8Z9eJWXCtxiHyhM2WbPIVM=
+  dependencies:
+    babel-runtime "^6.26.0"
+    core-js "^2.5.0"
+    regenerator-runtime "^0.10.5"
+
 babel-preset-es2015@^6.14.0:
   version "6.24.1"
   resolved "https://registry.yarnpkg.com/babel-preset-es2015/-/babel-preset-es2015-6.24.1.tgz#d44050d6bc2c9feea702aaf38d727a0210538939"
@@ -1050,18 +1050,6 @@ babel-preset-react@^6.11.1:
     babel-plugin-transform-react-jsx-source "^6.22.0"
     babel-preset-flow "^6.23.0"
 
-babel-register@^6.14.0, babel-register@^6.24.1:
-  version "6.24.1"
-  resolved "https://registry.yarnpkg.com/babel-register/-/babel-register-6.24.1.tgz#7e10e13a2f71065bdfad5a1787ba45bca6ded75f"
-  dependencies:
-    babel-core "^6.24.1"
-    babel-runtime "^6.22.0"
-    core-js "^2.4.0"
-    home-or-tmp "^2.0.0"
-    lodash "^4.2.0"
-    mkdirp "^0.5.1"
-    source-map-support "^0.4.2"
-
 babel-register@^6.26.0:
   version "6.26.0"
   resolved "https://registry.yarnpkg.com/babel-register/-/babel-register-6.26.0.tgz#6ed021173e2fcb486d7acb45c6009a856f647071"
@@ -1098,7 +1086,7 @@ babel-template@^6.16.0, babel-template@^6.26.0:
     babylon "^6.18.0"
     lodash "^4.17.4"
 
-babel-template@^6.24.1, babel-template@^6.25.0:
+babel-template@^6.24.1:
   version "6.25.0"
   resolved "https://registry.yarnpkg.com/babel-template/-/babel-template-6.25.0.tgz#665241166b7c2aa4c619d71e192969552b10c071"
   dependencies:
@@ -1509,10 +1497,6 @@ camel-case@3.0.x:
     no-case "^2.2.0"
     upper-case "^1.1.1"
 
-camelcase@^1.0.2:
-  version "1.2.1"
-  resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-1.2.1.tgz#9bb5304d2e0b56698b2c758b08a3eaa9daa58a39"
-
 camelcase@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-4.1.0.tgz#d545635be1e33c542649c69173e5de6acfae34dd"
@@ -1540,13 +1524,6 @@ caseless@~0.12.0:
   resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
   integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=
 
-center-align@^0.1.1:
-  version "0.1.3"
-  resolved "https://registry.yarnpkg.com/center-align/-/center-align-0.1.3.tgz#aa0d32629b6ee972200411cbd4461c907bc2b7ad"
-  dependencies:
-    align-text "^0.1.3"
-    lazy-cache "^1.0.3"
-
 chain-function@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/chain-function/-/chain-function-1.0.0.tgz#0d4ab37e7e18ead0bdc47b920764118ce58733dc"
@@ -1695,14 +1672,6 @@ cli-width@^2.0.0:
   resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-2.2.0.tgz#ff19ede8a9a5e579324147b0c11f0fbcbabed639"
   integrity sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=
 
-cliui@^2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/cliui/-/cliui-2.1.0.tgz#4b475760ff80264c762c3a1719032e91c7fea0d1"
-  dependencies:
-    center-align "^0.1.1"
-    right-align "^0.1.1"
-    wordwrap "0.0.2"
-
 cliui@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/cliui/-/cliui-4.0.0.tgz#743d4650e05f36d1ed2575b59638d87322bfbbcc"
@@ -1802,7 +1771,12 @@ commander@2.17.x, commander@~2.17.1:
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.17.1.tgz#bd77ab7de6de94205ceacc72f1716d29f20a77bf"
   integrity sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==
 
-commander@^2.2.0, commander@^2.8.1, commander@^2.9.0:
+commander@^2.11.0, commander@~2.20.0:
+  version "2.20.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.0.tgz#d58bb2b5c1ee8f87b0d340027e9e94e222c5a422"
+  integrity sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==
+
+commander@^2.2.0, commander@^2.9.0:
   version "2.10.0"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.10.0.tgz#e1f5d3245de246d1a5ca04702fa1ad1bd7e405fe"
   dependencies:
@@ -1893,14 +1867,17 @@ content-type@~1.0.4:
   resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b"
   integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==
 
-convert-source-map@^1.1.0:
-  version "1.5.0"
-  resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.5.0.tgz#9acd70851c6d5dfdd93d9282e5edf94a03ff46b5"
-
 convert-source-map@^1.4.0, convert-source-map@^1.5.0:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.5.1.tgz#b8278097b9bc229365de5c62cf5fcaed8b5599e5"
 
+convert-source-map@^1.5.1:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.6.0.tgz#51b537a8c43e0f04dec1993bffcdd504e758ac20"
+  integrity sha512-eFu7XigvxdZ1ETfbgPBohgyQ/Z++C0eEhTor0qRwBw9unw+L0/6V8wkSuGgzdThkiS5lSpdptOQPD8Ak40a+7A==
+  dependencies:
+    safe-buffer "~5.1.1"
+
 cookie-signature@1.0.6:
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c"
@@ -2170,7 +2147,7 @@ date-now@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b"
 
-debug@2.6.9, debug@^2.1.2, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8:
+debug@2.6.9, debug@^2.1.2, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
   dependencies:
@@ -2187,7 +2164,7 @@ debug@^0.7.4:
   version "0.7.4"
   resolved "https://registry.yarnpkg.com/debug/-/debug-0.7.4.tgz#06e1ea8082c2cb14e39806e22e2f6f757f92af39"
 
-debug@^2.1.1, debug@^2.1.3:
+debug@^2.1.3:
   version "2.6.8"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.8.tgz#e731531ca2ede27d188222427da17821d68ff4fc"
   dependencies:
@@ -2207,7 +2184,7 @@ debug@^4.1.0:
   dependencies:
     ms "^2.1.1"
 
-decamelize@^1.0.0, decamelize@^1.1.1, decamelize@^1.1.2, decamelize@^1.2.0:
+decamelize@^1.1.1, decamelize@^1.1.2, decamelize@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
   integrity sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=
@@ -2627,26 +2604,32 @@ enzyme-to-json@^3.3.0:
   dependencies:
     lodash "^4.17.4"
 
-enzyme@^3.3.0:
-  version "3.3.0"
-  resolved "https://registry.yarnpkg.com/enzyme/-/enzyme-3.3.0.tgz#0971abd167f2d4bf3f5bd508229e1c4b6dc50479"
+enzyme@^3.10.0:
+  version "3.10.0"
+  resolved "https://registry.yarnpkg.com/enzyme/-/enzyme-3.10.0.tgz#7218e347c4a7746e133f8e964aada4a3523452f6"
+  integrity sha512-p2yy9Y7t/PFbPoTvrWde7JIYB2ZyGC+NgTNbVEGvZ5/EyoYSr9aG/2rSbVvyNvMHEhw9/dmGUJHWtfQIEiX9pg==
   dependencies:
+    array.prototype.flat "^1.2.1"
     cheerio "^1.0.0-rc.2"
-    function.prototype.name "^1.0.3"
-    has "^1.0.1"
+    function.prototype.name "^1.1.0"
+    has "^1.0.3"
+    html-element-map "^1.0.0"
     is-boolean-object "^1.0.0"
-    is-callable "^1.1.3"
+    is-callable "^1.1.4"
     is-number-object "^1.0.3"
+    is-regex "^1.0.4"
     is-string "^1.0.4"
     is-subset "^0.1.1"
-    lodash "^4.17.4"
-    object-inspect "^1.5.0"
+    lodash.escape "^4.0.1"
+    lodash.isequal "^4.5.0"
+    object-inspect "^1.6.0"
     object-is "^1.0.1"
     object.assign "^4.1.0"
     object.entries "^1.0.4"
     object.values "^1.0.4"
     raf "^3.4.0"
     rst-selector-parser "^2.2.3"
+    string.prototype.trim "^1.1.2"
 
 errno@^0.1.1, errno@^0.1.3, errno@~0.1.7:
   version "0.1.7"
@@ -2662,6 +2645,18 @@ error-ex@^1.2.0:
   dependencies:
     is-arrayish "^0.2.1"
 
+es-abstract@^1.10.0, es-abstract@^1.5.0:
+  version "1.13.0"
+  resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.13.0.tgz#ac86145fdd5099d8dd49558ccba2eaf9b88e24e9"
+  integrity sha512-vDZfg/ykNxQVwup/8E1BZhVzFfBxs9NqMzGcvIJrqg5k2/5Za2bWo40dK2J1pgLngZ7c+Shh8lwYtLGyrwPutg==
+  dependencies:
+    es-to-primitive "^1.2.0"
+    function-bind "^1.1.1"
+    has "^1.0.3"
+    is-callable "^1.1.4"
+    is-regex "^1.0.4"
+    object-keys "^1.0.12"
+
 es-abstract@^1.5.1:
   version "1.10.0"
   resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.10.0.tgz#1ecb36c197842a00d8ee4c2dfd8646bb97d60864"
@@ -2681,7 +2676,7 @@ es-abstract@^1.6.1:
     is-callable "^1.1.3"
     is-regex "^1.0.3"
 
-es-to-primitive@^1.1.1:
+es-to-primitive@^1.1.1, es-to-primitive@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.0.tgz#edf72478033456e8dda8ef09e00ad9650707f377"
   integrity sha512-qZryBOJjV//LaxLTV6UC//WewneB3LcXOL9NP++ozKVXsIIIpm/2c13UDiD9Jp2eThsecw9m3jPqDwTyobcdbg==
@@ -3364,27 +3359,28 @@ fstream-ignore@^1.0.5:
     minimatch "^3.0.0"
 
 fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2:
-  version "1.0.11"
-  resolved "https://registry.yarnpkg.com/fstream/-/fstream-1.0.11.tgz#5c1fb1f117477114f0632a0eb4b71b3cb0fd3171"
-  integrity sha1-XB+x8RdHcRTwYyoOtLcbPLD9MXE=
+  version "1.0.12"
+  resolved "https://registry.yarnpkg.com/fstream/-/fstream-1.0.12.tgz#4e8ba8ee2d48be4f7d0de505455548eae5932045"
+  integrity sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==
   dependencies:
     graceful-fs "^4.1.2"
     inherits "~2.0.0"
     mkdirp ">=0.5 0"
     rimraf "2"
 
-function-bind@^1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.0.tgz#16176714c801798e4e8f2cf7f7529467bb4a5771"
-
-function-bind@^1.1.1:
+function-bind@^1.0.2, function-bind@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"
   integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==
 
-function.prototype.name@^1.0.3:
+function-bind@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.0.tgz#16176714c801798e4e8f2cf7f7529467bb4a5771"
+
+function.prototype.name@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.0.tgz#8bd763cc0af860a859cc5d49384d74b932cd2327"
+  integrity sha512-Bs0VRrTz4ghD8pTmbJQD1mZ8A/mN0ur/jGz+A6FBxPDUPkm1tNfF6bhTYPA7i7aF4lZJVr+OXTNNrnnIl58Wfg==
   dependencies:
     define-properties "^1.1.2"
     function-bind "^1.1.1"
@@ -3471,7 +3467,19 @@ glob-parent@^3.1.0:
     is-glob "^3.1.0"
     path-dirname "^1.0.0"
 
-glob@^7.0.0, glob@^7.1.1, glob@^7.1.2:
+glob@^7.0.3, glob@^7.0.5:
+  version "7.1.3"
+  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.3.tgz#3960832d3f1574108342dafd3a67b332c0969df1"
+  integrity sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==
+  dependencies:
+    fs.realpath "^1.0.0"
+    inflight "^1.0.4"
+    inherits "2"
+    minimatch "^3.0.4"
+    once "^1.3.0"
+    path-is-absolute "^1.0.0"
+
+glob@^7.1.1, glob@^7.1.2:
   version "7.1.2"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
   dependencies:
@@ -3482,10 +3490,10 @@ glob@^7.0.0, glob@^7.1.1, glob@^7.1.2:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
-glob@^7.0.3, glob@^7.0.5, glob@^7.1.3:
-  version "7.1.3"
-  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.3.tgz#3960832d3f1574108342dafd3a67b332c0969df1"
-  integrity sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==
+glob@^7.1.3:
+  version "7.1.4"
+  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.4.tgz#aa608a2f6c577ad357e1ae5a5c26d9a8d1969255"
+  integrity sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==
   dependencies:
     fs.realpath "^1.0.0"
     inflight "^1.0.4"
@@ -3546,11 +3554,7 @@ globby@^7.1.1:
     pify "^3.0.0"
     slash "^1.0.0"
 
-graceful-fs@^4.1.11, graceful-fs@^4.1.4:
-  version "4.1.11"
-  resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658"
-
-graceful-fs@^4.1.15, graceful-fs@^4.1.2:
+graceful-fs@^4.1.11, graceful-fs@^4.1.15, graceful-fs@^4.1.2, graceful-fs@^4.1.4:
   version "4.1.15"
   resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.15.tgz#ffb703e1066e8a0eeaa4c8b80ba9253eeefbfb00"
   integrity sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==
@@ -3570,14 +3574,15 @@ handle-thing@^2.0.0:
   integrity sha512-d4sze1JNC454Wdo2fkuyzCr6aHcbL6PGGuFAz0Li/NcOm1tCHGnWDRmJP85dh9IhQErTc2svWFEX5xHIOo//kQ==
 
 handlebars@^4.0.3:
-  version "4.0.11"
-  resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.11.tgz#630a35dfe0294bc281edae6ffc5d329fc7982dcc"
+  version "4.1.2"
+  resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.1.2.tgz#b6b37c1ced0306b221e094fc7aca3ec23b131b67"
+  integrity sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==
   dependencies:
-    async "^1.4.0"
+    neo-async "^2.6.0"
     optimist "^0.6.1"
-    source-map "^0.4.4"
+    source-map "^0.6.1"
   optionalDependencies:
-    uglify-js "^2.6"
+    uglify-js "^3.1.4"
 
 har-schema@^1.0.5:
   version "1.0.5"
@@ -3667,7 +3672,7 @@ has-values@^1.0.0:
     is-number "^3.0.0"
     kind-of "^4.0.0"
 
-has@^1.0.1:
+has@^1.0.1, has@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796"
   integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==
@@ -3781,6 +3786,13 @@ html-comment-regex@^1.1.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/html-comment-regex/-/html-comment-regex-1.1.1.tgz#668b93776eaae55ebde8f3ad464b307a4963625e"
 
+html-element-map@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/html-element-map/-/html-element-map-1.0.1.tgz#3c4fcb4874ebddfe4283b51c8994e7713782b592"
+  integrity sha512-BZSfdEm6n706/lBfXKWa4frZRZcT5k1cOusw95ijZsHlI+GdgY0v95h6IzO3iIDf2ROwq570YTwqNPqHcNMozw==
+  dependencies:
+    array-filter "^1.0.0"
+
 html-encoding-sniffer@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz#e70d84b94da53aa375e11fe3a351be6642ca46f8"
@@ -4907,6 +4919,11 @@ jsprim@^1.2.2:
     json-schema "0.2.3"
     verror "1.10.0"
 
+jwt-decode@^2.2.0:
+  version "2.2.0"
+  resolved "https://registry.yarnpkg.com/jwt-decode/-/jwt-decode-2.2.0.tgz#7d86bd56679f58ce6a84704a657dd392bba81a79"
+  integrity sha1-fYa9VmefWM5qhHBKZX3TkruoGnk=
+
 keycode@^2.1.2:
   version "2.1.9"
   resolved "https://registry.yarnpkg.com/keycode/-/keycode-2.1.9.tgz#964a23c54e4889405b4861a5c9f0480d45141dfa"
@@ -4940,10 +4957,6 @@ kind-of@^6.0.0, kind-of@^6.0.2:
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.2.tgz#01146b36a6218e64e58f3a8d66de5d7fc6f6d051"
   integrity sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==
 
-lazy-cache@^1.0.3:
-  version "1.0.4"
-  resolved "https://registry.yarnpkg.com/lazy-cache/-/lazy-cache-1.0.4.tgz#a1d78fc3a50474cb80845d3b3b6e1da49a446e8e"
-
 lcid@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/lcid/-/lcid-1.0.0.tgz#308accafa0bc483a3867b4b6f2b9506251d1b835"
@@ -5109,6 +5122,11 @@ lodash.deburr@^3.0.0:
   dependencies:
     lodash._root "^3.0.0"
 
+lodash.escape@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/lodash.escape/-/lodash.escape-4.0.1.tgz#c9044690c21e04294beaa517712fded1fa88de98"
+  integrity sha1-yQRGkMIeBClL6qUXcS/e0fqI3pg=
+
 lodash.flattendeep@^4.4.0:
   version "4.4.0"
   resolved "https://registry.yarnpkg.com/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz#fb030917f86a3134e5bc9bec0d69e0013ddfedb2"
@@ -5128,6 +5146,11 @@ lodash.isequal@^3.0:
     lodash._baseisequal "^3.0.0"
     lodash._bindcallback "^3.0.0"
 
+lodash.isequal@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0"
+  integrity sha1-QVxEePK8wwEgwizhDtMib30+GOA=
+
 lodash.isplainobject@^4.0.6:
   version "4.0.6"
   resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb"
@@ -5176,10 +5199,6 @@ loglevel@^1.4.1:
   resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.6.1.tgz#e0fc95133b6ef276cdc8887cdaf24aa6f156f8fa"
   integrity sha1-4PyVEztu8nbNyIh82vJKpvFW+Po=
 
-longest@^1.0.1:
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097"
-
 loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.2.0, loose-envify@^1.3.1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.3.1.tgz#d1a8ad33fa9ce0e713d65fdd0ac8b748d478c848"
@@ -5220,8 +5239,9 @@ lru-cache@^5.1.1:
     yallist "^3.0.2"
 
 macaddress@^0.2.8:
-  version "0.2.8"
-  resolved "https://registry.yarnpkg.com/macaddress/-/macaddress-0.2.8.tgz#5904dc537c39ec6dbefeae902327135fa8511f12"
+  version "0.2.9"
+  resolved "https://registry.yarnpkg.com/macaddress/-/macaddress-0.2.9.tgz#3579b8b9acd5b96b4553abf0f394185a86813cb3"
+  integrity sha512-k4F1JUof6cQXxNFzx3thLby4oJzXTXQueAOOts944Vqizn+Rjc2QNFenT9FJSLU1CH3PmrHRSyZs2E+Cqw+P2w==
 
 make-dir@^1.0.0:
   version "1.3.0"
@@ -5441,7 +5461,7 @@ minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.3, minimatch@^3.0.4:
   dependencies:
     brace-expansion "^1.1.7"
 
-minimist@0.0.8, minimist@~0.0.1:
+minimist@0.0.8:
   version "0.0.8"
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d"
   integrity sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=
@@ -5456,6 +5476,11 @@ minimist@^0.1.0:
   resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.1.0.tgz#99df657a52574c21c9057497df742790b2b4c0de"
   integrity sha1-md9lelJXTCHJBXSX33QnkLK0wN4=
 
+minimist@~0.0.1:
+  version "0.0.10"
+  resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf"
+  integrity sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8=
+
 minipass@^2.2.1, minipass@^2.3.4:
   version "2.3.5"
   resolved "https://registry.yarnpkg.com/minipass/-/minipass-2.3.5.tgz#cacebe492022497f656b0f0f51e2682a9ed2d848"
@@ -5517,9 +5542,10 @@ mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkd
   dependencies:
     minimist "0.0.8"
 
-moment@^2.15.1:
-  version "2.18.1"
-  resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f"
+moment@^2.24.0:
+  version "2.24.0"
+  resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b"
+  integrity sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==
 
 "mout@>=0.9 <2.0":
   version "1.0.0"
@@ -5617,10 +5643,10 @@ negotiator@0.6.1:
   resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.1.tgz#2b327184e8992101177b28563fb5e7102acd0ca9"
   integrity sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=
 
-neo-async@^2.5.0:
-  version "2.6.0"
-  resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.0.tgz#b9d15e4d71c6762908654b5183ed38b753340835"
-  integrity sha512-MFh0d/Wa7vkKO3Y3LlacqAEeHK0mckVqzDieUKTT+KGxi+zIpeVsFxymkIiRpbpDziHc290Xr9A1O4Om7otoRA==
+neo-async@^2.5.0, neo-async@^2.6.0:
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.1.tgz#ac27ada66167fa8849a6addd837f6b189ad2081c"
+  integrity sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw==
 
 nice-try@^1.0.4:
   version "1.0.5"
@@ -5848,8 +5874,9 @@ number-is-nan@^1.0.0:
   integrity sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=
 
 nwmatcher@^1.4.3:
-  version "1.4.3"
-  resolved "https://registry.yarnpkg.com/nwmatcher/-/nwmatcher-1.4.3.tgz#64348e3b3d80f035b40ac11563d278f8b72db89c"
+  version "1.4.4"
+  resolved "https://registry.yarnpkg.com/nwmatcher/-/nwmatcher-1.4.4.tgz#2285631f34a95f0d0395cd900c96ed39b58f346e"
+  integrity sha512-3iuY4N5dhgMpCUrOVnuAdGrgxVqV2cJpM+XNccjR2DKOB1RUP0aA+wGXEiNziG/UKboFyGBIoKOaNlJxx8bciQ==
 
 oauth-sign@~0.8.1, oauth-sign@~0.8.2:
   version "0.8.2"
@@ -5872,9 +5899,10 @@ object-inspect@^1.1.0:
   version "1.2.2"
   resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.2.2.tgz#c82115e4fcc888aea14d64c22e4f17f6a70d5e5a"
 
-object-inspect@^1.5.0:
-  version "1.5.0"
-  resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.5.0.tgz#9d876c11e40f485c79215670281b767488f9bfe3"
+object-inspect@^1.6.0:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.6.0.tgz#c70b6cbf72f274aab4c34c0c82f5167bf82cf15b"
+  integrity sha512-GJzfBZ6DgDAmnuaM3104jR4s1Myxr3Y3zfIyN4z3UdqN69oSRacNK8UhnobDdC+7J2AHCjGwxQubNJfE70SXXQ==
 
 object-is@^1.0.1:
   version "1.0.1"
@@ -5965,6 +5993,7 @@ on-headers@~1.0.1:
 once@^1.3.0, once@^1.3.1, once@^1.3.3, once@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
+  integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E=
   dependencies:
     wrappy "1"
 
@@ -6005,6 +6034,7 @@ opn@^5.1.0:
 optimist@^0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/optimist/-/optimist-0.6.1.tgz#da3ea74686fa21a19a111c326e90eb15a0196686"
+  integrity sha1-2j6nRob6IaGaERwybpDrFaAZZoY=
   dependencies:
     minimist "~0.0.1"
     wordwrap "~0.0.2"
@@ -6066,9 +6096,10 @@ osenv@^0.1.4:
     os-homedir "^1.0.0"
     os-tmpdir "^1.0.0"
 
-output-file-sync@^1.1.0:
+output-file-sync@^1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/output-file-sync/-/output-file-sync-1.1.2.tgz#d0a33eefe61a205facb90092e826598d5245ce76"
+  integrity sha1-0KM+7+YaIF+suQCS6CZZjVJFznY=
   dependencies:
     graceful-fs "^4.1.4"
     mkdirp "^0.5.1"
@@ -6642,7 +6673,7 @@ private@^0.1.6:
   version "0.1.7"
   resolved "https://registry.yarnpkg.com/private/-/private-0.1.7.tgz#68ce5e8a1ef0a23bb570cc28537b5332aba63ef1"
 
-private@^0.1.7:
+private@^0.1.7, private@^0.1.8:
   version "0.1.8"
   resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff"
 
@@ -7153,7 +7184,7 @@ regenerate@^1.2.1:
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.3.2.tgz#d1941c67bad437e1be76433add5b385f95b19260"
 
-regenerator-runtime@^0.10.0:
+regenerator-runtime@^0.10.0, regenerator-runtime@^0.10.5:
   version "0.10.5"
   resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.10.5.tgz#336c3efc1220adcedda2c9fab67b5a7955a33658"
 
@@ -7386,25 +7417,13 @@ ret@~0.1.10:
   version "0.1.15"
   resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc"
 
-right-align@^0.1.1:
-  version "0.1.3"
-  resolved "https://registry.yarnpkg.com/right-align/-/right-align-0.1.3.tgz#61339b722fe6a3515689210d24e14c96148613ef"
-  dependencies:
-    align-text "^0.1.1"
-
-rimraf@2, rimraf@^2.2.8, rimraf@^2.5.1, rimraf@^2.6.1, rimraf@^2.6.2:
+rimraf@2, rimraf@^2.2.8, rimraf@^2.5.1, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.2:
   version "2.6.3"
   resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab"
   integrity sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==
   dependencies:
     glob "^7.1.3"
 
-rimraf@^2.5.4:
-  version "2.6.2"
-  resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36"
-  dependencies:
-    glob "^7.0.5"
-
 ripemd160@^2.0.0, ripemd160@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/ripemd160/-/ripemd160-2.0.1.tgz#0f4584295c53a3628af7e6d79aca21ce57d1c6e7"
@@ -7774,12 +7793,6 @@ source-map-support@^0.4.15:
   dependencies:
     source-map "^0.5.6"
 
-source-map-support@^0.4.2:
-  version "0.4.15"
-  resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.4.15.tgz#03202df65c06d2bd8c7ec2362a193056fef8d3b1"
-  dependencies:
-    source-map "^0.5.6"
-
 source-map-support@^0.5.0:
   version "0.5.3"
   resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.3.tgz#2b3d5fff298cfa4d1afd7d4352d569e9a0158e76"
@@ -7799,17 +7812,7 @@ source-map-url@^0.4.0:
   resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.0.tgz#3e935d7ddd73631b97659956d55128e87b5084a3"
   integrity sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=
 
-source-map@^0.4.4:
-  version "0.4.4"
-  resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b"
-  dependencies:
-    amdefine ">=0.0.4"
-
-source-map@^0.5.0, source-map@^0.5.3:
-  version "0.5.6"
-  resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.6.tgz#75ce38f52bf0733c5a7f0c118d81334a2bb5f412"
-
-source-map@^0.5.6, source-map@~0.5.1, source-map@~0.5.6:
+source-map@^0.5.3, source-map@^0.5.6, source-map@^0.5.7, source-map@~0.5.6:
   version "0.5.7"
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc"
   integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=
@@ -7817,6 +7820,7 @@ source-map@^0.5.6, source-map@~0.5.1, source-map@~0.5.6:
 source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.0, source-map@~0.6.1:
   version "0.6.1"
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
+  integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
 
 spdx-correct@^3.0.0:
   version "3.1.0"
@@ -8002,6 +8006,15 @@ string-width@^2.0.0:
     is-fullwidth-code-point "^2.0.0"
     strip-ansi "^3.0.0"
 
+string.prototype.trim@^1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.1.2.tgz#d04de2c89e137f4d7d206f086b5ed2fae6be8cea"
+  integrity sha1-0E3iyJ4Tf019IG8Ia17S+ua+jOo=
+  dependencies:
+    define-properties "^1.1.2"
+    es-abstract "^1.5.0"
+    function-bind "^1.0.2"
+
 string_decoder@^1.0.0, string_decoder@~1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.3.tgz#0fc67d7c141825de94282dd536bec6b9bce860ab"
@@ -8400,18 +8413,13 @@ uglify-js@3.4.x:
     commander "~2.17.1"
     source-map "~0.6.1"
 
-uglify-js@^2.6:
-  version "2.8.29"
-  resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd"
+uglify-js@^3.1.4:
+  version "3.6.0"
+  resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.6.0.tgz#704681345c53a8b2079fb6cec294b05ead242ff5"
+  integrity sha512-W+jrUHJr3DXKhrsS7NUVxn3zqMOFn0hL/Ei6v0anCIMoKC93TjcflTagwIHLW7SfMFfiQuktQyFVCFHGUE0+yg==
   dependencies:
-    source-map "~0.5.1"
-    yargs "~3.10.0"
-  optionalDependencies:
-    uglify-to-browserify "~1.0.0"
-
-uglify-to-browserify@~1.0.0:
-  version "1.0.2"
-  resolved "https://registry.yarnpkg.com/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz#6e0924d6bda6b5afe349e39a6d632850a0f882b7"
+    commander "~2.20.0"
+    source-map "~0.6.1"
 
 uid-number@^0.0.6:
   version "0.0.6"
@@ -8584,9 +8592,10 @@ v8-compile-cache@^2.0.2:
   resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.0.2.tgz#a428b28bb26790734c4fc8bc9fa106fccebf6a6c"
   integrity sha512-1wFuMUIM16MDJRCrpbpuEPTUGmM5QMUg0cr3KFwra2XgOgFcPGDQHDh3CszSCD2Zewc/dh/pamNEW8CbfDebUw==
 
-v8flags@^2.0.10:
+v8flags@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/v8flags/-/v8flags-2.1.1.tgz#aab1a1fa30d45f88dd321148875ac02c0b55e5b4"
+  integrity sha1-qrGh+jDUX4jdMhFIh1rALAtV5bQ=
   dependencies:
     user-home "^1.1.1"
 
@@ -8847,17 +8856,10 @@ wide-align@^1.1.0:
   dependencies:
     string-width "^1.0.2 || 2"
 
-window-size@0.1.0:
-  version "0.1.0"
-  resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.1.0.tgz#5438cd2ea93b202efa3a19fe8887aee7c94f9c9d"
-
-wordwrap@0.0.2:
-  version "0.0.2"
-  resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-0.0.2.tgz#b79669bb42ecb409f83d583cad52ca17eaa1643f"
-
 wordwrap@~0.0.2:
   version "0.0.3"
   resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-0.0.3.tgz#a3d5da6cd5c0bc0008d37234bbaf1bed63059107"
+  integrity sha1-o9XabNXAvAAI03I0u68b7WMFkQc=
 
 wordwrap@~1.0.0:
   version "1.0.0"
@@ -9011,12 +9013,3 @@ yargs@~1.2.6:
   integrity sha1-nHtKgv1dWVsr8Xq23MQxNUMv40s=
   dependencies:
     minimist "^0.1.0"
-
-yargs@~3.10.0:
-  version "3.10.0"
-  resolved "https://registry.yarnpkg.com/yargs/-/yargs-3.10.0.tgz#f7ee7bd857dd7c1d2d38c0e74efbd681d1431fd1"
-  dependencies:
-    camelcase "^1.0.2"
-    cliui "^2.1.0"
-    decamelize "^1.0.0"
-    window-size "0.1.0"

cmd/admin-handlers.go

@@ -940,8 +940,12 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 
 	vars := mux.Vars(r)
 	accessKey := vars["accessKey"]
-	if err := globalIAMSys.DeleteUser(accessKey); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+	// Notify all other MinIO peers to delete user.
+	for _, nerr := range globalNotificationSys.DeleteUser(accessKey) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
 	}
 }
 
@@ -1006,8 +1010,8 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	// Notify all other MinIO peers to reload users
-	for _, nerr := range globalNotificationSys.LoadUsers() {
+	// Notify all other MinIO peers to reload user.
+	for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 			logger.LogIf(ctx, nerr.Err)
@@ -1065,8 +1069,8 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Notify all other MinIO peers to reload users
-	for _, nerr := range globalNotificationSys.LoadUsers() {
+	// Notify all other Minio peers to reload user
+	for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 			logger.LogIf(ctx, nerr.Err)
@@ -1083,7 +1087,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	policies, err := globalIAMSys.ListCannedPolicies()
+	policies, err := globalIAMSys.ListPolicies()
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1115,13 +1119,13 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	if err := globalIAMSys.DeleteCannedPolicy(policyName); err != nil {
+	if err := globalIAMSys.DeletePolicy(policyName); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Notify all other MinIO peers to reload users
-	for _, nerr := range globalNotificationSys.LoadUsers() {
+	// Notify all other MinIO peers to delete policy
+	for _, nerr := range globalNotificationSys.DeletePolicy(policyName) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 			logger.LogIf(ctx, nerr.Err)
@@ -1171,13 +1175,13 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	if err = globalIAMSys.SetCannedPolicy(policyName, *iamPolicy); err != nil {
+	if err = globalIAMSys.SetPolicy(policyName, *iamPolicy); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Notify all other MinIO peers to reload users
-	for _, nerr := range globalNotificationSys.LoadUsers() {
+	// Notify all other MinIO peers to reload policy
+	for _, nerr := range globalNotificationSys.LoadPolicy(policyName) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 			logger.LogIf(ctx, nerr.Err)
@@ -1214,8 +1218,8 @@ func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 	}
 
-	// Notify all other MinIO peers to reload users
-	for _, nerr := range globalNotificationSys.LoadUsers() {
+	// Notify all other Minio peers to reload user
+	for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 			logger.LogIf(ctx, nerr.Err)
@@ -1415,3 +1419,40 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 	// Send success response
 	writeSuccessResponseHeadersOnly(w)
 }
+
+// TraceHandler - POST /minio/admin/v1/trace
+// ----------
+// The handler sends http trace to the connected HTTP client.
+func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "HTTPTrace")
+	trcAll := r.URL.Query().Get("all") == "true"
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+	// Avoid reusing tcp connection if read timeout is hit
+	// This is needed to make r.Context().Done() work as
+	// expected in case of read timeout
+	w.Header().Add("Connection", "close")
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	traceCh := globalTrace.Trace(doneCh, trcAll)
+	for {
+		select {
+		case entry := <-traceCh:
+			if _, err := w.Write(entry); err != nil {
+				return
+			}
+			if _, err := w.Write([]byte("\n")); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-r.Context().Done():
+			return
+		case <-GlobalServiceDoneCh:
+			return
+		}
+	}
+}

cmd/admin-handlers_test.go

@@ -99,6 +99,8 @@ var (
         "enable": false,
         "brokers": null,
         "topic": "",
+        "queueDir": "",
+        "queueLimit": 0,
         "tls": {
           "enable": false,
           "skipVerify": false,
@@ -266,7 +268,9 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	initNSLock(isDistXL)
 
 	// Init global heal state
-	initAllHealState(globalIsXL)
+	if globalIsXL {
+		globalAllHealState = initHealState()
+	}
 
 	globalConfigSys = NewConfigSys()
 

cmd/admin-heal-ops.go

@@ -96,22 +96,15 @@ type allHealState struct {
 	healSeqMap map[string]*healSequence
 }
 
-var (
-	// global server heal state
-	globalAllHealState allHealState
-)
-
-// initAllHealState - initialize healing apparatus
-func initAllHealState(isErasureMode bool) {
-	if !isErasureMode {
-		return
-	}
-
-	globalAllHealState = allHealState{
+// initHealState - initialize healing apparatus
+func initHealState() *allHealState {
+	healState := &allHealState{
 		healSeqMap: make(map[string]*healSequence),
 	}
 
-	go globalAllHealState.periodicHealSeqsClean()
+	go healState.periodicHealSeqsClean()
+
+	return healState
 }
 
 func (ahs *allHealState) periodicHealSeqsClean() {
@@ -305,6 +298,14 @@ type healSequence struct {
 	// path is just pathJoin(bucket, objPrefix)
 	path string
 
+	// List of entities (format, buckets, objects) to heal
+	sourceCh chan string
+
+	// Report healing progress, false if this is a background
+	// healing since currently there is no entity which will
+	// receive realtime healing status
+	reportProgress bool
+
 	// time at which heal sequence was started
 	startTime time.Time
 
@@ -348,14 +349,15 @@ func newHealSequence(bucket, objPrefix, clientAddr string,
 	ctx := logger.SetReqInfo(context.Background(), reqInfo)
 
 	return &healSequence{
-		bucket:        bucket,
-		objPrefix:     objPrefix,
-		path:          pathJoin(bucket, objPrefix),
-		startTime:     UTCNow(),
-		clientToken:   mustGetUUID(),
-		clientAddress: clientAddr,
-		forceStarted:  forceStart,
-		settings:      hs,
+		bucket:         bucket,
+		objPrefix:      objPrefix,
+		path:           pathJoin(bucket, objPrefix),
+		reportProgress: true,
+		startTime:      UTCNow(),
+		clientToken:    mustGetUUID(),
+		clientAddress:  clientAddr,
+		forceStarted:   forceStart,
+		settings:       hs,
 		currentStatus: healSequenceStatus{
 			Summary:      healNotStartedStatus,
 			HealSettings: hs,
@@ -484,7 +486,11 @@ func (h *healSequence) healSequenceStart() {
 	h.currentStatus.StartTime = UTCNow()
 	h.currentStatus.updateLock.Unlock()
 
-	go h.traverseAndHeal()
+	if h.sourceCh == nil {
+		go h.traverseAndHeal()
+	} else {
+		go h.healFromSourceCh()
+	}
 
 	select {
 	case err, ok := <-h.traverseAndHealDoneCh:
@@ -519,6 +525,89 @@ func (h *healSequence) healSequenceStart() {
 	}
 }
 
+func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType) error {
+	var respCh = make(chan healResult)
+	defer close(respCh)
+	// Send heal request
+	globalBackgroundHealing.queueHealTask(healTask{path: path, responseCh: respCh, opts: h.settings})
+	// Wait for answer and push result to the client
+	res := <-respCh
+	if !h.reportProgress {
+		return nil
+	}
+	res.result.Type = healType
+	if res.err != nil {
+		// Object might have been deleted, by the time heal
+		// was attempted, we should ignore this object and return success.
+		if isErrObjectNotFound(res.err) {
+			return nil
+		}
+		// Only report object error
+		if healType != madmin.HealItemObject {
+			return res.err
+		}
+		res.result.Detail = res.err.Error()
+	}
+	return h.pushHealResultItem(res.result)
+}
+
+func (h *healSequence) healItemsFromSourceCh() error {
+	// Start healing the config prefix.
+	if err := h.healMinioSysMeta(minioConfigPrefix)(); err != nil {
+		return err
+	}
+
+	// Start healing the bucket config prefix.
+	if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil {
+		return err
+	}
+
+	for path := range h.sourceCh {
+		var itemType madmin.HealItemType
+		switch {
+		case path == "/":
+			itemType = madmin.HealItemMetadata
+		case !strings.Contains(path, "/"):
+			itemType = madmin.HealItemBucket
+		default:
+			itemType = madmin.HealItemObject
+		}
+
+		if err := h.queueHealTask(path, itemType); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (h *healSequence) healFromSourceCh() {
+	if err := h.healItemsFromSourceCh(); err != nil {
+		h.traverseAndHealDoneCh <- err
+	}
+	close(h.traverseAndHealDoneCh)
+}
+
+func (h *healSequence) healItems() error {
+	// Start with format healing
+	if err := h.healDiskFormat(); err != nil {
+		return err
+	}
+
+	// Start healing the config prefix.
+	if err := h.healMinioSysMeta(minioConfigPrefix)(); err != nil {
+		return err
+	}
+
+	// Start healing the bucket config prefix.
+	if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil {
+		return err
+	}
+
+	// Heal buckets and objects
+	return h.healBuckets()
+}
+
 // traverseAndHeal - traverses on-disk data and performs healing
 // according to settings. At each "safe" point it also checks if an
 // external quit signal has been received and quits if so. Since the
@@ -527,31 +616,10 @@ func (h *healSequence) healSequenceStart() {
 // has to wait until a safe point is reached, such as between scanning
 // two objects.
 func (h *healSequence) traverseAndHeal() {
-	var err error
-	checkErr := func(f func() error) {
-		switch {
-		case err != nil:
-			return
-		case h.isQuitting():
+	if err := h.healItems(); err != nil {
+		if h.isQuitting() {
 			err = errHealStopSignalled
-			return
 		}
-		err = f()
-	}
-
-	// Start with format healing
-	checkErr(h.healDiskFormat)
-
-	// Start healing the config prefix.
-	checkErr(h.healMinioSysMeta(minioConfigPrefix))
-
-	// Start healing the bucket config prefix.
-	checkErr(h.healMinioSysMeta(bucketConfigPrefix))
-
-	// Heal buckets and objects
-	checkErr(h.healBuckets)
-
-	if err != nil {
 		h.traverseAndHealDoneCh <- err
 	}
 
@@ -575,17 +643,14 @@ func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
 			if h.isQuitting() {
 				return errHealStopSignalled
 			}
-			res, herr := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun, h.settings.Remove, h.settings.ScanMode)
+
+			herr := h.queueHealTask(pathJoin(bucket, object), madmin.HealItemBucketMetadata)
 			// Object might have been deleted, by the time heal
 			// was attempted we ignore this object an move on.
 			if isErrObjectNotFound(herr) {
 				return nil
 			}
-			if herr != nil {
-				return herr
-			}
-			res.Type = madmin.HealItemBucketMetadata
-			return h.pushHealResultItem(res)
+			return herr
 		})
 	}
 }
@@ -603,26 +668,7 @@ func (h *healSequence) healDiskFormat() error {
 		return errServerNotInitialized
 	}
 
-	res, err := objectAPI.HealFormat(h.ctx, h.settings.DryRun)
-	// return any error, ignore error returned when disks have
-	// already healed.
-	if err != nil && err != errNoHealRequired {
-		return errFnHealFromAPIErr(h.ctx, err)
-	}
-
-	// Healing succeeded notify the peers to reload format and re-initialize disks.
-	// We will not notify peers only if healing succeeded.
-	if err == nil {
-		for _, nerr := range globalNotificationSys.ReloadFormat(h.settings.DryRun) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(h.ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(h.ctx, nerr.Err)
-			}
-		}
-	}
-
-	// Push format heal result
-	return h.pushHealResultItem(res)
+	return h.queueHealTask("/", madmin.HealItemMetadata)
 }
 
 // healBuckets - check for all buckets heal or just particular bucket.
@@ -664,13 +710,7 @@ func (h *healSequence) healBucket(bucket string) error {
 		return errServerNotInitialized
 	}
 
-	result, err := objectAPI.HealBucket(h.ctx, bucket, h.settings.DryRun, h.settings.Remove)
-	// handle heal-bucket error
-	if err != nil {
-		return err
-	}
-
-	if err = h.pushHealResultItem(result); err != nil {
+	if err := h.queueHealTask(bucket, madmin.HealItemBucket); err != nil {
 		return err
 	}
 
@@ -678,7 +718,7 @@ func (h *healSequence) healBucket(bucket string) error {
 		if h.objPrefix != "" {
 			// Check if an object named as the objPrefix exists,
 			// and if so heal it.
-			_, err = objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix, ObjectOptions{})
+			_, err := objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix, ObjectOptions{})
 			if err == nil {
 				if err = h.healObject(bucket, h.objPrefix); err != nil {
 					return err
@@ -689,8 +729,7 @@ func (h *healSequence) healBucket(bucket string) error {
 		return nil
 	}
 
-	if err = objectAPI.HealObjects(h.ctx, bucket,
-		h.objPrefix, h.healObject); err != nil {
+	if err := objectAPI.HealObjects(h.ctx, bucket, h.objPrefix, h.healObject); err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}
 	return nil
@@ -702,28 +741,11 @@ func (h *healSequence) healObject(bucket, object string) error {
 		return errHealStopSignalled
 	}
 
-	if globalHTTPServer != nil {
-		// Wait at max 1 minute for an inprogress request
-		// before proceeding to heal
-		waitCount := 60
-		// Any requests in progress, delay the heal.
-		for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
-			waitCount--
-			time.Sleep(1 * time.Second)
-		}
-	}
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
 
-	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun, h.settings.Remove, h.settings.ScanMode)
-	if isErrObjectNotFound(err) {
-		return nil
-	}
-	if err != nil {
-		hri.Detail = err.Error()
-	}
-	return h.pushHealResultItem(hri)
+	return h.queueHealTask(pathJoin(bucket, object), madmin.HealItemObject)
 }

cmd/admin-router.go

@@ -116,6 +116,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 	// Top locks
 	adminV1Router.Methods(http.MethodGet).Path("/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
 
+	// HTTP Trace
+	adminV1Router.Methods(http.MethodGet).Path("/trace").HandlerFunc(adminAPI.TraceHandler)
 	// If none of the routes match, return error.
 	adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandlerJSON))
 }

cmd/api-errors.go

@@ -27,7 +27,7 @@ import (
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 	"google.golang.org/api/googleapi"
 
-	minio "github.com/minio/minio-go"
+	minio "github.com/minio/minio-go/v6"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
@@ -51,6 +51,7 @@ type APIErrorResponse struct {
 	Key        string `xml:"Key,omitempty" json:"Key,omitempty"`
 	BucketName string `xml:"BucketName,omitempty" json:"BucketName,omitempty"`
 	Resource   string
+	Region     string `xml:"Region,omitempty" json:"Region,omitempty"`
 	RequestID  string `xml:"RequestId" json:"RequestId"`
 	HostID     string `xml:"HostId" json:"HostId"`
 }
@@ -879,16 +880,6 @@ var errorCodes = errorCodeMap{
 		Description:    "Object name already exists as a directory.",
 		HTTPStatusCode: http.StatusConflict,
 	},
-	ErrReadQuorum: {
-		Code:           "XMinioReadQuorum",
-		Description:    "Multiple disk failures, unable to reconstruct data.",
-		HTTPStatusCode: http.StatusServiceUnavailable,
-	},
-	ErrWriteQuorum: {
-		Code:           "XMinioWriteQuorum",
-		Description:    "Multiple disks failures, unable to write data.",
-		HTTPStatusCode: http.StatusServiceUnavailable,
-	},
 	ErrInvalidObjectName: {
 		Code:           "XMinioInvalidObjectName",
 		Description:    "Object name contains unsupported characters.",
@@ -1534,7 +1525,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrKMSAuthFailure
 	case errOperationTimedOut, context.Canceled, context.DeadlineExceeded:
 		apiErr = ErrOperationTimedOut
-	case errNetworkConnReset:
+	case errDiskNotFound:
 		apiErr = ErrSlowDown
 	}
 
@@ -1593,9 +1584,9 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case InvalidPart:
 		apiErr = ErrInvalidPart
 	case InsufficientWriteQuorum:
-		apiErr = ErrWriteQuorum
+		apiErr = ErrSlowDown
 	case InsufficientReadQuorum:
-		apiErr = ErrReadQuorum
+		apiErr = ErrSlowDown
 	case UnsupportedDelimiter:
 		apiErr = ErrNotImplemented
 	case InvalidMarkerPrefixCombination:
@@ -1745,6 +1736,7 @@ func getAPIErrorResponse(ctx context.Context, err APIError, resource, requestID,
 		BucketName: reqInfo.BucketName,
 		Key:        reqInfo.ObjectName,
 		Resource:   resource,
+		Region:     globalServerConfig.GetRegion(),
 		RequestID:  requestID,
 		HostID:     hostID,
 	}

cmd/api-errors_test.go

@@ -39,8 +39,8 @@ var toAPIErrorTests = []struct {
 	{err: ObjectNameInvalid{}, errCode: ErrInvalidObjectName},
 	{err: InvalidUploadID{}, errCode: ErrNoSuchUpload},
 	{err: InvalidPart{}, errCode: ErrInvalidPart},
-	{err: InsufficientReadQuorum{}, errCode: ErrReadQuorum},
-	{err: InsufficientWriteQuorum{}, errCode: ErrWriteQuorum},
+	{err: InsufficientReadQuorum{}, errCode: ErrSlowDown},
+	{err: InsufficientWriteQuorum{}, errCode: ErrSlowDown},
 	{err: UnsupportedDelimiter{}, errCode: ErrNotImplemented},
 	{err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented},
 	{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},

cmd/api-response.go

@@ -22,6 +22,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"strconv"
 	"strings"
 	"time"
 
@@ -523,6 +524,7 @@ func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType
 	if mType != mimeNone {
 		w.Header().Set("Content-Type", string(mType))
 	}
+	w.Header().Set("Content-Length", strconv.Itoa(len(response)))
 	w.WriteHeader(statusCode)
 	if response != nil {
 		w.Write(response)

cmd/background-heal-ops.go

@@ -0,0 +1,160 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"time"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+// healTask represents what to heal along with options
+//   path: '/' =>  Heal disk formats along with metadata
+//   path: 'bucket/' or '/bucket/' => Heal bucket
+//   path: 'bucket/object' => Heal object
+type healTask struct {
+	path string
+	opts madmin.HealOpts
+	// Healing response will be sent here
+	responseCh chan healResult
+}
+
+// healResult represents a healing result with a possible error
+type healResult struct {
+	result madmin.HealResultItem
+	err    error
+}
+
+// healRoutine receives heal tasks, to heal buckets, objects and format.json
+type healRoutine struct {
+	tasks  chan healTask
+	doneCh chan struct{}
+}
+
+// Add a new task in the tasks queue
+func (h *healRoutine) queueHealTask(task healTask) {
+	h.tasks <- task
+}
+
+// Wait for heal requests and process them
+func (h *healRoutine) run() {
+	ctx := context.Background()
+	for {
+		select {
+		case task, ok := <-h.tasks:
+			if !ok {
+				break
+			}
+			if globalHTTPServer != nil {
+				// Wait at max 1 minute for an inprogress request
+				// before proceeding to heal
+				waitCount := 60
+				// Any requests in progress, delay the heal.
+				for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
+					waitCount--
+					time.Sleep(1 * time.Second)
+				}
+			}
+
+			var res madmin.HealResultItem
+			var err error
+			bucket, object := urlPath2BucketObjectName(task.path)
+			switch {
+			case bucket == "" && object == "":
+				res, err = bgHealDiskFormat(ctx, task.opts)
+			case bucket != "" && object == "":
+				res, err = bgHealBucket(ctx, bucket, task.opts)
+			case bucket != "" && object != "":
+				res, err = bgHealObject(ctx, bucket, object, task.opts)
+			}
+			task.responseCh <- healResult{result: res, err: err}
+		case <-h.doneCh:
+			return
+		case <-GlobalServiceDoneCh:
+			return
+		}
+	}
+}
+
+func initHealRoutine() *healRoutine {
+	return &healRoutine{
+		tasks:  make(chan healTask),
+		doneCh: make(chan struct{}),
+	}
+
+}
+
+func initBackgroundHealing() {
+	healBg := initHealRoutine()
+	go healBg.run()
+
+	globalBackgroundHealing = healBg
+}
+
+// bgHealDiskFormat - heals format.json, return value indicates if a
+// failure error occurred.
+func bgHealDiskFormat(ctx context.Context, opts madmin.HealOpts) (madmin.HealResultItem, error) {
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return madmin.HealResultItem{}, errServerNotInitialized
+	}
+
+	res, err := objectAPI.HealFormat(ctx, opts.DryRun)
+
+	// return any error, ignore error returned when disks have
+	// already healed.
+	if err != nil && err != errNoHealRequired {
+		return madmin.HealResultItem{}, err
+	}
+
+	// Healing succeeded notify the peers to reload format and re-initialize disks.
+	// We will not notify peers if healing is not required.
+	if err == nil {
+		for _, nerr := range globalNotificationSys.ReloadFormat(opts.DryRun) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
+		}
+	}
+
+	return res, nil
+}
+
+// bghealBucket - traverses and heals given bucket
+func bgHealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return madmin.HealResultItem{}, errServerNotInitialized
+	}
+
+	return objectAPI.HealBucket(ctx, bucket, opts.DryRun, opts.Remove)
+}
+
+// bgHealObject - heal the given object and record result
+func bgHealObject(ctx context.Context, bucket, object string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return madmin.HealResultItem{}, errServerNotInitialized
+	}
+	return objectAPI.HealObject(ctx, bucket, object, opts.DryRun, opts.Remove, opts.ScanMode)
+}

cmd/bucket-handlers.go

@@ -30,7 +30,7 @@ import (
 
 	"github.com/gorilla/mux"
 
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"

cmd/common-main.go

@@ -29,7 +29,7 @@ import (
 	etcd "github.com/coreos/etcd/clientv3"
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/target/console"
 	"github.com/minio/minio/cmd/logger/target/http"
@@ -211,13 +211,6 @@ func handleCommonEnvVars() {
 		globalIsBrowserEnabled = bool(browserFlag)
 	}
 
-	traceFile := os.Getenv("MINIO_HTTP_TRACE")
-	if traceFile != "" {
-		var err error
-		globalHTTPTraceFile, err = os.OpenFile(traceFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660)
-		logger.FatalIf(err, "error opening file %s", traceFile)
-	}
-
 	etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
 	if ok {
 		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")

cmd/config-common.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"time"
 
 	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/minio/minio/cmd/logger"
@@ -53,8 +52,19 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
 }
 
 func deleteConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
-	_, err := client.Delete(ctx, configFile)
-	return err
+	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+
+	_, err := client.Delete(timeoutCtx, configFile)
+	if err != nil {
+		if err == context.DeadlineExceeded {
+			return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s",
+				client.Endpoints())
+		}
+		return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s",
+			err, client.Endpoints())
+	}
+	return nil
 }
 
 func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
@@ -89,9 +99,11 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
 	resp, err := client.Get(timeoutCtx, configFile)
 	if err != nil {
 		if err == context.DeadlineExceeded {
-			return nil, fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
+			return nil, fmt.Errorf("etcd setup is unreachable, please check your endpoints %s",
+				client.Endpoints())
 		}
-		return nil, fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
+		return nil, fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s",
+			err, client.Endpoints())
 	}
 	if resp.Count == 0 {
 		return nil, errConfigNotFound
@@ -104,54 +116,7 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
 	return nil, errConfigNotFound
 }
 
-// watchConfigEtcd - watches for changes on `configFile` on etcd and loads them.
-func watchConfigEtcd(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
-	for {
-		watchCh := globalEtcdClient.Watch(context.Background(), iamConfigPrefix)
-		select {
-		case <-GlobalServiceDoneCh:
-			return
-		case watchResp, ok := <-watchCh:
-			if !ok {
-				time.Sleep(1 * time.Second)
-				continue
-			}
-			if err := watchResp.Err(); err != nil {
-				logger.LogIf(context.Background(), err)
-				// log and retry.
-				time.Sleep(1 * time.Second)
-				continue
-			}
-			for _, event := range watchResp.Events {
-				if event.IsModify() || event.IsCreate() {
-					loadCfgFn(objAPI)
-				}
-			}
-		}
-	}
-}
-
-func checkConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
-	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
-	defer cancel()
-	resp, err := client.Get(timeoutCtx, configFile)
-	if err != nil {
-		if err == context.DeadlineExceeded {
-			return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
-		}
-		return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
-	}
-	if resp.Count == 0 {
-		return errConfigNotFound
-	}
-	return nil
-}
-
 func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
-	if globalEtcdClient != nil {
-		return checkConfigEtcd(ctx, globalEtcdClient, configFile)
-	}
-
 	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
 		// Treat object not found as config not found.
 		if isErrObjectNotFound(err) {

cmd/config-current.go

@@ -325,7 +325,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewKafkaTarget(k, v)
+		t, err := target.NewKafkaTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("kafka(%s): %s", k, err.Error())
 		}
@@ -667,7 +667,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.Kafka {
 		if args.Enable {
-			newTarget, err := target.NewKafkaTarget(id, args)
+			newTarget, err := target.NewKafkaTarget(id, args, GlobalServiceDoneCh)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue

cmd/config-current_test.go

@@ -200,7 +200,7 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
 
 		// Test 16 - Test Kafka
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "", "queueDir": "", "queueLimit": 0 } }}}`, false},
 
 		// Test 17 - Test Webhook
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},

cmd/config-migrate.go

@@ -29,7 +29,7 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
-	"github.com/minio/minio/pkg/iam/policy"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/iam/validator"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/quick"
@@ -2413,6 +2413,7 @@ func migrateV27ToV28() error {
 }
 
 // Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
+// if etcd is configured then migrates /config/config.json to '<export_path>/.minio.sys/config/config.json'
 func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	// Construct path to config.json for the given bucket.
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
@@ -2423,10 +2424,14 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	}
 
 	defer func() {
-		// Rename config.json to config.json.deprecated only upon
-		// success of this function.
 		if err == nil {
-			os.Rename(getConfigFile(), getConfigFile()+".deprecated")
+			if globalEtcdClient != nil {
+				deleteConfigEtcd(context.Background(), globalEtcdClient, configFile)
+			} else {
+				// Rename config.json to config.json.deprecated only upon
+				// success of this function.
+				os.Rename(getConfigFile(), getConfigFile()+".deprecated")
+			}
 		}
 	}()
 
@@ -2446,20 +2451,24 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 		return err
 	} // if errConfigNotFound proceed to migrate..
 
+	var configFiles = []string{
+		getConfigFile(),
+		getConfigFile() + ".deprecated",
+		configFile,
+	}
 	var config = &serverConfig{}
-	if _, err = Load(getConfigFile(), config); err != nil {
-		if !os.IsNotExist(err) {
-			return err
-		}
-		// Read from deprecate file as well if necessary.
-		if _, err = Load(getConfigFile()+".deprecated", config); err != nil {
+	for _, cfgFile := range configFiles {
+		if _, err = Load(cfgFile, config); err != nil {
 			if !os.IsNotExist(err) {
 				return err
 			}
-			// If all else fails simply initialize the server config.
-			return newSrvConfig(objAPI)
+			continue
 		}
-
+		break
+	}
+	if os.IsNotExist(err) {
+		// Initialize the server config, if no config exists.
+		return newSrvConfig(objAPI)
 	}
 	return saveServerConfig(context.Background(), objAPI, config)
 }

cmd/config-versions.go

@@ -884,7 +884,7 @@ type serverConfigV32 struct {
 	} `json:"policy"`
 }
 
-// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit with MQTT.
+// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in MQTT and kafka.
 type serverConfigV33 struct {
 	quick.Config `json:"-"` // ignore interfaces
 

cmd/config.go

@@ -49,10 +49,6 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
 	}
 
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	if globalEtcdClient != nil {
-		return saveConfigEtcd(ctx, globalEtcdClient, configFile, data)
-	}
-
 	// Create a backup of the current config
 	oldData, err := readConfig(ctx, objAPI, configFile)
 	if err == nil {
@@ -71,15 +67,8 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
 }
 
 func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, error) {
-	var configData []byte
-	var err error
-
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	if globalEtcdClient != nil {
-		configData, err = readConfigEtcd(ctx, globalEtcdClient, configFile)
-	} else {
-		configData, err = readConfig(ctx, objAPI, configFile)
-	}
+	configData, err := readConfig(ctx, objAPI, configFile)
 	if err != nil {
 		return nil, err
 	}
@@ -152,45 +141,25 @@ func initConfig(objAPI ObjectLayer) error {
 		return errServerNotInitialized
 	}
 
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	if globalEtcdClient != nil {
-		if err := checkConfigEtcd(context.Background(), globalEtcdClient, getConfigFile()); err != nil {
-			if err == errConfigNotFound {
-				// Migrates all configs at old location.
-				if err = migrateConfig(); err != nil {
-					return err
-				}
-				// Migrates etcd ${HOME}/.minio/config.json to '/config/config.json'
-				if err = migrateConfigToMinioSys(objAPI); err != nil {
-					return err
-				}
-			} else {
-				return err
-			}
-		}
-
-		// Watch config for changes and reloads them.
-		go watchConfigEtcd(objAPI, configFile, loadConfig)
-
-	} else {
-		if isFile(getConfigFile()) {
-			if err := migrateConfig(); err != nil {
-				return err
-			}
-		}
-		// Migrates ${HOME}/.minio/config.json or config.json.deprecated
-		// to '<export_path>/.minio.sys/config/config.json'
-		// ignore if the file doesn't exist.
-		if err := migrateConfigToMinioSys(objAPI); err != nil {
-			return err
-		}
-
-		// Migrates backend '<export_path>/.minio.sys/config/config.json' to latest version.
-		if err := migrateMinioSysConfig(objAPI); err != nil {
+	if isFile(getConfigFile()) {
+		if err := migrateConfig(); err != nil {
 			return err
 		}
 	}
 
+	// Migrates ${HOME}/.minio/config.json or config.json.deprecated
+	// to '<export_path>/.minio.sys/config/config.json'
+	// ignore if the file doesn't exist.
+	// If etcd is set then migrates /config/config.json
+	// to '<export_path>/.minio.sys/config/config.json'
+	if err := migrateConfigToMinioSys(objAPI); err != nil {
+		return err
+	}
+
+	// Migrates backend '<export_path>/.minio.sys/config/config.json' to latest version.
+	if err := migrateMinioSysConfig(objAPI); err != nil {
+		return err
+	}
+
 	return loadConfig(objAPI)
 }

cmd/daily-heal-ops.go

@@ -0,0 +1,89 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+const (
+	bgHealingUUID = "0000-0000-0000-0000"
+)
+
+// NewBgHealSequence creates a background healing sequence
+// operation which crawls all objects and heal them.
+func newBgHealSequence(numDisks int) *healSequence {
+
+	reqInfo := &logger.ReqInfo{API: "BackgroundHeal"}
+	ctx := logger.SetReqInfo(context.Background(), reqInfo)
+
+	hs := madmin.HealOpts{
+		// Remove objects that do not have read-quorum
+		Remove:   true,
+		ScanMode: madmin.HealDeepScan,
+	}
+
+	return &healSequence{
+		sourceCh:    make(chan string),
+		startTime:   UTCNow(),
+		clientToken: bgHealingUUID,
+		settings:    hs,
+		currentStatus: healSequenceStatus{
+			Summary:      healNotStartedStatus,
+			HealSettings: hs,
+			NumDisks:     numDisks,
+			updateLock:   &sync.RWMutex{},
+		},
+		traverseAndHealDoneCh: make(chan error),
+		stopSignalCh:          make(chan struct{}),
+		ctx:                   ctx,
+		reportProgress:        false,
+	}
+}
+
+func initDailyHeal() {
+	go startDailyHeal()
+}
+
+func startDailyHeal() {
+	var objAPI ObjectLayer
+	var ctx = context.Background()
+
+	// Wait until the object API is ready
+	for {
+		objAPI = newObjectLayerFn()
+		if objAPI == nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		break
+	}
+
+	// Find number of disks in the setup
+	info := objAPI.StorageInfo(ctx)
+	numDisks := info.Backend.OnlineDisks + info.Backend.OfflineDisks
+
+	nh := newBgHealSequence(numDisks)
+	globalSweepHealState.LaunchNewHealSequence(nh)
+
+	registerDailySweepListener(nh.sourceCh)
+}

cmd/daily-sweeper.go

@@ -0,0 +1,145 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+// The list of modules listening for the daily listing of all objects
+// such as the daily heal ops, disk usage and bucket lifecycle management.
+var globalDailySweepListeners = make([]chan string, 0)
+var globalDailySweepListenersMu = sync.Mutex{}
+
+// Add a new listener to the daily objects listing
+func registerDailySweepListener(ch chan string) {
+	globalDailySweepListenersMu.Lock()
+	defer globalDailySweepListenersMu.Unlock()
+
+	globalDailySweepListeners = append(globalDailySweepListeners, ch)
+}
+
+// Safe copy of globalDailySweepListeners content
+func copyDailySweepListeners() []chan string {
+	globalDailySweepListenersMu.Lock()
+	defer globalDailySweepListenersMu.Unlock()
+
+	var listenersCopy = make([]chan string, len(globalDailySweepListeners))
+	copy(listenersCopy, globalDailySweepListeners)
+
+	return listenersCopy
+}
+
+// sweepRound will list all objects, having read quorum or not and
+// feeds to all listeners, such as the background healing
+func sweepRound(ctx context.Context, objAPI ObjectLayer) error {
+	zeroDuration := time.Millisecond
+	zeroDynamicTimeout := newDynamicTimeout(zeroDuration, zeroDuration)
+
+	// General lock so we avoid parallel daily sweep by different instances.
+	sweepLock := globalNSMutex.NewNSLock("system", "daily-sweep")
+	if err := sweepLock.GetLock(zeroDynamicTimeout); err != nil {
+		return err
+	}
+	defer sweepLock.Unlock()
+
+	buckets, err := objAPI.ListBuckets(ctx)
+	if err != nil {
+		return err
+	}
+
+	// List all objects, having read quorum or not in all buckets
+	// and send them to all the registered sweep listeners
+	for _, bucket := range buckets {
+		// Send bucket names to all listeners
+		for _, l := range copyDailySweepListeners() {
+			l <- bucket.Name
+		}
+
+		marker := ""
+		for {
+			res, err := objAPI.ListObjectsHeal(ctx, bucket.Name, "", marker, "", 1000)
+			if err != nil {
+				continue
+			}
+			for _, obj := range res.Objects {
+				for _, l := range copyDailySweepListeners() {
+					l <- pathJoin(bucket.Name, obj.Name)
+				}
+			}
+			if !res.IsTruncated {
+				break
+			} else {
+				marker = res.NextMarker
+			}
+		}
+	}
+
+	return nil
+}
+
+// initDailySweeper creates a go-routine which will list all
+// objects in all buckets in a daily basis
+func initDailySweeper() {
+	go dailySweeper()
+}
+
+// List all objects in all buckets in a daily basis
+func dailySweeper() {
+	var lastSweepTime time.Time
+	var objAPI ObjectLayer
+
+	var ctx = context.Background()
+
+	// Wait until the object layer is ready
+	for {
+		objAPI = newObjectLayerFn()
+		if objAPI == nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		break
+	}
+
+	// Perform a sweep round each 24 hours
+	for {
+		if time.Since(lastSweepTime) < 24*time.Hour {
+			time.Sleep(time.Hour)
+			continue
+		}
+
+		err := sweepRound(ctx, objAPI)
+		if err != nil {
+			switch err.(type) {
+			// Unable to hold a lock means there is another
+			// instance doing the sweep round
+			case OperationTimedOut:
+				lastSweepTime = time.Now()
+			default:
+				logger.LogIf(ctx, err)
+				time.Sleep(time.Minute)
+				continue
+			}
+		} else {
+			lastSweepTime = time.Now()
+		}
+	}
+}

cmd/disk-cache.go

@@ -435,7 +435,7 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 	if delimiter == slashSeparator {
 		recursive = false
 	}
-	walkResultCh, endWalkCh := c.listPool.Release(listParams{bucket, recursive, marker, prefix})
+	walkResultCh, endWalkCh := c.listPool.Release(listParams{bucket, recursive, marker, prefix, false})
 	if walkResultCh == nil {
 		endWalkCh = make(chan struct{})
 
@@ -494,7 +494,7 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 		}
 	}
 
-	params := listParams{bucket, recursive, nextMarker, prefix}
+	params := listParams{bucket, recursive, nextMarker, prefix, false}
 	if !eof {
 		c.listPool.Set(params, walkResultCh, endWalkCh)
 	}

cmd/encryption-v1.go

@@ -29,7 +29,7 @@ import (
 	"path"
 	"strconv"
 
-	"github.com/minio/minio-go/pkg/encrypt"
+	"github.com/minio/minio-go/v6/pkg/encrypt"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/ioutil"

cmd/encryption-v1_test.go

@@ -23,7 +23,7 @@ import (
 	"testing"
 
 	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/minio-go/pkg/encrypt"
+	"github.com/minio/minio-go/v6/pkg/encrypt"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/sio"
 )

cmd/endpoint-ellipses.go

@@ -22,7 +22,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/pkg/ellipses"
 )
 
@@ -213,12 +213,12 @@ func parseEndpointSet(args ...string) (ep endpointSet, err error) {
 	return ep, nil
 }
 
-// Parses all ellipses input arguments, expands them into corresponding
-// list of endpoints chunked evenly in accordance with a specific
-// set size.
+// GetAllSets - parses all ellipses input arguments, expands them into
+// corresponding list of endpoints chunked evenly in accordance with a
+// specific set size.
 // For example: {1...64} is divided into 4 sets each of size 16.
 // This applies to even distributed setup syntax as well.
-func getAllSets(args ...string) ([][]string, error) {
+func GetAllSets(args ...string) ([][]string, error) {
 	if len(args) == 0 {
 		return nil, errInvalidArgument
 	}
@@ -266,7 +266,7 @@ func getAllSets(args ...string) ([][]string, error) {
 // CreateServerEndpoints - validates and creates new endpoints from input args, supports
 // both ellipses and without ellipses transparently.
 func createServerEndpoints(serverAddr string, args ...string) (string, EndpointList, SetupType, int, int, error) {
-	setArgs, err := getAllSets(args...)
+	setArgs, err := GetAllSets(args...)
 	if err != nil {
 		return serverAddr, nil, -1, 0, 0, err
 	}

cmd/endpoint.go

@@ -31,7 +31,7 @@ import (
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/cli"
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/disk"

cmd/fs-v1.go

@@ -30,7 +30,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/minio/minio-go/pkg/s3utils"
+	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/madmin"
@@ -431,8 +431,9 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 
 		// Save objects' metadata in `fs.json`.
 		fsMeta := newFSMetaV1()
-		if _, err = fsMeta.ReadFrom(ctx, wlk); err != nil && err != io.EOF {
-			return oi, toObjectErr(err, srcBucket, srcObject)
+		if _, err = fsMeta.ReadFrom(ctx, wlk); err != nil {
+			// For any error to read fsMeta, set default ETag and proceed.
+			fsMeta = fs.defaultFsJSON(srcObject)
 		}
 
 		fsMeta.Meta = srcInfo.UserDefined
@@ -710,10 +711,7 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
 		_, rerr := fsMeta.ReadFrom(ctx, rlk.LockedFile)
 		fs.rwPool.Close(fsMetaPath)
 		if rerr != nil {
-			if rerr != io.EOF {
-				return oi, rerr
-			}
-			// Set Default ETag, if fs.json is empty
+			// For any error to read fsMeta, set default ETag and proceed.
 			fsMeta = fs.defaultFsJSON(object)
 		}
 	}
@@ -1151,6 +1149,12 @@ func (fs *FSObjects) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error)
 	return []BucketInfo{}, NotImplemented{}
 }
 
+// ListObjectsHeal - list all objects to be healed. Valid only for XL
+func (fs *FSObjects) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
+	logger.LogIf(ctx, NotImplemented{})
+	return ListObjectsInfo{}, NotImplemented{}
+}
+
 // SetBucketPolicy sets policy on bucket
 func (fs *FSObjects) SetBucketPolicy(ctx context.Context, bucket string, policy *policy.Policy) error {
 	return savePolicyConfig(ctx, fs, bucket, policy)

cmd/gateway-common.go

@@ -24,7 +24,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
 
-	minio "github.com/minio/minio-go"
+	minio "github.com/minio/minio-go/v6"
 )
 
 var (

cmd/gateway-main.go

@@ -41,14 +41,14 @@ var (
 	gatewayCmd = cli.Command{
 		Name:            "gateway",
 		Usage:           "start object storage gateway",
-		Flags:           append(serverFlags, globalFlags...),
+		Flags:           append(ServerFlags, GlobalFlags...),
 		HideHelpCommand: true,
 	}
 )
 
 // RegisterGatewayCommand registers a new command for gateway.
 func RegisterGatewayCommand(cmd cli.Command) error {
-	cmd.Flags = append(append(cmd.Flags, append(cmd.Flags, serverFlags...)...), globalFlags...)
+	cmd.Flags = append(append(cmd.Flags, append(cmd.Flags, ServerFlags...)...), GlobalFlags...)
 	gatewayCmd.Subcommands = append(gatewayCmd.Subcommands, cmd)
 	return nil
 }
@@ -148,6 +148,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	initNSLock(false) // Enable local namespace lock.
 
+	// Set when gateway is enabled
+	globalIsGateway = true
+
 	router := mux.NewRouter().SkipClean(true)
 
 	if globalEtcdClient != nil {
@@ -155,6 +158,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		registerSTSRouter(router)
 	}
 
+	// initialize globalTrace system
+	globalTrace = NewTraceSys(context.Background(), globalEndpoints)
+
 	enableConfigOps := globalEtcdClient != nil && gatewayName == "nas"
 	enableIAMOps := globalEtcdClient != nil
 
@@ -221,6 +227,11 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.FatalIf(err, "Unable to initialize gateway backend")
 	}
 
+	// Populate existing buckets to the etcd backend
+	if globalDNSConfig != nil {
+		initFederatorBackend(newObject)
+	}
+
 	if enableConfigOps {
 		// Create a new config system.
 		globalConfigSys = NewConfigSys()

cmd/gateway-unsupported.go

@@ -101,6 +101,11 @@ func (a GatewayUnsupported) ListBucketsHeal(ctx context.Context) (buckets []Buck
 	return nil, NotImplemented{}
 }
 
+// ListObjectsHeal - Not implemented stub
+func (a GatewayUnsupported) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
+	return ListObjectsInfo{}, NotImplemented{}
+}
+
 // HealObject - Not implemented stub
 func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (h madmin.HealResultItem, e error) {
 	return h, NotImplemented{}

cmd/gateway/azure/gateway-azure.go

@@ -36,7 +36,7 @@ import (
 	"github.com/Azure/go-autorest/autorest/azure"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/cli"
-	miniogopolicy "github.com/minio/minio-go/pkg/policy"
+	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
 	"github.com/minio/minio/cmd"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"

cmd/gateway/b2/gateway-b2.go

@@ -31,7 +31,7 @@ import (
 
 	b2 "github.com/minio/blazer/base"
 	"github.com/minio/cli"
-	miniogopolicy "github.com/minio/minio-go/pkg/policy"
+	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	h2 "github.com/minio/minio/pkg/hash"

cmd/gateway/gcs/gateway-gcs.go

@@ -38,7 +38,7 @@ import (
 	"cloud.google.com/go/storage"
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/cli"
-	miniogopolicy "github.com/minio/minio-go/pkg/policy"
+	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/policy"
@@ -395,10 +395,6 @@ func (l *gcsGateway) CleanupGCSMinioSysTmp(ctx context.Context) {
 		for {
 			attrs, err := it.Next()
 			if err != nil {
-				if err != iterator.Done {
-					ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{})
-					logger.LogIf(ctx, err)
-				}
 				break
 			}
 			l.CleanupGCSMinioSysTmpBucket(ctx, attrs.Name)
@@ -461,7 +457,6 @@ func (l *gcsGateway) ListBuckets(ctx context.Context) (buckets []minio.BucketInf
 		}
 
 		if ierr != nil {
-			logger.LogIf(ctx, ierr)
 			return buckets, gcsToObjectError(ierr)
 		}
 

cmd/gateway/gcs/gateway-gcs_test.go

@@ -28,7 +28,7 @@ import (
 	"cloud.google.com/go/storage"
 	"google.golang.org/api/googleapi"
 
-	miniogo "github.com/minio/minio-go"
+	miniogo "github.com/minio/minio-go/v6"
 	minio "github.com/minio/minio/cmd"
 )
 

cmd/gateway/hdfs/gateway-hdfs-utils.go

@@ -19,7 +19,7 @@ package hdfs
 import (
 	"strings"
 
-	"github.com/minio/minio-go/pkg/s3utils"
+	"github.com/minio/minio-go/v6/pkg/s3utils"
 	minio "github.com/minio/minio/cmd"
 )
 

cmd/gateway/hdfs/gateway-hdfs.go

@@ -31,7 +31,7 @@ import (
 
 	"github.com/minio/cli"
 	"github.com/minio/hdfs/v3"
-	"github.com/minio/minio-go/pkg/s3utils"
+	"github.com/minio/minio-go/v6/pkg/s3utils"
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"

cmd/gateway/oss/gateway-oss.go

@@ -29,8 +29,8 @@ import (
 	humanize "github.com/dustin/go-humanize"
 
 	"github.com/minio/cli"
-	miniogopolicy "github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio-go/pkg/s3utils"
+	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
+	"github.com/minio/minio-go/v6/pkg/s3utils"
 	minio "github.com/minio/minio/cmd"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"

cmd/gateway/s3/gateway-s3-sse.go

@@ -26,7 +26,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/minio-go/pkg/encrypt"
+	"github.com/minio/minio-go/v6/pkg/encrypt"
 	minio "github.com/minio/minio/cmd"
 
 	"github.com/minio/minio/cmd/logger"

cmd/gateway/s3/gateway-s3.go

@@ -27,12 +27,12 @@ import (
 	"time"
 
 	"github.com/minio/cli"
-	miniogo "github.com/minio/minio-go"
-	"github.com/minio/minio-go/pkg/credentials"
+	miniogo "github.com/minio/minio-go/v6"
+	"github.com/minio/minio-go/v6/pkg/credentials"
 	minio "github.com/minio/minio/cmd"
 
-	"github.com/minio/minio-go/pkg/encrypt"
-	"github.com/minio/minio-go/pkg/s3utils"
+	"github.com/minio/minio-go/v6/pkg/encrypt"
+	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/policy"
@@ -169,14 +169,6 @@ func randString(n int, src rand.Source, prefix string) string {
 	return prefix + string(b[0:30-len(prefix)])
 }
 
-func isAmazonS3Endpoint(urlStr string) bool {
-	u, err := url.Parse(urlStr)
-	if err != nil {
-		panic(err)
-	}
-	return s3utils.IsAmazonEndpoint(*u)
-}
-
 // Chains all credential types, in the following order:
 //  - AWS env vars (i.e. AWS_ACCESS_KEY_ID)
 //  - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials)
@@ -210,6 +202,11 @@ func newS3(urlStr string) (*miniogo.Core, error) {
 		urlStr = "https://s3.amazonaws.com"
 	}
 
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		return nil, err
+	}
+
 	// Override default params if the host is provided
 	endpoint, secure, err := minio.ParseGatewayEndpoint(urlStr)
 	if err != nil {
@@ -217,15 +214,23 @@ func newS3(urlStr string) (*miniogo.Core, error) {
 	}
 
 	var creds *credentials.Credentials
-	if isAmazonS3Endpoint(urlStr) {
+	if s3utils.IsAmazonEndpoint(*u) {
 		// If we see an Amazon S3 endpoint, then we use more ways to fetch backend credentials.
 		// Specifically IAM style rotating credentials are only supported with AWS S3 endpoint.
 		creds = credentials.NewChainCredentials(defaultAWSCredProviders)
+
 	} else {
 		creds = credentials.NewChainCredentials(defaultProviders)
 	}
 
-	clnt, err := miniogo.NewWithCredentials(endpoint, creds, secure, "")
+	options := miniogo.Options{
+		Creds:        creds,
+		Secure:       secure,
+		Region:       s3utils.GetRegionFromURL(*u),
+		BucketLookup: miniogo.BucketLookupAuto,
+	}
+
+	clnt, err := miniogo.NewWithOptions(endpoint, &options)
 	if err != nil {
 		return nil, err
 	}
@@ -234,9 +239,12 @@ func newS3(urlStr string) (*miniogo.Core, error) {
 	clnt.SetCustomTransport(minio.NewCustomHTTPTransport())
 
 	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
+
 	// Check if the provided keys are valid.
 	if _, err = clnt.BucketExists(probeBucketName); err != nil {
-		return nil, err
+		if miniogo.ToErrorResponse(err).Code != "AccessDenied" {
+			return nil, err
+		}
 	}
 
 	return &miniogo.Core{Client: clnt}, nil
@@ -313,7 +321,19 @@ func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket, location
 func (l *s3Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, e error) {
 	buckets, err := l.Client.ListBuckets()
 	if err != nil {
-		return bi, minio.ErrorRespToObjectError(err, bucket)
+		// Listbuckets may be disallowed, proceed to check if
+		// bucket indeed exists, if yes return success.
+		var ok bool
+		if ok, err = l.Client.BucketExists(bucket); err != nil {
+			return bi, minio.ErrorRespToObjectError(err, bucket)
+		}
+		if !ok {
+			return bi, minio.BucketNotFound{Bucket: bucket}
+		}
+		return minio.BucketInfo{
+			Name:    bi.Name,
+			Created: time.Now().UTC(),
+		}, nil
 	}
 
 	for _, bi := range buckets {

cmd/gateway/s3/gateway-s3_test.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 	"testing"
 
-	miniogo "github.com/minio/minio-go"
+	miniogo "github.com/minio/minio-go/v6"
 	"github.com/minio/minio/pkg/hash"
 
 	minio "github.com/minio/minio/cmd"

cmd/generic-handlers.go

@@ -24,7 +24,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/crypto"
@@ -379,6 +379,20 @@ type resourceHandler struct {
 
 // setCorsHandler handler for CORS (Cross Origin Resource Sharing)
 func setCorsHandler(h http.Handler) http.Handler {
+	commonS3Headers := []string{
+		"Date",
+		"ETag",
+		"Server",
+		"Connection",
+		"Accept-Ranges",
+		"Content-Range",
+		"Content-Encoding",
+		"Content-Length",
+		"Content-Type",
+		"X-Amz*",
+		"x-amz*",
+		"*",
+	}
 
 	c := cors.New(cors.Options{
 		AllowedOrigins: []string{"*"},
@@ -391,8 +405,8 @@ func setCorsHandler(h http.Handler) http.Handler {
 			http.MethodOptions,
 			http.MethodPatch,
 		},
-		AllowedHeaders:   []string{"*"},
-		ExposedHeaders:   []string{"*"},
+		AllowedHeaders:   commonS3Headers,
+		ExposedHeaders:   commonS3Headers,
 		AllowCredentials: true,
 	})
 

cmd/globals.go

@@ -23,7 +23,7 @@ import (
 	"time"
 
 	isatty "github.com/mattn/go-isatty"
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 
 	etcd "github.com/coreos/etcd/clientv3"
 	humanize "github.com/dustin/go-humanize"
@@ -82,8 +82,6 @@ const (
 	// GlobalMultipartCleanupInterval - Cleanup interval when the stale multipart cleanup is initiated.
 	GlobalMultipartCleanupInterval = time.Hour * 24 // 24 hrs.
 
-	// Refresh interval to update in-memory bucket policy cache.
-	globalRefreshBucketPolicyInterval = 5 * time.Minute
 	// Refresh interval to update in-memory iam config cache.
 	globalRefreshIAMInterval = 5 * time.Minute
 
@@ -111,6 +109,9 @@ var (
 	// Indicates if the running minio server is an erasure-code backend.
 	globalIsXL = false
 
+	// Indicates if the running minio is in gateway mode.
+	globalIsGateway = false
+
 	// This flag is set to 'true' by default
 	globalIsBrowserEnabled = true
 
@@ -158,8 +159,9 @@ var (
 	globalHTTPServerErrorCh = make(chan error)
 	globalOSSignalCh        = make(chan os.Signal, 1)
 
-	// File to log HTTP request/response headers and body.
-	globalHTTPTraceFile *os.File
+	// global Trace system to send HTTP request/response logs to
+	// registered listeners
+	globalTrace *HTTPTraceSys
 
 	globalEndpoints EndpointList
 
@@ -259,6 +261,11 @@ var (
 	// GlobalGatewaySSE sse options
 	GlobalGatewaySSE gatewaySSE
 
+	// The always present healing routine ready to heal objects
+	globalBackgroundHealing *healRoutine
+	globalAllHealState      *allHealState
+	globalSweepHealState    *allHealState
+
 	// Add new variable global values here.
 )
 

cmd/handler-utils.go

@@ -30,7 +30,6 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/handlers"
-	httptracer "github.com/minio/minio/pkg/handlers"
 )
 
 // Parses location constraint from the incoming reader.
@@ -326,18 +325,26 @@ func extractPostPolicyFormValues(ctx context.Context, form *multipart.Form) (fil
 
 // Log headers and body.
 func httpTraceAll(f http.HandlerFunc) http.HandlerFunc {
-	if globalHTTPTraceFile == nil {
-		return f
+	return func(w http.ResponseWriter, r *http.Request) {
+		if !globalTrace.HasTraceListeners() {
+			f.ServeHTTP(w, r)
+			return
+		}
+		trace := Trace(f, true, w, r)
+		globalTrace.Publish(trace)
 	}
-	return httptracer.TraceReqHandlerFunc(f, globalHTTPTraceFile, true)
 }
 
 // Log only the headers.
 func httpTraceHdrs(f http.HandlerFunc) http.HandlerFunc {
-	if globalHTTPTraceFile == nil {
-		return f
+	return func(w http.ResponseWriter, r *http.Request) {
+		if !globalTrace.HasTraceListeners() {
+			f.ServeHTTP(w, r)
+			return
+		}
+		trace := Trace(f, false, w, r)
+		globalTrace.Publish(trace)
 	}
-	return httptracer.TraceReqHandlerFunc(f, globalHTTPTraceFile, false)
 }
 
 // Returns "/bucketName/objectName" for path-style or virtual-host-style requests.

cmd/http-tracer.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package handlers
+package cmd
 
 import (
 	"bytes"
@@ -24,8 +24,12 @@ import (
 	"net/http"
 	"reflect"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
+
+	xnet "github.com/minio/minio/pkg/net"
+	trace "github.com/minio/minio/pkg/trace"
 )
 
 // recordRequest - records the first recLen bytes
@@ -109,92 +113,64 @@ func (r *recordResponseWriter) Flush() {
 	r.ResponseWriter.(http.Flusher).Flush()
 }
 
-// Return response headers.
-func (r *recordResponseWriter) Headers() []byte {
-	return r.headers.Bytes()
-}
-
 // Return response body.
 func (r *recordResponseWriter) Body() []byte {
 	return r.body.Bytes()
 }
 
-// TraceReqHandlerFunc logs request/response headers and body.
-func TraceReqHandlerFunc(f http.HandlerFunc, output io.Writer, logBody bool) http.HandlerFunc {
+// Trace gets trace of http request
+func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Request) trace.Info {
+
 	name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
 	name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.")
+	name = strings.TrimSuffix(name, "Handler-fm")
+
 	bodyPlaceHolder := []byte("<BODY>")
+	var reqBodyRecorder *recordRequest
 
-	return func(w http.ResponseWriter, r *http.Request) {
-		const timeFormat = "2006-01-02 15:04:05 -0700"
-		var reqBodyRecorder *recordRequest
+	t := trace.Info{FuncName: name}
+	reqBodyRecorder = &recordRequest{Reader: r.Body, logBody: logBody}
+	r.Body = ioutil.NopCloser(reqBodyRecorder)
 
-		// Generate short random request ID
-		reqID := fmt.Sprintf("%f", float64(time.Now().UnixNano())/1e10)
-
-		reqBodyRecorder = &recordRequest{Reader: r.Body, logBody: logBody}
-		r.Body = ioutil.NopCloser(reqBodyRecorder)
-
-		// Setup a http response body recorder
-		respBodyRecorder := &recordResponseWriter{ResponseWriter: w, logBody: logBody}
-
-		b := bytes.NewBuffer(nil)
-		fmt.Fprintf(b, "[REQUEST %s] [%s] [%s]\n", name, reqID, time.Now().Format(timeFormat))
-
-		f(respBodyRecorder, r)
-
-		// Build request log and write it to log file
-		fmt.Fprintf(b, "%s %s", r.Method, r.URL.Path)
-		if r.URL.RawQuery != "" {
-			fmt.Fprintf(b, "?%s", r.URL.RawQuery)
-		}
-		fmt.Fprintf(b, "\n")
-
-		fmt.Fprintf(b, "Host: %s\n", r.Host)
-		for k, v := range r.Header {
-			fmt.Fprintf(b, "%s: %s\n", k, v[0])
-		}
-		fmt.Fprintf(b, "\n")
-		if logBody {
-			bodyContents := reqBodyRecorder.Data()
-			if bodyContents != nil {
-				// If body logging is disabled then we print <BODY> as a placeholder
-				// for the actual body.
-				b.Write(bodyContents)
-				fmt.Fprintf(b, "\n")
-			}
-		} else {
-			b.Write(bodyPlaceHolder)
-			fmt.Fprintf(b, "\n")
-		}
-
-		fmt.Fprintf(b, "\n")
-
-		// Build response log and write it to log file
-		fmt.Fprintf(b, "[RESPONSE] [%s] [%s]\n", reqID, time.Now().Format(timeFormat))
-
-		b.Write(respBodyRecorder.Headers())
-		fmt.Fprintf(b, "\n")
-
-		// recordResponseWriter{} is configured to record only
-		// responses with http code != 200 &  != 206, we don't
-		// have to check for logBody value here.
-		bodyContents := respBodyRecorder.Body()
-		if bodyContents != nil {
-			b.Write(bodyContents)
-			fmt.Fprintf(b, "\n")
-		} else {
-			if !logBody {
-				// If there was no error response and body logging is disabled
-				// then we print <BODY> as a placeholder for the actual body.
-				b.Write(bodyPlaceHolder)
-				fmt.Fprintf(b, "\n")
-			}
-		}
-
-		fmt.Fprintf(b, "\n")
-
-		// Write the contents in one shot so that logs don't get interspersed.
-		output.Write(b.Bytes())
+	host, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	if err == nil {
+		t.NodeName = host.Name
 	}
+	rq := trace.RequestInfo{Time: time.Now().UTC(), Method: r.Method, Path: r.URL.Path, RawQuery: r.URL.RawQuery}
+	rq.Headers = cloneHeader(r.Header)
+	rq.Headers.Set("Content-Length", strconv.Itoa(int(r.ContentLength)))
+	rq.Headers.Set("Host", r.Host)
+	for _, enc := range r.TransferEncoding {
+		rq.Headers.Add("Transfer-Encoding", enc)
+	}
+	if logBody {
+		// If body logging is disabled then we print <BODY> as a placeholder
+		// for the actual body.
+		rq.Body = reqBodyRecorder.Data()
+
+	} else {
+		rq.Body = bodyPlaceHolder
+	}
+	// Setup a http response body recorder
+	respBodyRecorder := &recordResponseWriter{ResponseWriter: w, logBody: logBody}
+	f(respBodyRecorder, r)
+
+	rs := trace.ResponseInfo{Time: time.Now().UTC()}
+	rs.Headers = cloneHeader(respBodyRecorder.Header())
+	rs.StatusCode = respBodyRecorder.statusCode
+	if rs.StatusCode == 0 {
+		rs.StatusCode = http.StatusOK
+	}
+	bodyContents := respBodyRecorder.Body()
+	if bodyContents != nil {
+		rs.Body = bodyContents
+	}
+	if !logBody {
+		// If there was no error response and body logging is disabled
+		// then we print <BODY> as a placeholder for the actual body.
+		rs.Body = bodyPlaceHolder
+	}
+	t.ReqInfo = rq
+	t.RespInfo = rs
+	return t
 }

cmd/http/listener_test.go

@@ -28,7 +28,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 )
 
 var serverPort uint32 = 60000

cmd/http/server.go

@@ -26,9 +26,8 @@ import (
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
-	"golang.org/x/net/http2"
 
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/pkg/certs"
 )
 
@@ -56,9 +55,6 @@ const (
 
 	// DefaultMaxHeaderBytes - default maximum HTTP header size in bytes.
 	DefaultMaxHeaderBytes = 1 * humanize.MiByte
-
-	// DefaultHTTP2MaxConcurrentStreams - default value for HTTP 2.0 maximum concurrent streams allowed.
-	DefaultHTTP2MaxConcurrentStreams = 1024
 )
 
 // Server - extended http.Server supports multiple addresses to serve and enhanced connection handling.
@@ -133,11 +129,6 @@ func (srv *Server) Start() (err error) {
 
 	// Start servicing with listener.
 	if tlsConfig != nil {
-		if err = http2.ConfigureServer(&srv.Server, &http2.Server{
-			MaxConcurrentStreams: DefaultHTTP2MaxConcurrentStreams,
-		}); err != nil {
-			return err
-		}
 		return srv.Server.Serve(tls.NewListener(listener, tlsConfig))
 	}
 	return srv.Server.Serve(listener)
@@ -211,7 +202,14 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
 			CipherSuites:             defaultCipherSuites,
 			CurvePreferences:         secureCurves,
 			MinVersion:               tls.VersionTLS12,
-			NextProtos:               []string{"h2", "http/1.1"},
+			// Do not edit the next line, protos priority is kept
+			// on purpose in this manner for HTTP 2.0, we would
+			// still like HTTP 2.0 clients to negotiate connection
+			// to server if needed but by default HTTP 1.1 is
+			// expected. We need to change this in future
+			// when we wish to go back to HTTP 2.0 as default
+			// priority for HTTP protocol negotiation.
+			NextProtos: []string{"http/1.1", "h2"},
 		}
 		tlsConfig.GetCertificate = getCert
 	}

cmd/httptrace.go

@@ -0,0 +1,115 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"strings"
+	"sync"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/pubsub"
+	"github.com/minio/minio/pkg/trace"
+)
+
+//HTTPTraceSys holds global trace state
+type HTTPTraceSys struct {
+	peers  []*peerRESTClient
+	pubsub *pubsub.PubSub
+}
+
+// NewTraceSys - creates new HTTPTraceSys with all nodes subscribed to
+// the trace pub sub system
+func NewTraceSys(ctx context.Context, endpoints EndpointList) *HTTPTraceSys {
+	remoteHosts := getRemoteHosts(endpoints)
+	remoteClients, err := getRestClients(remoteHosts)
+	if err != nil {
+		logger.FatalIf(err, "Unable to start httptrace sub system")
+	}
+
+	ps := pubsub.New()
+	return &HTTPTraceSys{
+		remoteClients, ps,
+	}
+}
+
+// HasTraceListeners returns true if trace listeners are registered
+// for this node or peers
+func (sys *HTTPTraceSys) HasTraceListeners() bool {
+	return sys != nil && sys.pubsub.HasSubscribers()
+}
+
+// Publish - publishes trace message to the http trace pubsub system
+func (sys *HTTPTraceSys) Publish(traceMsg trace.Info) {
+	sys.pubsub.Publish(traceMsg)
+}
+
+// Trace writes http trace to writer
+func (sys *HTTPTraceSys) Trace(doneCh chan struct{}, trcAll bool) chan []byte {
+	traceCh := make(chan []byte)
+	go func() {
+		defer close(traceCh)
+
+		var wg = &sync.WaitGroup{}
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			buf := &bytes.Buffer{}
+			ch := sys.pubsub.Subscribe()
+			defer sys.pubsub.Unsubscribe(ch)
+			for {
+				select {
+				case entry := <-ch:
+					trcInfo := entry.(trace.Info)
+					path := strings.TrimPrefix(trcInfo.ReqInfo.Path, "/")
+					// omit inter-node traffic if trcAll is false
+					if !trcAll && strings.HasPrefix(path, minioReservedBucket) {
+						continue
+					}
+					buf.Reset()
+					enc := json.NewEncoder(buf)
+					enc.SetEscapeHTML(false)
+					if err := enc.Encode(trcInfo); err != nil {
+						continue
+					}
+					traceCh <- buf.Bytes()
+				case <-doneCh:
+					return
+				}
+			}
+		}()
+
+		for _, peer := range sys.peers {
+			wg.Add(1)
+			go func(peer *peerRESTClient) {
+				defer wg.Done()
+				ch, err := peer.Trace(doneCh, trcAll)
+				if err != nil {
+					return
+				}
+				for entry := range ch {
+					traceCh <- entry
+				}
+			}(peer)
+		}
+		wg.Wait()
+	}()
+	return traceCh
+}

cmd/iam.go

@@ -25,7 +25,8 @@ import (
 	"time"
 
 	etcd "github.com/coreos/etcd/clientv3"
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/coreos/etcd/mvcc/mvccpb"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
@@ -60,6 +61,45 @@ type IAMSys struct {
 	iamCannedPolicyMap map[string]iampolicy.Policy
 }
 
+// LoadPolicy - reloads a specific canned policy from backend disks or etcd.
+func (sys *IAMSys) LoadPolicy(objAPI ObjectLayer, policyName string) error {
+	if objAPI == nil {
+		return errInvalidArgument
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	prefix := iamConfigPoliciesPrefix
+	if globalEtcdClient == nil {
+		return reloadPolicy(context.Background(), objAPI, prefix, policyName, sys.iamCannedPolicyMap)
+	}
+
+	// When etcd is set, we use watch APIs so this code is not needed.
+	return nil
+}
+
+// LoadUser - reloads a specific user from backend disks or etcd.
+func (sys *IAMSys) LoadUser(objAPI ObjectLayer, accessKey string, temp bool) error {
+	if objAPI == nil {
+		return errInvalidArgument
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	prefix := iamConfigUsersPrefix
+	if temp {
+		prefix = iamConfigSTSPrefix
+	}
+
+	if globalEtcdClient == nil {
+		return reloadUser(context.Background(), objAPI, prefix, accessKey, sys.iamUsersMap, sys.iamPolicyMap)
+	}
+	// When etcd is set, we use watch APIs so this code is not needed.
+	return nil
+}
+
 // Load - loads iam subsystem
 func (sys *IAMSys) Load(objAPI ObjectLayer) error {
 	if globalEtcdClient != nil {
@@ -68,6 +108,105 @@ func (sys *IAMSys) Load(objAPI ObjectLayer) error {
 	return sys.refresh(objAPI)
 }
 
+func (sys *IAMSys) reloadFromEvent(event *etcd.Event) {
+	eventCreate := event.IsModify() || event.IsCreate()
+	eventDelete := event.Type == etcd.EventTypeDelete
+	usersPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigUsersPrefix)
+	stsPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigSTSPrefix)
+	policyPrefix := strings.HasPrefix(string(event.Kv.Key), iamConfigPoliciesPrefix)
+
+	ctx, cancel := context.WithTimeout(context.Background(),
+		defaultContextTimeout)
+	defer cancel()
+
+	switch {
+	case eventCreate:
+		switch {
+		case usersPrefix:
+			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
+				iamConfigUsersPrefix))
+			reloadEtcdUser(ctx, iamConfigUsersPrefix, accessKey,
+				sys.iamUsersMap, sys.iamPolicyMap)
+		case stsPrefix:
+			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
+				iamConfigSTSPrefix))
+			reloadEtcdUser(ctx, iamConfigSTSPrefix, accessKey,
+				sys.iamUsersMap, sys.iamPolicyMap)
+		case policyPrefix:
+			policyName := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
+				iamConfigPoliciesPrefix))
+			reloadEtcdPolicy(ctx, iamConfigPoliciesPrefix,
+				policyName, sys.iamCannedPolicyMap)
+		}
+	case eventDelete:
+		switch {
+		case usersPrefix:
+			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
+				iamConfigUsersPrefix))
+			delete(sys.iamUsersMap, accessKey)
+			delete(sys.iamPolicyMap, accessKey)
+		case stsPrefix:
+			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
+				iamConfigSTSPrefix))
+			delete(sys.iamUsersMap, accessKey)
+			delete(sys.iamPolicyMap, accessKey)
+		case policyPrefix:
+			policyName := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
+				iamConfigPoliciesPrefix))
+			delete(sys.iamCannedPolicyMap, policyName)
+		}
+	}
+}
+
+// Watch etcd entries for IAM
+func (sys *IAMSys) watchIAMEtcd() {
+	watchEtcd := func() {
+		// Refresh IAMSys with etcd watch.
+		for {
+			watchCh := globalEtcdClient.Watch(context.Background(),
+				iamConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
+			select {
+			case <-GlobalServiceDoneCh:
+				return
+			case watchResp, ok := <-watchCh:
+				if !ok {
+					time.Sleep(1 * time.Second)
+					continue
+				}
+				if err := watchResp.Err(); err != nil {
+					logger.LogIf(context.Background(), err)
+					// log and retry.
+					time.Sleep(1 * time.Second)
+					continue
+				}
+				for _, event := range watchResp.Events {
+					sys.Lock()
+					sys.reloadFromEvent(event)
+					sys.Unlock()
+				}
+			}
+		}
+	}
+	go watchEtcd()
+}
+
+func (sys *IAMSys) watchIAMDisk(objAPI ObjectLayer) {
+	watchDisk := func() {
+		ticker := time.NewTicker(globalRefreshIAMInterval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-GlobalServiceDoneCh:
+				return
+			case <-ticker.C:
+				sys.refresh(objAPI)
+			}
+		}
+	}
+	// Refresh IAMSys in background.
+	go watchDisk()
+}
+
 // Init - initializes config system from iam.json
 func (sys *IAMSys) Init(objAPI ObjectLayer) error {
 	if objAPI == nil {
@@ -75,50 +214,9 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
 	}
 
 	if globalEtcdClient != nil {
-		defer func() {
-			go func() {
-				// Refresh IAMSys with etcd watch.
-				for {
-					watchCh := globalEtcdClient.Watch(context.Background(), iamConfigPrefix)
-					select {
-					case <-GlobalServiceDoneCh:
-						return
-					case watchResp, ok := <-watchCh:
-						if !ok {
-							time.Sleep(1 * time.Second)
-							continue
-						}
-						if err := watchResp.Err(); err != nil {
-							logger.LogIf(context.Background(), err)
-							// log and retry.
-							time.Sleep(1 * time.Second)
-							continue
-						}
-						for _, event := range watchResp.Events {
-							if event.IsModify() || event.IsCreate() || event.Type == etcd.EventTypeDelete {
-								sys.refreshEtcd()
-							}
-						}
-					}
-				}
-			}()
-		}()
+		defer sys.watchIAMEtcd()
 	} else {
-		defer func() {
-			// Refresh IAMSys in background.
-			go func() {
-				ticker := time.NewTicker(globalRefreshIAMInterval)
-				defer ticker.Stop()
-				for {
-					select {
-					case <-GlobalServiceDoneCh:
-						return
-					case <-ticker.C:
-						sys.refresh(objAPI)
-					}
-				}
-			}()
-		}()
+		defer sys.watchIAMDisk(objAPI)
 	}
 
 	doneCh := make(chan struct{})
@@ -147,8 +245,8 @@ func (sys *IAMSys) Init(objAPI ObjectLayer) error {
 	return nil
 }
 
-// DeleteCannedPolicy - deletes a canned policy.
-func (sys *IAMSys) DeleteCannedPolicy(policyName string) error {
+// DeletePolicy - deletes a canned policy from backend or etcd.
+func (sys *IAMSys) DeletePolicy(policyName string) error {
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
@@ -159,11 +257,17 @@ func (sys *IAMSys) DeleteCannedPolicy(policyName string) error {
 	}
 
 	var err error
-	configFile := pathJoin(iamConfigPoliciesPrefix, policyName, iamPolicyFile)
+	pFile := pathJoin(iamConfigPoliciesPrefix, policyName, iamPolicyFile)
 	if globalEtcdClient != nil {
-		err = deleteConfigEtcd(context.Background(), globalEtcdClient, configFile)
+		err = deleteConfigEtcd(context.Background(), globalEtcdClient, pFile)
 	} else {
-		err = deleteConfig(context.Background(), objectAPI, configFile)
+		err = deleteConfig(context.Background(), objectAPI, pFile)
+	}
+
+	switch err.(type) {
+	case ObjectNotFound:
+		// Ignore error if policy is already deleted.
+		err = nil
 	}
 
 	sys.Lock()
@@ -173,8 +277,8 @@ func (sys *IAMSys) DeleteCannedPolicy(policyName string) error {
 	return err
 }
 
-// ListCannedPolicies - lists all canned policies.
-func (sys *IAMSys) ListCannedPolicies() (map[string][]byte, error) {
+// ListPolicies - lists all canned policies.
+func (sys *IAMSys) ListPolicies() (map[string][]byte, error) {
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return nil, errServerNotInitialized
@@ -196,8 +300,8 @@ func (sys *IAMSys) ListCannedPolicies() (map[string][]byte, error) {
 	return cannedPolicyMap, nil
 }
 
-// SetCannedPolicy - sets a new canned policy.
-func (sys *IAMSys) SetCannedPolicy(policyName string, p iampolicy.Policy) error {
+// SetPolicy - sets a new canned policy.
+func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error {
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
@@ -278,11 +382,11 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 	pFile := pathJoin(iamConfigUsersPrefix, accessKey, iamPolicyFile)
 	iFile := pathJoin(iamConfigUsersPrefix, accessKey, iamIdentityFile)
 	if globalEtcdClient != nil {
-		// It is okay to ingnore errors when deleting policy.json for the user.
-		_ = deleteConfigEtcd(context.Background(), globalEtcdClient, pFile)
+		// It is okay to ignore errors when deleting policy.json for the user.
+		deleteConfigEtcd(context.Background(), globalEtcdClient, pFile)
 		err = deleteConfigEtcd(context.Background(), globalEtcdClient, iFile)
 	} else {
-		// It is okay to ingnore errors when deleting policy.json for the user.
+		// It is okay to ignore errors when deleting policy.json for the user.
 		_ = deleteConfig(context.Background(), objectAPI, pFile)
 		err = deleteConfig(context.Background(), objectAPI, iFile)
 	}
@@ -290,7 +394,8 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 	//
 	switch err.(type) {
 	case ObjectNotFound:
-		err = errNoSuchUser
+		// ignore if user is already deleted.
+		err = nil
 	}
 
 	sys.Lock()
@@ -490,6 +595,51 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 	return nil
 }
 
+// SetUserSecretKey - sets user secret key
+func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	sys.Lock()
+	defer sys.Unlock()
+
+	cred, ok := sys.iamUsersMap[accessKey]
+	if !ok {
+		return errNoSuchUser
+	}
+
+	uinfo := madmin.UserInfo{
+		SecretKey: secretKey,
+		Status:    madmin.AccountStatus(cred.Status),
+	}
+
+	configFile := pathJoin(iamConfigUsersPrefix, accessKey, iamIdentityFile)
+	data, err := json.Marshal(uinfo)
+	if err != nil {
+		return err
+	}
+
+	if globalEtcdClient != nil {
+		err = saveConfigEtcd(context.Background(), globalEtcdClient, configFile, data)
+	} else {
+		err = saveConfig(context.Background(), objectAPI, configFile, data)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	sys.iamUsersMap[accessKey] = auth.Credentials{
+		AccessKey: accessKey,
+		SecretKey: secretKey,
+		Status:    string(uinfo.Status),
+	}
+
+	return nil
+}
+
 // GetUser - get user credentials
 func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 	sys.RLock()
@@ -521,21 +671,9 @@ func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
 
 var defaultContextTimeout = 30 * time.Second
 
-// Similar to reloadUsers but updates users, policies maps from etcd server,
-func reloadEtcdUsers(prefix string, usersMap map[string]auth.Credentials, policyMap map[string]string) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
-	defer cancel()
-	r, err := globalEtcdClient.Get(ctx, prefix, etcd.WithPrefix(), etcd.WithKeysOnly())
-	if err != nil {
-		return err
-	}
-	// No users are created yet.
-	if r.Count == 0 {
-		return nil
-	}
-
+func etcdKvsToSet(prefix string, kvs []*mvccpb.KeyValue) set.StringSet {
 	users := set.NewStringSet()
-	for _, kv := range r.Kvs {
+	for _, kv := range kvs {
 		// Extract user by stripping off the `prefix` value as suffix,
 		// then strip off the remaining basename to obtain the prefix
 		// value, usually in the following form.
@@ -549,46 +687,48 @@ func reloadEtcdUsers(prefix string, usersMap map[string]auth.Credentials, policy
 			users.Add(user)
 		}
 	}
+	return users
+}
+
+// Similar to reloadUsers but updates users, policies maps from etcd server,
+func reloadEtcdUsers(prefix string, usersMap map[string]auth.Credentials, policyMap map[string]string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+	defer cancel()
+	r, err := globalEtcdClient.Get(ctx, prefix, etcd.WithPrefix(), etcd.WithKeysOnly())
+	if err != nil {
+		return err
+	}
+	// No users are created yet.
+	if r.Count == 0 {
+		return nil
+	}
+
+	users := etcdKvsToSet(prefix, r.Kvs)
 
 	// Reload config and policies for all users.
 	for _, user := range users.ToSlice() {
-		idFile := pathJoin(prefix, user, iamIdentityFile)
-		pFile := pathJoin(prefix, user, iamPolicyFile)
-		cdata, cerr := readConfigEtcd(ctx, globalEtcdClient, idFile)
-		pdata, perr := readConfigEtcd(ctx, globalEtcdClient, pFile)
-		if cerr != nil && cerr != errConfigNotFound {
-			return cerr
-		}
-		if perr != nil && perr != errConfigNotFound {
-			return perr
-		}
-		if cerr == errConfigNotFound && perr == errConfigNotFound {
-			continue
-		}
-		if cerr == nil {
-			var cred auth.Credentials
-			if err = json.Unmarshal(cdata, &cred); err != nil {
-				return err
-			}
-			cred.AccessKey = user
-			if cred.IsExpired() {
-				deleteConfigEtcd(ctx, globalEtcdClient, idFile)
-				deleteConfigEtcd(ctx, globalEtcdClient, pFile)
-				continue
-			}
-			usersMap[cred.AccessKey] = cred
-		}
-		if perr == nil {
-			var policyName string
-			if err = json.Unmarshal(pdata, &policyName); err != nil {
-				return err
-			}
-			policyMap[user] = policyName
+		if err = reloadEtcdUser(ctx, prefix, user, usersMap, policyMap); err != nil {
+			return err
 		}
 	}
 	return nil
 }
 
+func reloadEtcdPolicy(ctx context.Context, prefix string, policyName string,
+	cannedPolicyMap map[string]iampolicy.Policy) error {
+	pFile := pathJoin(prefix, policyName, iamPolicyFile)
+	pdata, err := readConfigEtcd(ctx, globalEtcdClient, pFile)
+	if err != nil {
+		return err
+	}
+	var p iampolicy.Policy
+	if err = json.Unmarshal(pdata, &p); err != nil {
+		return err
+	}
+	cannedPolicyMap[policyName] = p
+	return nil
+}
+
 func reloadEtcdPolicies(prefix string, cannedPolicyMap map[string]iampolicy.Policy) error {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
 	defer cancel()
@@ -601,38 +741,32 @@ func reloadEtcdPolicies(prefix string, cannedPolicyMap map[string]iampolicy.Poli
 		return nil
 	}
 
-	policies := set.NewStringSet()
-	for _, kv := range r.Kvs {
-		// Extract policy by stripping off the `prefix` value as suffix,
-		// then strip off the remaining basename to obtain the prefix
-		// value, usually in the following form.
-		//
-		//  key := "config/iam/policies/newpolicy/identity.json"
-		//  prefix := "config/iam/policies/"
-		//  v := trim(trim(key, prefix), base(key)) == "newpolicy"
-		//
-		policyName := path.Clean(strings.TrimSuffix(strings.TrimPrefix(string(kv.Key), prefix), path.Base(string(kv.Key))))
-		if !policies.Contains(policyName) {
-			policies.Add(policyName)
-		}
-	}
+	policies := etcdKvsToSet(prefix, r.Kvs)
 
 	// Reload config and policies for all policys.
 	for _, policyName := range policies.ToSlice() {
-		pFile := pathJoin(prefix, policyName, iamPolicyFile)
-		pdata, perr := readConfigEtcd(ctx, globalEtcdClient, pFile)
-		if perr != nil {
-			return perr
-		}
-		var p iampolicy.Policy
-		if err = json.Unmarshal(pdata, &p); err != nil {
+		if err = reloadEtcdPolicy(ctx, prefix, policyName, cannedPolicyMap); err != nil {
 			return err
 		}
-		cannedPolicyMap[policyName] = p
 	}
 	return nil
 }
 
+func reloadPolicy(ctx context.Context, objectAPI ObjectLayer, prefix string,
+	policyName string, cannedPolicyMap map[string]iampolicy.Policy) error {
+	pFile := pathJoin(prefix, policyName, iamPolicyFile)
+	pdata, err := readConfig(context.Background(), objectAPI, pFile)
+	if err != nil {
+		return err
+	}
+	var p iampolicy.Policy
+	if err = json.Unmarshal(pdata, &p); err != nil {
+		return err
+	}
+	cannedPolicyMap[policyName] = p
+	return nil
+}
+
 func reloadPolicies(objectAPI ObjectLayer, prefix string, cannedPolicyMap map[string]iampolicy.Policy) error {
 	marker := ""
 	for {
@@ -644,16 +778,10 @@ func reloadPolicies(objectAPI ObjectLayer, prefix string, cannedPolicyMap map[st
 		}
 		marker = lo.NextMarker
 		for _, prefix := range lo.Prefixes {
-			pFile := pathJoin(prefix, iamPolicyFile)
-			pdata, perr := readConfig(context.Background(), objectAPI, pFile)
-			if perr != nil {
-				return perr
-			}
-			var p iampolicy.Policy
-			if err = json.Unmarshal(pdata, &p); err != nil {
+			if err = reloadPolicy(context.Background(), objectAPI, iamConfigPoliciesPrefix,
+				path.Base(prefix), cannedPolicyMap); err != nil {
 				return err
 			}
-			cannedPolicyMap[path.Base(prefix)] = p
 		}
 		if !lo.IsTruncated {
 			break
@@ -663,6 +791,86 @@ func reloadPolicies(objectAPI ObjectLayer, prefix string, cannedPolicyMap map[st
 
 }
 
+func reloadEtcdUser(ctx context.Context, prefix string, accessKey string,
+	usersMap map[string]auth.Credentials, policyMap map[string]string) error {
+	idFile := pathJoin(prefix, accessKey, iamIdentityFile)
+	pFile := pathJoin(prefix, accessKey, iamPolicyFile)
+	cdata, cerr := readConfigEtcd(ctx, globalEtcdClient, idFile)
+	pdata, perr := readConfigEtcd(ctx, globalEtcdClient, pFile)
+	if cerr != nil && cerr != errConfigNotFound {
+		return cerr
+	}
+	if perr != nil && perr != errConfigNotFound {
+		return perr
+	}
+	if cerr == errConfigNotFound && perr == errConfigNotFound {
+		return nil
+	}
+	if cerr == nil {
+		var cred auth.Credentials
+		if err := json.Unmarshal(cdata, &cred); err != nil {
+			return err
+		}
+		cred.AccessKey = path.Base(accessKey)
+		if cred.IsExpired() {
+			// Delete expired identity.
+			deleteConfigEtcd(ctx, globalEtcdClient, idFile)
+			// Delete expired identity policy.
+			deleteConfigEtcd(ctx, globalEtcdClient, pFile)
+			return nil
+		}
+		usersMap[cred.AccessKey] = cred
+	}
+	if perr == nil {
+		var policyName string
+		if err := json.Unmarshal(pdata, &policyName); err != nil {
+			return err
+		}
+		policyMap[path.Base(accessKey)] = policyName
+	}
+	return nil
+}
+
+func reloadUser(ctx context.Context, objectAPI ObjectLayer, prefix string, accessKey string,
+	usersMap map[string]auth.Credentials, policyMap map[string]string) error {
+	idFile := pathJoin(prefix, accessKey, iamIdentityFile)
+	pFile := pathJoin(prefix, accessKey, iamPolicyFile)
+	cdata, cerr := readConfig(ctx, objectAPI, idFile)
+	pdata, perr := readConfig(ctx, objectAPI, pFile)
+	if cerr != nil && cerr != errConfigNotFound {
+		return cerr
+	}
+	if perr != nil && perr != errConfigNotFound {
+		return perr
+	}
+	if cerr == errConfigNotFound && perr == errConfigNotFound {
+		return nil
+	}
+	if cerr == nil {
+		var cred auth.Credentials
+		if err := json.Unmarshal(cdata, &cred); err != nil {
+			return err
+		}
+		cred.AccessKey = path.Base(accessKey)
+		if cred.IsExpired() {
+			// Delete expired identity.
+			objectAPI.DeleteObject(context.Background(), minioMetaBucket, idFile)
+			// Delete expired identity policy.
+			objectAPI.DeleteObject(context.Background(), minioMetaBucket, pFile)
+			return nil
+		}
+		usersMap[cred.AccessKey] = cred
+	}
+	if perr == nil {
+		var policyName string
+		if err := json.Unmarshal(pdata, &policyName); err != nil {
+			return err
+		}
+		policyMap[path.Base(accessKey)] = policyName
+	}
+	return nil
+}
+
 // reloadUsers reads an updates users, policies from object layer into user and policy maps.
 func reloadUsers(objectAPI ObjectLayer, prefix string, usersMap map[string]auth.Credentials, policyMap map[string]string) error {
 	marker := ""
@@ -675,40 +883,9 @@ func reloadUsers(objectAPI ObjectLayer, prefix string, usersMap map[string]auth.
 		}
 		marker = lo.NextMarker
 		for _, prefix := range lo.Prefixes {
-			idFile := pathJoin(prefix, iamIdentityFile)
-			pFile := pathJoin(prefix, iamPolicyFile)
-			cdata, cerr := readConfig(context.Background(), objectAPI, idFile)
-			pdata, perr := readConfig(context.Background(), objectAPI, pFile)
-			if cerr != nil && cerr != errConfigNotFound {
-				return cerr
-			}
-			if perr != nil && perr != errConfigNotFound {
-				return perr
-			}
-			if cerr == errConfigNotFound && perr == errConfigNotFound {
-				continue
-			}
-			if cerr == nil {
-				var cred auth.Credentials
-				if err = json.Unmarshal(cdata, &cred); err != nil {
-					return err
-				}
-				cred.AccessKey = path.Base(prefix)
-				if cred.IsExpired() {
-					// Delete expired identity.
-					objectAPI.DeleteObject(context.Background(), minioMetaBucket, idFile)
-					// Delete expired identity policy.
-					objectAPI.DeleteObject(context.Background(), minioMetaBucket, pFile)
-					continue
-				}
-				usersMap[cred.AccessKey] = cred
-			}
-			if perr == nil {
-				var policyName string
-				if err = json.Unmarshal(pdata, &policyName); err != nil {
-					return err
-				}
-				policyMap[path.Base(prefix)] = policyName
+			// Prefix is empty because prefix is already part of the List output.
+			if err = reloadUser(context.Background(), objectAPI, "", prefix, usersMap, policyMap); err != nil {
+				return err
 			}
 		}
 		if !lo.IsTruncated {

cmd/jwt.go

@@ -47,6 +47,7 @@ var (
 	errChangeCredNotAllowed = errors.New("Changing access key and secret key not allowed")
 	errAuthentication       = errors.New("Authentication failed, check your access credentials")
 	errNoAuthToken          = errors.New("JWT token missing")
+	errIncorrectCreds       = errors.New("Current access key or secret key is incorrect")
 )
 
 func authenticateJWTUsers(accessKey, secretKey string, expiry time.Duration) (string, error) {

cmd/logger/logger.go

@@ -29,7 +29,7 @@ import (
 	"time"
 
 	"github.com/minio/highwayhash"
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/logger/message/log"
 )
 

cmd/logger/target/console/console.go

@@ -74,6 +74,11 @@ func (c *Target) Send(e interface{}) error {
 	apiString += ")"
 	timeString := "Time: " + time.Now().Format(logger.TimeFormat)
 
+	var deploymentID string
+	if entry.DeploymentID != "" {
+		deploymentID = "\nDeploymentID: " + entry.DeploymentID
+	}
+
 	var requestID string
 	if entry.RequestID != "" {
 		requestID = "\nRequestID: " + entry.RequestID
@@ -94,8 +99,8 @@ func (c *Target) Send(e interface{}) error {
 	}
 
 	var msg = logger.ColorFgRed(logger.ColorBold(entry.Trace.Message))
-	var output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
-		apiString, timeString, requestID, remoteHost, userAgent,
+	var output = fmt.Sprintf("\n%s\n%s%s%s%s%s\nError: %s%s\n%s",
+		apiString, timeString, deploymentID, requestID, remoteHost, userAgent,
 		msg, tagString, strings.Join(trace, "\n"))
 
 	fmt.Println(output)

cmd/main.go

@@ -27,8 +27,8 @@ import (
 	"github.com/minio/minio/pkg/words"
 )
 
-// global flags for minio.
-var globalFlags = []cli.Flag{
+// GlobalFlags - global flags for minio.
+var GlobalFlags = []cli.Flag{
 	cli.StringFlag{
 		Name:  "config-dir, C",
 		Value: defaultConfigDir.Get(),
@@ -131,7 +131,7 @@ func newApp(name string) *cli.App {
 	app.Version = Version
 	app.Usage = "Cloud Storage Server."
 	app.Description = `MinIO is an Amazon S3 compatible object storage server. Use it to store photos, videos, VMs, containers, log files, or any blob of data as objects.`
-	app.Flags = globalFlags
+	app.Flags = GlobalFlags
 	app.HideVersion = true     // Hide `--version` flag, we already have `minio version`.
 	app.HideHelpCommand = true // Hide `help, h` command, we already have `minio --help`.
 	app.Commands = commands

cmd/namespace-lock.go

@@ -29,7 +29,7 @@ import (
 
 	"github.com/minio/dsync"
 	"github.com/minio/lsync"
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 	xnet "github.com/minio/minio/pkg/net"
 )

cmd/net.go

@@ -27,7 +27,7 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 )
 

cmd/net_test.go

@@ -24,7 +24,7 @@ import (
 	"runtime"
 	"testing"
 
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/set"
 )
 
 func TestMustSplitHostPort(t *testing.T) {

cmd/notification.go

@@ -31,6 +31,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
 	xnet "github.com/minio/minio/pkg/net"
@@ -156,6 +157,66 @@ func (sys *NotificationSys) ReloadFormat(dryRun bool) []NotificationPeerErr {
 	return ng.Wait()
 }
 
+// DeletePolicy - deletes policy across all peers.
+func (sys *NotificationSys) DeletePolicy(policyName string) []NotificationPeerErr {
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		client := client
+		ng.Go(context.Background(), func() error {
+			return client.DeletePolicy(policyName)
+		}, idx, *client.host)
+	}
+	return ng.Wait()
+}
+
+// LoadPolicy - reloads a specific modified policy across all peers
+func (sys *NotificationSys) LoadPolicy(policyName string) []NotificationPeerErr {
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		client := client
+		ng.Go(context.Background(), func() error {
+			return client.LoadPolicy(policyName)
+		}, idx, *client.host)
+	}
+	return ng.Wait()
+}
+
+// DeleteUser - deletes a specific user across all peers
+func (sys *NotificationSys) DeleteUser(accessKey string) []NotificationPeerErr {
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		client := client
+		ng.Go(context.Background(), func() error {
+			return client.DeleteUser(accessKey)
+		}, idx, *client.host)
+	}
+	return ng.Wait()
+}
+
+// LoadUser - reloads a specific user across all peers
+func (sys *NotificationSys) LoadUser(accessKey string, temp bool) []NotificationPeerErr {
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		client := client
+		ng.Go(context.Background(), func() error {
+			return client.LoadUser(accessKey, temp)
+		}, idx, *client.host)
+	}
+	return ng.Wait()
+}
+
 // LoadUsers - calls LoadUsers RPC call on all peers.
 func (sys *NotificationSys) LoadUsers() []NotificationPeerErr {
 	ng := WithNPeers(len(sys.peerClients))
@@ -595,7 +656,9 @@ func (sys *NotificationSys) refresh(objAPI ObjectLayer) error {
 		ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{BucketName: bucket.Name})
 		config, err := readNotificationConfig(ctx, objAPI, bucket.Name)
 		if err != nil && err != errNoSuchNotifications {
-			return err
+			if _, ok := err.(*event.ErrARNNotFound); ok {
+				continue
+			}
 		}
 		if err == errNoSuchNotifications {
 			continue
@@ -910,6 +973,11 @@ func (args eventArgs) ToEvent() event.Event {
 }
 
 func sendEvent(args eventArgs) {
+
+	// remove sensitive encryption entries in metadata.
+	crypto.RemoveSensitiveEntries(args.Object.UserDefined)
+	crypto.RemoveInternalEntries(args.Object.UserDefined)
+
 	// globalNotificationSys is not initialized in gateway mode.
 	if globalNotificationSys == nil {
 		return

cmd/object-api-common.go

@@ -272,7 +272,7 @@ func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, d
 		recursive = false
 	}
 
-	walkResultCh, endWalkCh := tpool.Release(listParams{bucket, recursive, marker, prefix})
+	walkResultCh, endWalkCh := tpool.Release(listParams{bucket, recursive, marker, prefix, false})
 	if walkResultCh == nil {
 		endWalkCh = make(chan struct{})
 		walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, endWalkCh)
@@ -333,7 +333,7 @@ func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, d
 	}
 
 	// Save list routine for the next marker if we haven't reached EOF.
-	params := listParams{bucket, recursive, nextMarker, prefix}
+	params := listParams{bucket, recursive, nextMarker, prefix, false}
 	if !eof {
 		tpool.Set(params, walkResultCh, endWalkCh)
 	}

cmd/object-api-errors.go

@@ -27,22 +27,6 @@ import (
 // underlying storage layer.
 func toObjectErr(err error, params ...string) error {
 	switch err {
-	case errDiskNotFound:
-		switch len(params) {
-		case 1:
-			err = BucketNotFound{Bucket: params[0]}
-		case 2:
-			err = ObjectNotFound{
-				Bucket: params[0],
-				Object: params[1],
-			}
-		case 3:
-			err = InvalidUploadID{
-				Bucket:   params[0],
-				Object:   params[1],
-				UploadID: params[2],
-			}
-		}
 	case errVolumeNotFound:
 		if len(params) >= 1 {
 			err = BucketNotFound{Bucket: params[0]}

cmd/object-api-interface.go

@@ -21,7 +21,7 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/minio/minio-go/pkg/encrypt"
+	"github.com/minio/minio-go/v6/pkg/encrypt"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/policy"
 )
@@ -90,9 +90,11 @@ type ObjectLayer interface {
 	HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error)
 	HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (madmin.HealResultItem, error)
 	HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (madmin.HealResultItem, error)
-	ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error)
 	HealObjects(ctx context.Context, bucket, prefix string, healObjectFn func(string, string) error) error
 
+	ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error)
+	ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
+
 	// Policy operations
 	SetBucketPolicy(context.Context, string, *policy.Policy) error
 	GetBucketPolicy(context.Context, string) (*policy.Policy, error)

cmd/object-api-multipart_test.go

@@ -255,8 +255,8 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [
 	}
 
 	// As all disks at not available, bucket not found.
-	expectedErr2 := BucketNotFound{Bucket: testCase.bucketName}
-	if err.Error() != expectedErr2.Error() {
+	expectedErr2 := errDiskNotFound
+	if err != errDiskNotFound {
 		t.Fatalf("Test %s: expected error %s, got %s instead.", instanceType, expectedErr2, err)
 	}
 }

cmd/object-api-utils.go

@@ -34,7 +34,7 @@ import (
 	"unicode/utf8"
 
 	snappy "github.com/golang/snappy"
-	"github.com/minio/minio-go/pkg/s3utils"
+	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"

cmd/object-handlers.go

@@ -33,8 +33,8 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-	miniogo "github.com/minio/minio-go"
-	"github.com/minio/minio-go/pkg/encrypt"
+	miniogo "github.com/minio/minio-go/v6"
+	"github.com/minio/minio-go/v6/pkg/encrypt"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"

cmd/peer-rest-client-target.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2018, 2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

cmd/peer-rest-client.go

@@ -17,12 +17,15 @@
 package cmd
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"crypto/tls"
 	"encoding/gob"
 	"io"
 	"net/url"
+	"strconv"
+	"time"
 
 	"github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -51,9 +54,16 @@ func (client *peerRESTClient) reConnect() error {
 // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
 func (client *peerRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
+	return client.callWithContext(context.Background(), method, values, body, length)
+}
+
+// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
+// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
+// after verifying format.json
+func (client *peerRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
 	if !client.connected {
 		err := client.reConnect()
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(ctx, err)
 		if err != nil {
 			return nil, err
 		}
@@ -63,7 +73,7 @@ func (client *peerRESTClient) call(method string, values url.Values, body io.Rea
 		values = make(url.Values)
 	}
 
-	respBody, err = client.restClient.Call(method, values, body, length)
+	respBody, err = client.restClient.CallWithContext(ctx, method, values, body, length)
 	if err == nil {
 		return respBody, nil
 	}
@@ -337,6 +347,59 @@ func (client *peerRESTClient) PutBucketNotification(bucket string, rulesMap even
 	return nil
 }
 
+// DeletePolicy - delete a specific canned policy.
+func (client *peerRESTClient) DeletePolicy(policyName string) (err error) {
+	values := make(url.Values)
+	values.Set(peerRESTPolicy, policyName)
+
+	respBody, err := client.call(peerRESTMethodDeletePolicy, values, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
+// LoadPolicy - reload a specific canned policy.
+func (client *peerRESTClient) LoadPolicy(policyName string) (err error) {
+	values := make(url.Values)
+	values.Set(peerRESTPolicy, policyName)
+
+	respBody, err := client.call(peerRESTMethodLoadPolicy, values, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
+// DeleteUser - delete a specific user.
+func (client *peerRESTClient) DeleteUser(accessKey string) (err error) {
+	values := make(url.Values)
+	values.Set(peerRESTUser, accessKey)
+
+	respBody, err := client.call(peerRESTMethodDeleteUser, values, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
+// LoadUser - reload a specific user.
+func (client *peerRESTClient) LoadUser(accessKey string, temp bool) (err error) {
+	values := make(url.Values)
+	values.Set(peerRESTUser, accessKey)
+	values.Set(peerRESTUserTemp, strconv.FormatBool(temp))
+
+	respBody, err := client.call(peerRESTMethodLoadUser, values, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
 // LoadUsers - send load users command to peer nodes.
 func (client *peerRESTClient) LoadUsers() (err error) {
 	respBody, err := client.call(peerRESTMethodLoadUsers, nil, nil, -1)
@@ -359,6 +422,54 @@ func (client *peerRESTClient) SignalService(sig serviceSignal) error {
 	return nil
 }
 
+// Trace - send http trace request to peer nodes
+func (client *peerRESTClient) Trace(doneCh chan struct{}, trcAll bool) (chan []byte, error) {
+	ch := make(chan []byte)
+	go func() {
+		cleanupFn := func(cancel context.CancelFunc, ch chan []byte, respBody io.ReadCloser) {
+			close(ch)
+			if cancel != nil {
+				cancel()
+			}
+			http.DrainBody(respBody)
+		}
+		for {
+			values := make(url.Values)
+			values.Set(peerRESTTraceAll, strconv.FormatBool(trcAll))
+			// get cancellation context to properly unsubscribe peers
+			ctx, cancel := context.WithCancel(context.Background())
+			respBody, err := client.callWithContext(ctx, peerRESTMethodTrace, values, nil, -1)
+			if err != nil {
+				//retry
+				time.Sleep(5 * time.Second)
+				select {
+				case <-doneCh:
+					cleanupFn(cancel, ch, respBody)
+					return
+				default:
+				}
+				continue
+			}
+			bio := bufio.NewScanner(respBody)
+			go func() {
+				<-doneCh
+				cancel()
+			}()
+			// Unmarshal each line, returns marshaled values.
+			for bio.Scan() {
+				ch <- bio.Bytes()
+			}
+			select {
+			case <-doneCh:
+				cleanupFn(cancel, ch, respBody)
+				return
+			default:
+			}
+		}
+	}()
+	return ch, nil
+}
+
 func getRemoteHosts(endpoints EndpointList) []*xnet.Host {
 	var remoteHosts []*xnet.Host
 	for _, hostStr := range GetRemotePeers(endpoints) {

cmd/peer-rest-common.go

@@ -16,7 +16,7 @@
 
 package cmd
 
-const peerRESTVersion = "v1"
+const peerRESTVersion = "v2"
 const peerRESTPath = minioReservedBucketPath + "/peer/" + peerRESTVersion
 
 const (
@@ -28,6 +28,10 @@ const (
 	peerRESTMethodSignalService            = "signalservice"
 	peerRESTMethodGetLocks                 = "getlocks"
 	peerRESTMethodBucketPolicyRemove       = "removebucketpolicy"
+	peerRESTMethodLoadUser                 = "loaduser"
+	peerRESTMethodDeleteUser               = "deleteuser"
+	peerRESTMethodLoadPolicy               = "loadpolicy"
+	peerRESTMethodDeletePolicy             = "deletepolicy"
 	peerRESTMethodLoadUsers                = "loadusers"
 	peerRESTMethodStartProfiling           = "startprofiling"
 	peerRESTMethodDownloadProfilingData    = "downloadprofilingdata"
@@ -37,11 +41,16 @@ const (
 	peerRESTMethodReloadFormat             = "reloadformat"
 	peerRESTMethodTargetExists             = "targetexists"
 	peerRESTMethodSendEvent                = "sendevent"
+	peerRESTMethodTrace                    = "trace"
 )
 
 const (
 	peerRESTBucket   = "bucket"
+	peerRESTUser     = "user"
+	peerRESTUserTemp = "user-temp"
+	peerRESTPolicy   = "policy"
 	peerRESTSignal   = "signal"
 	peerRESTProfiler = "profiler"
 	peerRESTDryRun   = "dry-run"
+	peerRESTTraceAll = "all"
 )

cmd/peer-rest-server.go

@@ -19,10 +19,12 @@ package cmd
 import (
 	"context"
 	"encoding/gob"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -31,6 +33,7 @@ import (
 	"github.com/minio/minio/pkg/event"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/policy"
+	trace "github.com/minio/minio/pkg/trace"
 )
 
 // To abstract a node over network.
@@ -117,7 +120,125 @@ func (s *peerRESTServer) GetLocksHandler(w http.ResponseWriter, r *http.Request)
 
 }
 
-// LoadUsersHandler - returns server info.
+// DeletePolicyHandler - deletes a policy on the server.
+func (s *peerRESTServer) DeletePolicyHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		s.writeErrorResponse(w, errServerNotInitialized)
+		return
+	}
+
+	vars := mux.Vars(r)
+	policyName := vars[peerRESTPolicy]
+	if policyName == "" {
+		s.writeErrorResponse(w, errors.New("policyName is missing"))
+		return
+	}
+
+	if err := globalIAMSys.DeletePolicy(policyName); err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// LoadPolicyHandler - reloads a policy on the server.
+func (s *peerRESTServer) LoadPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		s.writeErrorResponse(w, errServerNotInitialized)
+		return
+	}
+
+	vars := mux.Vars(r)
+	policyName := vars[peerRESTPolicy]
+	if policyName == "" {
+		s.writeErrorResponse(w, errors.New("policyName is missing"))
+		return
+	}
+
+	if err := globalIAMSys.LoadPolicy(objAPI, policyName); err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// DeleteUserHandler - deletes a user on the server.
+func (s *peerRESTServer) DeleteUserHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		s.writeErrorResponse(w, errServerNotInitialized)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars[peerRESTUser]
+	if accessKey == "" {
+		s.writeErrorResponse(w, errors.New("username is missing"))
+		return
+	}
+
+	if err := globalIAMSys.DeleteUser(accessKey); err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// LoadUserHandler - reloads a user on the server.
+func (s *peerRESTServer) LoadUserHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		s.writeErrorResponse(w, errServerNotInitialized)
+		return
+	}
+
+	vars := mux.Vars(r)
+	accessKey := vars[peerRESTUser]
+	if accessKey == "" {
+		s.writeErrorResponse(w, errors.New("username is missing"))
+		return
+	}
+
+	temp, err := strconv.ParseBool(vars[peerRESTUserTemp])
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	if err = globalIAMSys.LoadUser(objAPI, accessKey, temp); err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// LoadUsersHandler - reloads all users and canned policies.
 func (s *peerRESTServer) LoadUsersHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		s.writeErrorResponse(w, errors.New("Invalid request"))
@@ -547,6 +668,47 @@ func (s *peerRESTServer) SignalServiceHandler(w http.ResponseWriter, r *http.Req
 	}
 }
 
+// TraceHandler sends http trace messages back to peer rest client
+func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+	trcAll := r.URL.Query().Get(peerRESTTraceAll) == "true"
+
+	w.Header().Set("Connection", "close")
+	w.WriteHeader(http.StatusOK)
+	w.(http.Flusher).Flush()
+	ch := globalTrace.pubsub.Subscribe()
+	defer globalTrace.pubsub.Unsubscribe(ch)
+
+	enc := json.NewEncoder(w)
+	enc.SetEscapeHTML(false)
+	for {
+		select {
+		case entry := <-ch:
+			trcInfo := entry.(trace.Info)
+			path := strings.TrimPrefix(trcInfo.ReqInfo.Path, "/")
+			// omit inter-node traffic if trcAll is false
+			if !trcAll && strings.HasPrefix(path, minioReservedBucket) {
+				continue
+			}
+
+			if err := enc.Encode(trcInfo); err != nil {
+				return
+			}
+
+			if _, err := w.Write([]byte("\n")); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-r.Context().Done():
+			return
+
+		}
+	}
+}
+
 func (s *peerRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
 	w.WriteHeader(http.StatusForbidden)
 	w.Write([]byte(err.Error()))
@@ -576,6 +738,10 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBucketPolicyRemove).HandlerFunc(httpTraceAll(server.RemoveBucketPolicyHandler)).Queries(restQueries(peerRESTBucket)...)
 	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBucketPolicySet).HandlerFunc(httpTraceHdrs(server.SetBucketPolicyHandler)).Queries(restQueries(peerRESTBucket)...)
 
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodDeletePolicy).HandlerFunc(httpTraceAll(server.LoadPolicyHandler)).Queries(restQueries(peerRESTPolicy)...)
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodLoadPolicy).HandlerFunc(httpTraceAll(server.LoadPolicyHandler)).Queries(restQueries(peerRESTPolicy)...)
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodDeleteUser).HandlerFunc(httpTraceAll(server.LoadUserHandler)).Queries(restQueries(peerRESTUser)...)
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodLoadUser).HandlerFunc(httpTraceAll(server.LoadUserHandler)).Queries(restQueries(peerRESTUser, peerRESTUserTemp)...)
 	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodLoadUsers).HandlerFunc(httpTraceAll(server.LoadUsersHandler))
 
 	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodStartProfiling).HandlerFunc(httpTraceAll(server.StartProfilingHandler)).Queries(restQueries(peerRESTProfiler)...)
@@ -588,5 +754,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 
 	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodReloadFormat).HandlerFunc(httpTraceHdrs(server.ReloadFormatHandler)).Queries(restQueries(peerRESTDryRun)...)
 
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodTrace).HandlerFunc(server.TraceHandler)
+
 	router.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
 }

cmd/policy.go

@@ -25,10 +25,9 @@ import (
 	"path"
 	"strings"
 	"sync"
-	"time"
 
-	miniogopolicy "github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio-go/pkg/set"
+	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
@@ -61,6 +60,11 @@ func (sys *PolicySys) removeDeletedBuckets(bucketInfos []BucketInfo) {
 
 // Set - sets policy to given bucket name.  If policy is empty, existing policy is removed.
 func (sys *PolicySys) Set(bucketName string, policy policy.Policy) {
+	if globalIsGateway {
+		// Set policy is a non-op under gateway mode.
+		return
+	}
+
 	sys.Lock()
 	defer sys.Unlock()
 
@@ -81,12 +85,24 @@ func (sys *PolicySys) Remove(bucketName string) {
 
 // IsAllowed - checks given policy args is allowed to continue the Rest API.
 func (sys *PolicySys) IsAllowed(args policy.Args) bool {
-	sys.RLock()
-	defer sys.RUnlock()
+	if globalIsGateway {
+		// When gateway is enabled, no cached value
+		// is used to validate bucket policies.
+		objAPI := newObjectLayerFn()
+		if objAPI != nil {
+			config, err := objAPI.GetBucketPolicy(context.Background(), args.BucketName)
+			if err == nil {
+				return config.IsAllowed(args)
+			}
+		}
+	} else {
+		sys.RLock()
+		defer sys.RUnlock()
 
-	// If policy is available for given bucket, check the policy.
-	if p, found := sys.bucketPolicyMap[args.BucketName]; found {
-		return p.IsAllowed(args)
+		// If policy is available for given bucket, check the policy.
+		if p, found := sys.bucketPolicyMap[args.BucketName]; found {
+			return p.IsAllowed(args)
+		}
 	}
 
 	// As policy is not available for given bucket name, returns IsOwner i.e.
@@ -98,7 +114,6 @@ func (sys *PolicySys) IsAllowed(args policy.Args) bool {
 func (sys *PolicySys) refresh(objAPI ObjectLayer) error {
 	buckets, err := objAPI.ListBuckets(context.Background())
 	if err != nil {
-		logger.LogIf(context.Background(), err)
 		return err
 	}
 	sys.removeDeletedBuckets(buckets)
@@ -135,21 +150,11 @@ func (sys *PolicySys) Init(objAPI ObjectLayer) error {
 		return errInvalidArgument
 	}
 
-	defer func() {
-		// Refresh PolicySys in background.
-		go func() {
-			ticker := time.NewTicker(globalRefreshBucketPolicyInterval)
-			defer ticker.Stop()
-			for {
-				select {
-				case <-GlobalServiceDoneCh:
-					return
-				case <-ticker.C:
-					sys.refresh(objAPI)
-				}
-			}
-		}()
-	}()
+	// In gateway mode, we don't need to load the policies
+	// from the backend.
+	if globalIsGateway {
+		return nil
+	}
 
 	doneCh := make(chan struct{})
 	defer close(doneCh)

cmd/policy_test.go

@@ -20,8 +20,8 @@ import (
 	"reflect"
 	"testing"
 
-	miniogopolicy "github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio-go/pkg/set"
+	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
+	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/policy/condition"
 )

cmd/postpolicyform_test.go

@@ -22,7 +22,7 @@ import (
 	"net/http"
 	"testing"
 
-	minio "github.com/minio/minio-go"
+	minio "github.com/minio/minio-go/v6"
 )
 
 // Test Post Policy parsing and checking conditions

cmd/rest/client.go

@@ -28,12 +28,22 @@ import (
 	"time"
 
 	xhttp "github.com/minio/minio/cmd/http"
-	"golang.org/x/net/http2"
 )
 
 // DefaultRESTTimeout - default RPC timeout is one minute.
 const DefaultRESTTimeout = 1 * time.Minute
 
+// NetworkError - error type in case of errors related to http/transport
+// for ex. connection refused, connection reset, dns resolution failure etc.
+// All errors returned by storage-rest-server (ex errFileNotFound, errDiskNotFound) are not considered to be network errors.
+type NetworkError struct {
+	Err error
+}
+
+func (n *NetworkError) Error() string {
+	return n.Err.Error()
+}
+
 // Client - http based RPC client.
 type Client struct {
 	httpClient          *http.Client
@@ -42,13 +52,13 @@ type Client struct {
 	newAuthToken        func() string
 }
 
-// Call - make a REST call.
-func (c *Client) Call(method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
+// CallWithContext - make a REST call with context.
+func (c *Client) CallWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
 	req, err := http.NewRequest(http.MethodPost, c.url.String()+"/"+method+"?"+values.Encode(), body)
 	if err != nil {
-		return nil, err
+		return nil, &NetworkError{err}
 	}
-
+	req = req.WithContext(ctx)
 	req.Header.Set("Authorization", "Bearer "+c.newAuthToken())
 	req.Header.Set("X-Minio-Time", time.Now().UTC().Format(time.RFC3339))
 	if length > 0 {
@@ -56,7 +66,7 @@ func (c *Client) Call(method string, values url.Values, body io.Reader, length i
 	}
 	resp, err := c.httpClient.Do(req)
 	if err != nil {
-		return nil, err
+		return nil, &NetworkError{err}
 	}
 
 	if resp.StatusCode != http.StatusOK {
@@ -74,6 +84,12 @@ func (c *Client) Call(method string, values url.Values, body io.Reader, length i
 	return resp.Body, nil
 }
 
+// Call - make a REST call.
+func (c *Client) Call(method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
+	ctx := context.Background()
+	return c.CallWithContext(ctx, method, values, body, length)
+}
+
 // Close closes all idle connections of the underlying http client
 func (c *Client) Close() {
 	if c.httpIdleConnsCloser != nil {
@@ -107,12 +123,6 @@ func NewClient(url *url.URL, tlsConfig *tls.Config, timeout time.Duration, newAu
 		TLSClientConfig:       tlsConfig,
 		DisableCompression:    true,
 	}
-	if tlsConfig != nil {
-		// If TLS is enabled configure http2
-		if err := http2.ConfigureTransport(tr); err != nil {
-			return nil, err
-		}
-	}
 	return &Client{
 		httpClient:          &http.Client{Transport: tr},
 		httpIdleConnsCloser: tr.CloseIdleConnections,

cmd/server-main.go

@@ -38,7 +38,8 @@ func init() {
 	logger.RegisterUIError(fmtError)
 }
 
-var serverFlags = []cli.Flag{
+// ServerFlags - server command specific flags
+var ServerFlags = []cli.Flag{
 	cli.StringFlag{
 		Name:  "address",
 		Value: ":" + globalMinioDefaultPort,
@@ -49,7 +50,7 @@ var serverFlags = []cli.Flag{
 var serverCmd = cli.Command{
 	Name:   "server",
 	Usage:  "start object storage server",
-	Flags:  append(serverFlags, globalFlags...),
+	Flags:  append(ServerFlags, GlobalFlags...),
 	Action: serverMain,
 	CustomHelpTemplate: `NAME:
   {{.HelpName}} - {{.Usage}}
@@ -287,8 +288,14 @@ func serverMain(ctx *cli.Context) {
 	// Initialize name space lock.
 	initNSLock(globalIsDistXL)
 
-	// Init global heal state
-	initAllHealState(globalIsXL)
+	if globalIsXL {
+		// Init global heal state
+		globalAllHealState = initHealState()
+		globalSweepHealState = initHealState()
+	}
+
+	// initialize globalTrace system
+	globalTrace = NewTraceSys(context.Background(), globalEndpoints)
 
 	// Configure server.
 	var handler http.Handler
@@ -369,6 +376,12 @@ func serverMain(ctx *cli.Context) {
 		logger.Fatal(errors.New("Invalid KMS configuration"), "auto-encryption is enabled but server does not support encryption")
 	}
 
+	if globalIsXL {
+		initBackgroundHealing()
+		initDailyHeal()
+		initDailySweeper()
+	}
+
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = newObject
 	globalObjLayerMutex.Unlock()

cmd/signals.go

@@ -74,7 +74,6 @@ func handleSignals() {
 
 			exit(err == nil && oerr == nil)
 		case osSignal := <-globalOSSignalCh:
-			stopHTTPTrace()
 			logger.Info("Exiting on signal: %s", strings.ToUpper(osSignal.String()))
 			exit(stopProcess())
 		case signal := <-globalServiceSignalCh:
@@ -83,14 +82,12 @@ func handleSignals() {
 				// Ignore this at the moment.
 			case serviceRestart:
 				logger.Info("Restarting on service signal")
-				stopHTTPTrace()
 				stop := stopProcess()
 				rerr := restartProcess()
 				logger.LogIf(context.Background(), rerr)
 				exit(stop && rerr == nil)
 			case serviceStop:
 				logger.Info("Stopping on service signal")
-				stopHTTPTrace()
 				exit(stopProcess())
 			}
 		}

cmd/signature-v4.go

@@ -35,7 +35,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/minio-go/pkg/s3utils"
+	"github.com/minio/minio-go/v6/pkg/s3utils"
 	sha256 "github.com/minio/sha256-simd"
 )
 

cmd/storage-rest-client.go

@@ -21,7 +21,6 @@ import (
 	"crypto/tls"
 	"io"
 	"io/ioutil"
-	"net"
 	"net/url"
 	"path"
 	"strconv"
@@ -44,32 +43,10 @@ func isNetworkError(err error) bool {
 	if err.Error() == errConnectionStale.Error() {
 		return true
 	}
-	if strings.Contains(err.Error(), "connection reset by peer") {
+	if _, ok := err.(*rest.NetworkError); ok {
 		return true
 	}
-	if uerr, isURLError := err.(*url.Error); isURLError {
-		if uerr.Timeout() {
-			return true
-		}
-
-		err = uerr.Err
-	}
-
-	_, isNetOpError := err.(*net.OpError)
-	return isNetOpError
-}
-
-// Attempt to approximate network error with a
-// typed network error, otherwise default to
-// errDiskNotFound
-func toNetworkError(err error) error {
-	if err == nil {
-		return err
-	}
-	if strings.Contains(err.Error(), "connection reset by peer") {
-		return errNetworkConnReset
-	}
-	return errDiskNotFound
+	return false
 }
 
 // Converts rpc.ServerError to underlying error. This function is
@@ -81,7 +58,7 @@ func toStorageErr(err error) error {
 	}
 
 	if isNetworkError(err) {
-		return toNetworkError(err)
+		return errDiskNotFound
 	}
 
 	switch err.Error() {

cmd/sts-handlers.go

@@ -189,7 +189,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Notify all other MinIO peers to reload temp users
-	for _, nerr := range globalNotificationSys.LoadUsers() {
+	for _, nerr := range globalNotificationSys.LoadUser(cred.AccessKey, true) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 			logger.LogIf(ctx, nerr.Err)
@@ -306,7 +306,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
 	}
 
 	// Notify all other MinIO peers to reload temp users
-	for _, nerr := range globalNotificationSys.LoadUsers() {
+	for _, nerr := range globalNotificationSys.LoadUser(cred.AccessKey, true) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 			logger.LogIf(ctx, nerr.Err)

cmd/test-utils_test.go

@@ -54,8 +54,8 @@ import (
 
 	"github.com/fatih/color"
 	"github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/s3signer"
-	"github.com/minio/minio-go/pkg/s3utils"
+	"github.com/minio/minio-go/v6/pkg/s3signer"
+	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/bpool"
@@ -470,6 +470,9 @@ func resetGlobalStorageEnvs() {
 
 // reset global heal state
 func resetGlobalHealState() {
+	if globalAllHealState == nil {
+		return
+	}
 	globalAllHealState.Lock()
 	defer globalAllHealState.Unlock()
 	for _, v := range globalAllHealState.healSeqMap {

cmd/tree-walk-pool.go

@@ -34,6 +34,7 @@ type listParams struct {
 	recursive bool
 	marker    string
 	prefix    string
+	heal      bool
 }
 
 // errWalkAbort - returned by doTreeWalk() if it returns prematurely.

cmd/typed-errors.go

@@ -85,6 +85,3 @@ var errNoSuchPolicy = errors.New("Specified canned policy does not exist")
 
 // error returned when access is denied.
 var errAccessDenied = errors.New("Do not have enough permissions to access this resource")
-
-// errNetworkConnReset - connection reset by peer
-var errNetworkConnReset = errors.New("connection reset by peer")

cmd/utils.go

@@ -59,16 +59,6 @@ func IsErr(err error, errs ...error) bool {
 	return false
 }
 
-// Close Http tracing file.
-func stopHTTPTrace() {
-	if globalHTTPTraceFile != nil {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("traceFile", globalHTTPTraceFile.Name())
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogIf(ctx, globalHTTPTraceFile.Close())
-		globalHTTPTraceFile = nil
-	}
-}
-
 // make a copy of http.Header
 func cloneHeader(h http.Header) http.Header {
 	h2 := make(http.Header, len(h))

cmd/utils_test.go

@@ -54,19 +54,6 @@ func TestCloneHeader(t *testing.T) {
 	}
 }
 
-// Tests closing http tracing file.
-func TestStopHTTPTrace(t *testing.T) {
-	var err error
-	globalHTTPTraceFile, err = ioutil.TempFile("", "")
-	if err != nil {
-		defer os.Remove(globalHTTPTraceFile.Name())
-		stopHTTPTrace()
-		if globalHTTPTraceFile != nil {
-			t.Errorf("globalHTTPTraceFile is not nil, it is expected to be nil")
-		}
-	}
-}
-
 // Tests maximum object size.
 func TestMaxObjectSize(t *testing.T) {
 	sizes := []struct {

cmd/web-handlers_test.go

@@ -35,7 +35,7 @@ import (
 
 	jwtgo "github.com/dgrijalva/jwt-go"
 	humanize "github.com/dustin/go-humanize"
-	miniogopolicy "github.com/minio/minio-go/pkg/policy"
+	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/policy/condition"
@@ -701,18 +701,21 @@ func testSetAuthWebHandler(obj ObjectLayer, instanceType string, t TestErrHandle
 	}
 
 	testCases := []struct {
-		username string
-		password string
-		success  bool
+		currentAccessKey string
+		currentSecretKey string
+		newAccessKey     string
+		newSecretKey     string
+		success          bool
 	}{
-		{"", "", false},
-		{"1", "1", false},
-		{"azerty", "foooooooooooooo", true},
+		{"", "", "", "", false},
+		{"1", "1", "1", "1", false},
+		{credentials.AccessKey, credentials.SecretKey, "azerty", "bar", false},
+		{credentials.AccessKey, credentials.SecretKey, "azerty", "foooooooooooooo", true},
 	}
 
 	// Iterating over the test cases, calling the function under test and asserting the response.
 	for i, testCase := range testCases {
-		setAuthRequest := SetAuthArgs{AccessKey: testCase.username, SecretKey: testCase.password}
+		setAuthRequest := SetAuthArgs{CurrentAccessKey: testCase.currentAccessKey, CurrentSecretKey: testCase.currentSecretKey, NewAccessKey: testCase.newAccessKey, NewSecretKey: testCase.newSecretKey}
 		setAuthReply := &SetAuthReply{}
 		req, err := newTestWebRPCRequest("Web.SetAuth", authorization, setAuthRequest)
 		if err != nil {
@@ -735,42 +738,6 @@ func testSetAuthWebHandler(obj ObjectLayer, instanceType string, t TestErrHandle
 	}
 }
 
-// Wrapper for calling Get Auth Handler
-func TestWebHandlerGetAuth(t *testing.T) {
-	ExecObjectLayerTest(t, testGetAuthWebHandler)
-}
-
-// testGetAuthWebHandler - Test GetAuth web handler
-func testGetAuthWebHandler(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	// Register the API end points with XL/FS object layer.
-	apiRouter := initTestWebRPCEndPoint(obj)
-	credentials := globalServerConfig.GetCredential()
-
-	rec := httptest.NewRecorder()
-	authorization, err := getWebRPCToken(apiRouter, credentials.AccessKey, credentials.SecretKey)
-	if err != nil {
-		t.Fatal("Cannot authenticate")
-	}
-
-	getAuthRequest := WebGenericArgs{}
-	getAuthReply := &GetAuthReply{}
-	req, err := newTestWebRPCRequest("Web.GetAuth", authorization, getAuthRequest)
-	if err != nil {
-		t.Fatalf("Failed to create HTTP request: <ERROR> %v", err)
-	}
-	apiRouter.ServeHTTP(rec, req)
-	if rec.Code != http.StatusOK {
-		t.Fatalf("Expected the response status to be 200, but instead found `%d`", rec.Code)
-	}
-	err = getTestWebRPCResponse(rec, &getAuthReply)
-	if err != nil {
-		t.Fatalf("Failed, %v", err)
-	}
-	if getAuthReply.AccessKey != credentials.AccessKey || getAuthReply.SecretKey != credentials.SecretKey {
-		t.Fatalf("Failed to get correct auth keys")
-	}
-}
-
 func TestWebCreateURLToken(t *testing.T) {
 	ExecObjectLayerTest(t, testCreateURLToken)
 }
@@ -1518,7 +1485,7 @@ func TestWebCheckAuthorization(t *testing.T) {
 	webRPCs := []string{
 		"ServerInfo", "StorageInfo", "MakeBucket",
 		"ListBuckets", "ListObjects", "RemoveObject",
-		"GenerateAuth", "SetAuth", "GetAuth",
+		"GenerateAuth", "SetAuth",
 		"GetBucketPolicy", "SetBucketPolicy", "ListAllBucketPolicies",
 		"PresignedGet",
 	}

cmd/xl-sets.go

@@ -844,6 +844,10 @@ func leastEntry(entriesCh []FileInfoCh, readQuorum int) (FileInfo, bool) {
 		entriesCh[i].Push(entries[i])
 	}
 
+	if readQuorum < 0 {
+		return lentry, isTruncated
+	}
+
 	quorum := lentry.Quorum
 	if quorum == 0 {
 		quorum = readQuorum
@@ -906,7 +910,7 @@ func (s *xlSets) startMergeWalks(ctx context.Context, bucket, prefix, marker str
 // ListObjects - implements listing of objects across disks, each disk is indepenently
 // walked and merged at this layer. Resulting value through the merge process sends
 // the data in lexically sorted order.
-func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
+func (s *xlSets) listObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int, heal bool) (loi ListObjectsInfo, err error) {
 	if err = checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, s); err != nil {
 		return loi, err
 	}
@@ -944,13 +948,18 @@ func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimi
 		recursive = false
 	}
 
-	entryChs, endWalkCh := s.pool.Release(listParams{bucket, recursive, marker, prefix})
+	entryChs, endWalkCh := s.pool.Release(listParams{bucket, recursive, marker, prefix, heal})
 	if entryChs == nil {
 		endWalkCh = make(chan struct{})
 		entryChs = s.startMergeWalks(context.Background(), bucket, prefix, marker, recursive, endWalkCh)
 	}
 
-	entries := mergeEntriesCh(entryChs, maxKeys, s.drivesPerSet/2)
+	readQuorum := s.drivesPerSet / 2
+	if heal {
+		readQuorum = -1
+	}
+
+	entries := mergeEntriesCh(entryChs, maxKeys, readQuorum)
 	if len(entries.Files) == 0 {
 		return loi, nil
 	}
@@ -1004,11 +1013,18 @@ func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimi
 		loi.Objects = append(loi.Objects, objInfo)
 	}
 	if loi.IsTruncated {
-		s.pool.Set(listParams{bucket, recursive, loi.NextMarker, prefix}, entryChs, endWalkCh)
+		s.pool.Set(listParams{bucket, recursive, loi.NextMarker, prefix, heal}, entryChs, endWalkCh)
 	}
 	return loi, nil
 }
 
+// ListObjects - implements listing of objects across disks, each disk is indepenently
+// walked and merged at this layer. Resulting value through the merge process sends
+// the data in lexically sorted order.
+func (s *xlSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
+	return s.listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys, false)
+}
+
 func (s *xlSets) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
 	// In list multipart uploads we are going to treat input prefix as the object,
 	// this means that we are not supporting directory navigation.
@@ -1536,3 +1552,7 @@ func (s *xlSets) HealObjects(ctx context.Context, bucket, prefix string, healObj
 
 	return nil
 }
+
+func (s *xlSets) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
+	return s.listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys, true)
+}

cmd/xl-v1-bucket.go

@@ -21,7 +21,7 @@ import (
 	"sort"
 	"sync"
 
-	"github.com/minio/minio-go/pkg/s3utils"
+	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/policy"
 )

cmd/xl-v1-list-objects-heal.go

@@ -23,6 +23,11 @@ func (xl xlObjects) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error) {
 	return nil, nil
 }
 
+// This is not implemented, look for xl-sets.ListObjectsHeal()
+func (xl xlObjects) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
+	return ListObjectsInfo{}, nil
+}
+
 // This is not implemented/needed anymore, look for xl-sets.HealObjects()
 func (xl xlObjects) HealObjects(ctx context.Context, bucket, prefix string, healFn func(string, string) error) (e error) {
 	return nil