config: More fixes in parsing Audit & Logger env variables (#9474)

- Add support of missed legacy Logger webhook
- Disable enabling Audit or logger if _ENABLE
  if not explicitly set to "on".

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: MinIO Community Support
+    url: https://slack.min.io
+    about: Please ask and answer questions here.
+  - name: MinIO SUBNET Support
+    url: https://min.io/pricing
+    about: Join this for Enterprise Support.

.github/lock.yml

@@ -0,0 +1,39 @@
+# Configuration for Lock Threads - https://github.com/dessant/lock-threads-app
+
+# Number of days of inactivity before a closed issue or pull request is locked
+daysUntilLock: 365
+
+# Skip issues and pull requests created before a given timestamp. Timestamp must
+# follow ISO 8601 (`YYYY-MM-DD`). Set to `false` to disable
+skipCreatedBefore: false
+
+# Issues and pull requests with these labels will be ignored. Set to `[]` to disable
+exemptLabels: []
+
+# Label to add before locking, such as `outdated`. Set to `false` to disable
+lockLabel: false
+
+# Comment to post before locking. Set to `false` to disable
+lockComment: >-
+
+  This thread has been automatically locked since there has not been
+  any recent activity after it was closed. Please open a new issue for
+  related bugs.
+
+# Assign `resolved` as the reason for locking. Set to `false` to disable
+setLockReason: true
+
+# Limit to only `issues` or `pulls`
+only: issues
+
+# Optionally, specify configuration settings just for `issues` or `pulls`
+# issues:
+#   exemptLabels:
+#     - help-wanted
+#   lockLabel: outdated
+
+# pulls:
+#   daysUntilLock: 30
+
+# Repository to extend settings from
+# _extends: repo

.github/stale.yml

@@ -0,0 +1,59 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 90
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+daysUntilClose: 30
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
+exemptLabels:
+  - "security"
+  - "pending discussion"
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking as stale
+staleLabel: stale
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >-
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed after 21 days if no further activity
+  occurs. Thank you for your contributions.
+# Comment to post when removing the stale label.
+# unmarkComment: >
+#   Your comment here.
+
+# Comment to post when closing a stale Issue or Pull Request.
+# closeComment: >
+#   Your comment here.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 1
+
+# Limit to only `issues` or `pulls`
+# only: issues
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed

Dockerfile

@@ -22,8 +22,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 EXPOSE 9000
 
 COPY --from=0 /go/bin/minio /usr/bin/minio
-COPY CREDITS /third_party/
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/
+COPY --from=0 /go/minio/CREDITS /third_party/
+COPY --from=0 /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \

Dockerfile.mint

@@ -1,17 +1,11 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 ENV DEBIAN_FRONTEND noninteractive
-
 ENV LANG C.UTF-8
-
 ENV GOROOT /usr/local/go
-
-ENV GOPATH /usr/local
-
+ENV GOPATH /usr/local/gopath
 ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
-
 ENV MINT_ROOT_DIR /mint
-
 COPY mint /mint
 
 RUN apt-get --yes update && apt-get --yes upgrade && \

Dockerfile.simpleci

@@ -57,7 +57,7 @@ RUN yarn test
 #-------------------------------------------------------------
 # Stage 3: Run Gateway Tests
 #-------------------------------------------------------------
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 COPY --from=0 /go/src/github.com/minio/minio/minio /usr/bin/minio
 COPY buildscripts/gateway-tests.sh /usr/bin/gateway-tests.sh
@@ -66,7 +66,7 @@ COPY mint /mint
 ENV DEBIAN_FRONTEND noninteractive
 ENV LANG C.UTF-8
 ENV GOROOT /usr/local/go
-ENV GOPATH /usr/local
+ENV GOPATH /usr/local/gopath
 ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
 ENV SIMPLE_CI 1
 ENV MINT_ROOT_DIR /mint

README.md

@@ -185,11 +185,5 @@ mc admin update <minio alias, e.g., myminio>
 ## Contribute to MinIO Project
 Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
 
-## Caveats
-MinIO in its default mode doesn't use MD5Sum checkums of incoming streams unless requested by the client in `Content-Md5` header for validation. This may lead to incompatibility with rare S3 clients like `s3ql` which unfortunately do not set `Content-Md5` but depend on hex MD5Sum for the stream to be calculated by the server. MinIO considers this as a bug in `s3ql` and should be fixed on the client side because MD5Sum is a poor way to checksum and validate the authenticity of the objects. Although MinIO provides a workaround until client applications are fixed use `--compat` option instead to start the server.
-```sh
-./minio --compat server /data
-```
-
 ## License
 [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fminio%2Fminio.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fminio%2Fminio?ref=badge_large)

browser/app/js/browser/ChangePasswordModal.js

@@ -88,11 +88,6 @@ export class ChangePasswordModal extends React.Component {
 
   canChangePassword() {
     const { serverInfo } = this.props
-    // Password change is not allowed in WORM mode
-    if (serverInfo.info.isWorm) {
-      return false
-    }
-
     // Password change is not allowed for temporary users(STS)
     if(serverInfo.userInfo.isTempUser) {
       return false

browser/app/js/browser/__tests__/ChangePasswordModal.test.js

@@ -64,17 +64,6 @@ describe("ChangePasswordModal", () => {
     shallow(<ChangePasswordModal serverInfo={serverInfo} />)
   })
 
-  it("should not allow changing password when isWorm is true", () => {
-    const newServerInfo = { ...serverInfo, info: { isWorm: true } }
-    const wrapper = shallow(<ChangePasswordModal serverInfo={newServerInfo} />)
-    expect(
-      wrapper
-        .find("ModalBody")
-        .childAt(0)
-        .text()
-    ).toBe("Credentials of this user cannot be updated through MinIO Browser.")
-  })
-
   it("should not allow changing password when not IAM user", () => {
     const newServerInfo = {
       ...serverInfo,

browser/app/js/mime.js

@@ -14,30 +14,67 @@
  * limitations under the License.
  */
 
-import mimedb from 'mime-types'
+import mimedb from "mime-types"
 
 const isFolder = (name, contentType) => {
-  if (name.endsWith('/')) return true
+  if (name.endsWith("/")) return true
   return false
 }
 
 const isPdf = (name, contentType) => {
-  if (contentType === 'application/pdf') return true
+  if (contentType === "application/pdf") return true
+  return false
+}
+const isImage = (name, contentType) => {
+  if (
+    contentType === "image/jpeg" ||
+    contentType === "image/gif" ||
+    contentType === "image/x-icon" ||
+    contentType === "image/png" ||
+    contentType === "image/svg+xml" ||
+    contentType === "image/tiff" ||
+    contentType === "image/webp"
+  )
+    return true
   return false
 }
 
 const isZip = (name, contentType) => {
-  if (!contentType || !contentType.includes('/')) return false
-  if (contentType.split('/')[1].includes('zip')) return true
+  if (!contentType || !contentType.includes("/")) return false
+  if (contentType.split("/")[1].includes("zip")) return true
   return false
 }
 
 const isCode = (name, contentType) => {
-  const codeExt = ['c', 'cpp', 'go', 'py', 'java', 'rb', 'js', 'pl', 'fs',
-    'php', 'css', 'less', 'scss', 'coffee', 'net', 'html',
-    'rs', 'exs', 'scala', 'hs', 'clj', 'el', 'scm', 'lisp',
-    'asp', 'aspx']
-  const ext = name.split('.').reverse()[0]
+  const codeExt = [
+    "c",
+    "cpp",
+    "go",
+    "py",
+    "java",
+    "rb",
+    "js",
+    "pl",
+    "fs",
+    "php",
+    "css",
+    "less",
+    "scss",
+    "coffee",
+    "net",
+    "html",
+    "rs",
+    "exs",
+    "scala",
+    "hs",
+    "clj",
+    "el",
+    "scm",
+    "lisp",
+    "asp",
+    "aspx",
+  ]
+  const ext = name.split(".").reverse()[0]
   for (var i in codeExt) {
     if (ext === codeExt[i]) return true
   }
@@ -45,9 +82,9 @@ const isCode = (name, contentType) => {
 }
 
 const isExcel = (name, contentType) => {
-  if (!contentType || !contentType.includes('/')) return false
-  const types = ['excel', 'spreadsheet']
-  const subType = contentType.split('/')[1]
+  if (!contentType || !contentType.includes("/")) return false
+  const types = ["excel", "spreadsheet"]
+  const subType = contentType.split("/")[1]
   for (var i in types) {
     if (subType.includes(types[i])) return true
   }
@@ -55,9 +92,9 @@ const isExcel = (name, contentType) => {
 }
 
 const isDoc = (name, contentType) => {
-  if (!contentType || !contentType.includes('/')) return false
-  const types = ['word', '.document']
-  const subType = contentType.split('/')[1]
+  if (!contentType || !contentType.includes("/")) return false
+  const types = ["word", ".document"]
+  const subType = contentType.split("/")[1]
   for (var i in types) {
     if (subType.includes(types[i])) return true
   }
@@ -65,9 +102,9 @@ const isDoc = (name, contentType) => {
 }
 
 const isPresentation = (name, contentType) => {
-  if (!contentType || !contentType.includes('/')) return false
-  var types = ['powerpoint', 'presentation']
-  const subType = contentType.split('/')[1]
+  if (!contentType || !contentType.includes("/")) return false
+  var types = ["powerpoint", "presentation"]
+  const subType = contentType.split("/")[1]
   for (var i in types) {
     if (subType.includes(types[i])) return true
   }
@@ -76,31 +113,32 @@ const isPresentation = (name, contentType) => {
 
 const typeToIcon = (type) => {
   return (name, contentType) => {
-    if (!contentType || !contentType.includes('/')) return false
-    if (contentType.split('/')[0] === type) return true
+    if (!contentType || !contentType.includes("/")) return false
+    if (contentType.split("/")[0] === type) return true
     return false
   }
 }
 
 export const getDataType = (name, contentType) => {
   if (contentType === "") {
-    contentType = mimedb.lookup(name) || 'application/octet-stream'
+    contentType = mimedb.lookup(name) || "application/octet-stream"
   }
   const check = [
-    ['folder', isFolder],
-    ['code', isCode],
-    ['audio', typeToIcon('audio')],
-    ['image', typeToIcon('image')],
-    ['video', typeToIcon('video')],
-    ['text', typeToIcon('text')],
-    ['pdf', isPdf],
-    ['zip', isZip],
-    ['excel', isExcel],
-    ['doc', isDoc],
-    ['presentation', isPresentation]
+    ["folder", isFolder],
+    ["code", isCode],
+    ["audio", typeToIcon("audio")],
+    ["image", typeToIcon("image")],
+    ["video", typeToIcon("video")],
+    ["text", typeToIcon("text")],
+    ["pdf", isPdf],
+    ["image", isImage],
+    ["zip", isZip],
+    ["excel", isExcel],
+    ["doc", isDoc],
+    ["presentation", isPresentation],
   ]
   for (var i in check) {
     if (check[i][1](name, contentType)) return check[i][0]
   }
-  return 'other'
+  return "other"
 }

browser/app/js/objects/ObjectActions.js

@@ -19,18 +19,22 @@ import { connect } from "react-redux"
 import { Dropdown } from "react-bootstrap"
 import ShareObjectModal from "./ShareObjectModal"
 import DeleteObjectConfirmModal from "./DeleteObjectConfirmModal"
+import PreviewObjectModal from "./PreviewObjectModal"
+
 import * as objectsActions from "./actions"
+import { getDataType } from "../mime.js"
 import {
   SHARE_OBJECT_EXPIRY_DAYS,
   SHARE_OBJECT_EXPIRY_HOURS,
-  SHARE_OBJECT_EXPIRY_MINUTES
+  SHARE_OBJECT_EXPIRY_MINUTES,
 } from "../constants"
 
 export class ObjectActions extends React.Component {
   constructor(props) {
     super(props)
     this.state = {
-      showDeleteConfirmation: false
+      showDeleteConfirmation: false,
+      showPreview: false,
     }
   }
   shareObject(e) {
@@ -53,7 +57,20 @@ export class ObjectActions extends React.Component {
   }
   hideDeleteConfirmModal() {
     this.setState({
-      showDeleteConfirmation: false
+      showDeleteConfirmation: false,
+    })
+  }
+  getObjectURL(objectname, callback) {
+    const { getObjectURL } = this.props
+    getObjectURL(objectname, callback)
+  }
+  showPreviewModal(e) {
+    e.preventDefault()
+    this.setState({ showPreview: true })
+  }
+  hidePreviewModal() {
+    this.setState({
+      showPreview: false,
     })
   }
   render() {
@@ -69,6 +86,15 @@ export class ObjectActions extends React.Component {
           >
             <i className="fas fa-share-alt" />
           </a>
+          {getDataType(object.name, object.contentType) == "image" && (
+            <a
+              href=""
+              className="fiad-action"
+              onClick={this.showPreviewModal.bind(this)}
+            >
+              <i className="far fa-file-image" />
+            </a>
+          )}
           <a
             href=""
             className="fiad-action"
@@ -77,14 +103,22 @@ export class ObjectActions extends React.Component {
             <i className="fas fa-trash-alt" />
           </a>
         </Dropdown.Menu>
-        {(showShareObjectModal && shareObjectName === object.name) &&
-          <ShareObjectModal object={object} />}
+        {showShareObjectModal && shareObjectName === object.name && (
+          <ShareObjectModal object={object} />
+        )}
         {this.state.showDeleteConfirmation && (
           <DeleteObjectConfirmModal
             deleteObject={this.deleteObject.bind(this)}
             hideDeleteConfirmModal={this.hideDeleteConfirmModal.bind(this)}
           />
         )}
+        {this.state.showPreview && (
+          <PreviewObjectModal
+            object={object}
+            hidePreviewModal={this.hidePreviewModal.bind(this)}
+            getObjectURL={this.getObjectURL.bind(this)}
+          />
+        )}
       </Dropdown>
     )
   }
@@ -94,15 +128,17 @@ const mapStateToProps = (state, ownProps) => {
   return {
     object: ownProps.object,
     showShareObjectModal: state.objects.shareObject.show,
-    shareObjectName: state.objects.shareObject.object
+    shareObjectName: state.objects.shareObject.object,
   }
 }
 
-const mapDispatchToProps = dispatch => {
+const mapDispatchToProps = (dispatch) => {
   return {
     shareObject: (object, days, hours, minutes) =>
       dispatch(objectsActions.shareObject(object, days, hours, minutes)),
-    deleteObject: object => dispatch(objectsActions.deleteObject(object))
+    deleteObject: (object) => dispatch(objectsActions.deleteObject(object)),
+    getObjectURL: (object, callback) =>
+      dispatch(objectsActions.getObjectURL(object, callback)),
   }
 }
 

browser/app/js/objects/PreviewObjectModal.js

@@ -0,0 +1,65 @@
+/*
+ * MinIO Cloud Storage (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from "react"
+import { Modal, ModalHeader, ModalBody } from "react-bootstrap"
+
+class PreviewObjectModal extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      url: "",
+    }
+    props.getObjectURL(props.object.name, (url) => {
+      this.setState({
+        url: url,
+      })
+    })
+  }
+
+  render() {
+    const { hidePreviewModal } = this.props
+    return (
+      <Modal
+        show={true}
+        animation={false}
+        onHide={hidePreviewModal}
+        bsSize="large"
+      >
+        <ModalHeader>Preview</ModalHeader>
+        <ModalBody>
+          <div className="input-group">
+            {this.state.url && (
+              <img
+                alt="Image broken"
+                src={this.state.url}
+                style={{ display: "block", width: "100%" }}
+              />
+            )}
+          </div>
+        </ModalBody>
+        <div className="modal-footer">
+          {
+            <button className="btn btn-link" onClick={hidePreviewModal}>
+              Cancel
+            </button>
+          }
+        </div>
+      </Modal>
+    )
+  }
+}
+export default PreviewObjectModal

browser/app/js/objects/__tests__/ObjectActions.test.js

@@ -66,6 +66,49 @@ describe("ObjectActions", () => {
     expect(deleteObject).toHaveBeenCalledWith("obj1")
   })
 
+
+
+
+  it("should show PreviewObjectModal when preview action is clicked", () => {
+    const wrapper = shallow(
+      <ObjectActions 
+      object={{ name: "obj1", contentType: "image/jpeg"}} 
+      currentPrefix={"pre1/"} />
+    )
+    wrapper
+      .find("a")
+      .at(1)
+      .simulate("click", { preventDefault: jest.fn() })
+    expect(wrapper.state("showPreview")).toBeTruthy()
+    expect(wrapper.find("PreviewObjectModal").length).toBe(1)
+  })
+
+  it("should hide PreviewObjectModal when cancel button is clicked", () => {
+    const wrapper = shallow(
+      <ObjectActions 
+        object={{ name: "obj1" , contentType: "image/jpeg"}}
+        currentPrefix={"pre1/"} />
+    )
+    wrapper
+      .find("a")
+      .at(1)
+      .simulate("click", { preventDefault: jest.fn() })
+    wrapper.find("PreviewObjectModal").prop("hidePreviewModal")()
+    wrapper.update()
+    expect(wrapper.state("showPreview")).toBeFalsy()
+    expect(wrapper.find("PreviewObjectModal").length).toBe(0)
+  })
+  it("should not show PreviewObjectModal when preview action is clicked if object is not an image", () => {
+    const wrapper = shallow(
+      <ObjectActions 
+      object={{ name: "obj1"}} 
+      currentPrefix={"pre1/"} />
+    )
+    expect(wrapper
+      .find("a")
+      .length).toBe(2) // find only the other 2
+  })
+
   it("should call shareObject with object and expiry", () => {
     const shareObject = jest.fn()
     const wrapper = shallow(

browser/app/js/objects/actions.js

@@ -19,7 +19,7 @@ import history from "../history"
 import {
   sortObjectsByName,
   sortObjectsBySize,
-  sortObjectsByDate
+  sortObjectsByDate,
 } from "../utils"
 import { getCurrentBucket } from "../buckets/selectors"
 import { getCurrentPrefix, getCheckedList } from "./selectors"
@@ -31,7 +31,7 @@ import {
   SORT_BY_SIZE,
   SORT_BY_LAST_MODIFIED,
   SORT_ORDER_ASC,
-  SORT_ORDER_DESC
+  SORT_ORDER_DESC,
 } from "../constants"
 
 export const SET_LIST = "objects/SET_LIST"
@@ -48,35 +48,35 @@ export const CHECKED_LIST_REMOVE = "objects/CHECKED_LIST_REMOVE"
 export const CHECKED_LIST_RESET = "objects/CHECKED_LIST_RESET"
 export const SET_LIST_LOADING = "objects/SET_LIST_LOADING"
 
-export const setList = objects => ({
+export const setList = (objects) => ({
   type: SET_LIST,
-  objects
+  objects,
 })
 
 export const resetList = () => ({
-  type: RESET_LIST
+  type: RESET_LIST,
 })
 
-export const setListLoading = listLoading => ({
+export const setListLoading = (listLoading) => ({
   type: SET_LIST_LOADING,
-  listLoading
+  listLoading,
 })
 
 export const fetchObjects = () => {
-  return function(dispatch, getState) {
+  return function (dispatch, getState) {
     dispatch(resetList())
     const {
       buckets: { currentBucket },
-      objects: { currentPrefix }
+      objects: { currentPrefix },
     } = getState()
     if (currentBucket) {
       dispatch(setListLoading(true))
       return web
         .ListObjects({
           bucketName: currentBucket,
-          prefix: currentPrefix
+          prefix: currentPrefix,
         })
-        .then(res => {
+        .then((res) => {
           // we need to check if the bucket name and prefix are the same as
           // when the request was made before updating the displayed objects
           if (
@@ -85,10 +85,10 @@ export const fetchObjects = () => {
           ) {
             let objects = []
             if (res.objects) {
-              objects = res.objects.map(object => {
+              objects = res.objects.map((object) => {
                 return {
                   ...object,
-                  name: object.name.replace(currentPrefix, "")
+                  name: object.name.replace(currentPrefix, ""),
                 }
               })
             }
@@ -104,13 +104,13 @@ export const fetchObjects = () => {
             dispatch(setListLoading(false))
           }
         })
-        .catch(err => {
+        .catch((err) => {
           if (web.LoggedIn()) {
             dispatch(
               alertActions.set({
                 type: "danger",
                 message: err.message,
-                autoClear: true
+                autoClear: true,
               })
             )
             dispatch(resetList())
@@ -123,8 +123,8 @@ export const fetchObjects = () => {
   }
 }
 
-export const sortObjects = sortBy => {
-  return function(dispatch, getState) {
+export const sortObjects = (sortBy) => {
+  return function (dispatch, getState) {
     const { objects } = getState()
     let sortOrder = SORT_ORDER_ASC
     // Reverse sort order if the list is already sorted on same field
@@ -149,18 +149,18 @@ const sortObjectsList = (list, sortBy, sortOrder) => {
   }
 }
 
-export const setSortBy = sortBy => ({
+export const setSortBy = (sortBy) => ({
   type: SET_SORT_BY,
-  sortBy
+  sortBy,
 })
 
-export const setSortOrder = sortOrder => ({
+export const setSortOrder = (sortOrder) => ({
   type: SET_SORT_ORDER,
-  sortOrder
+  sortOrder,
 })
 
-export const selectPrefix = prefix => {
-  return function(dispatch, getState) {
+export const selectPrefix = (prefix) => {
+  return function (dispatch, getState) {
     dispatch(setCurrentPrefix(prefix))
     dispatch(fetchObjects())
     dispatch(resetCheckedList())
@@ -169,49 +169,49 @@ export const selectPrefix = prefix => {
   }
 }
 
-export const setCurrentPrefix = prefix => {
+export const setCurrentPrefix = (prefix) => {
   return {
     type: SET_CURRENT_PREFIX,
-    prefix
+    prefix,
   }
 }
 
-export const setPrefixWritable = prefixWritable => ({
+export const setPrefixWritable = (prefixWritable) => ({
   type: SET_PREFIX_WRITABLE,
-  prefixWritable
+  prefixWritable,
 })
 
-export const deleteObject = object => {
-  return function(dispatch, getState) {
+export const deleteObject = (object) => {
+  return function (dispatch, getState) {
     const currentBucket = getCurrentBucket(getState())
     const currentPrefix = getCurrentPrefix(getState())
     const objectName = `${currentPrefix}${object}`
     return web
       .RemoveObject({
         bucketName: currentBucket,
-        objects: [objectName]
+        objects: [objectName],
       })
       .then(() => {
         dispatch(removeObject(object))
       })
-      .catch(e => {
+      .catch((e) => {
         dispatch(
           alertActions.set({
             type: "danger",
-            message: e.message
+            message: e.message,
           })
         )
       })
   }
 }
 
-export const removeObject = object => ({
+export const removeObject = (object) => ({
   type: REMOVE,
-  object
+  object,
 })
 
 export const deleteCheckedObjects = () => {
-  return function(dispatch, getState) {
+  return function (dispatch, getState) {
     const checkedObjects = getCheckedList(getState())
     for (let i = 0; i < checkedObjects.length; i++) {
       dispatch(deleteObject(checkedObjects[i]))
@@ -221,7 +221,7 @@ export const deleteCheckedObjects = () => {
 }
 
 export const shareObject = (object, days, hours, minutes) => {
-  return function(dispatch, getState) {
+  return function (dispatch, getState) {
     const currentBucket = getCurrentBucket(getState())
     const currentPrefix = getCurrentPrefix(getState())
     const objectName = `${currentPrefix}${object}`
@@ -232,22 +232,22 @@ export const shareObject = (object, days, hours, minutes) => {
           host: location.host,
           bucket: currentBucket,
           object: objectName,
-          expiry: expiry
+          expiry: expiry,
         })
-        .then(obj => {
+        .then((obj) => {
           dispatch(showShareObject(object, obj.url))
           dispatch(
             alertActions.set({
               type: "success",
-              message: `Object shared. Expires in ${days} days ${hours} hours ${minutes} minutes`
+              message: `Object shared. Expires in ${days} days ${hours} hours ${minutes} minutes`,
             })
           )
         })
-        .catch(err => {
+        .catch((err) => {
           dispatch(
             alertActions.set({
               type: "danger",
-              message: err.message
+              message: err.message,
             })
           )
         })
@@ -265,7 +265,7 @@ export const shareObject = (object, days, hours, minutes) => {
       dispatch(
         alertActions.set({
           type: "success",
-          message: `Object shared.`
+          message: `Object shared.`,
         })
       )
     }
@@ -276,18 +276,17 @@ export const showShareObject = (object, url) => ({
   type: SET_SHARE_OBJECT,
   show: true,
   object,
-  url
+  url,
 })
 
 export const hideShareObject = (object, url) => ({
   type: SET_SHARE_OBJECT,
   show: false,
   object: "",
-  url: ""
+  url: "",
 })
-
-export const downloadObject = object => {
-  return function(dispatch, getState) {
+export const getObjectURL = (object, callback) => {
+  return function (dispatch, getState) {
     const currentBucket = getCurrentBucket(getState())
     const currentPrefix = getCurrentPrefix(getState())
     const objectName = `${currentPrefix}${object}`
@@ -295,52 +294,73 @@ export const downloadObject = object => {
     if (web.LoggedIn()) {
       return web
         .CreateURLToken()
-        .then(res => {
-          const url = `${
-            window.location.origin
-          }${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=${
-            res.token
-          }`
-          window.location = url
+        .then((res) => {
+          const url = `${window.location.origin}${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=${res.token}`
+          callback(url)
         })
-        .catch(err => {
+        .catch((err) => {
           dispatch(
             alertActions.set({
               type: "danger",
-              message: err.message
+              message: err.message,
             })
           )
         })
     } else {
-      const url = `${
-        window.location.origin
-      }${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=`
+      const url = `${window.location.origin}${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=`
+      callback(url)
+    }
+  }
+}
+export const downloadObject = (object) => {
+  return function (dispatch, getState) {
+    const currentBucket = getCurrentBucket(getState())
+    const currentPrefix = getCurrentPrefix(getState())
+    const objectName = `${currentPrefix}${object}`
+    const encObjectName = encodeURI(objectName)
+    if (web.LoggedIn()) {
+      return web
+        .CreateURLToken()
+        .then((res) => {
+          const url = `${window.location.origin}${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=${res.token}`
+          window.location = url
+        })
+        .catch((err) => {
+          dispatch(
+            alertActions.set({
+              type: "danger",
+              message: err.message,
+            })
+          )
+        })
+    } else {
+      const url = `${window.location.origin}${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=`
       window.location = url
     }
   }
 }
 
-export const checkObject = object => ({
+export const checkObject = (object) => ({
   type: CHECKED_LIST_ADD,
-  object
+  object,
 })
 
-export const uncheckObject = object => ({
+export const uncheckObject = (object) => ({
   type: CHECKED_LIST_REMOVE,
-  object
+  object,
 })
 
 export const resetCheckedList = () => ({
-  type: CHECKED_LIST_RESET
+  type: CHECKED_LIST_RESET,
 })
 
 export const downloadCheckedObjects = () => {
-  return function(dispatch, getState) {
+  return function (dispatch, getState) {
     const state = getState()
     const req = {
       bucketName: getCurrentBucket(state),
       prefix: getCurrentPrefix(state),
-      objects: getCheckedList(state)
+      objects: getCheckedList(state),
     }
     if (!web.LoggedIn()) {
       const requestUrl = location.origin + "/minio/zip?token="
@@ -348,17 +368,15 @@ export const downloadCheckedObjects = () => {
     } else {
       return web
         .CreateURLToken()
-        .then(res => {
-          const requestUrl = `${
-            location.origin
-          }${minioBrowserPrefix}/zip?token=${res.token}`
+        .then((res) => {
+          const requestUrl = `${location.origin}${minioBrowserPrefix}/zip?token=${res.token}`
           downloadZip(requestUrl, req, dispatch)
         })
-        .catch(err =>
+        .catch((err) =>
           dispatch(
             alertActions.set({
               type: "danger",
-              message: err.message
+              message: err.message,
             })
           )
         )
@@ -374,11 +392,11 @@ const downloadZip = (url, req, dispatch) => {
   xhr.open("POST", url, true)
   xhr.responseType = "blob"
 
-  xhr.onload = function(e) {
+  xhr.onload = function (e) {
     if (this.status == 200) {
       dispatch(resetCheckedList())
       var blob = new Blob([this.response], {
-        type: "octet/stream"
+        type: "octet/stream",
       })
       var blobUrl = window.URL.createObjectURL(blob)
       var separator = req.prefix.length > 1 ? "-" : ""

buildscripts/gateway-tests.sh

@@ -46,7 +46,10 @@ function main()
     gw_pid="$(start_minio_gateway_s3)"
 
     SERVER_ENDPOINT=127.0.0.1:24240 ENABLE_HTTPS=0 ACCESS_KEY=minio \
-                   SECRET_KEY=minio123 MINT_MODE="full" /mint/entrypoint.sh
+                   SECRET_KEY=minio123 MINT_MODE="full" /mint/entrypoint.sh \
+                   aws-sdk-go aws-sdk-java aws-sdk-php aws-sdk-ruby awscli \
+                   healthcheck mc minio-dotnet minio-js \
+                   minio-py s3cmd s3select security
     rv=$?
 
     kill "$sr_pid"

buildscripts/race.sh

@@ -3,5 +3,5 @@
 set -e
 
 for d in $(go list ./... | grep -v browser); do
-    CGO_ENABLED=1 go test -v -race --timeout 20m "$d"
+    CGO_ENABLED=1 go test -v -race --timeout 50m "$d"
 done

cmd/admin-handlers-config-kv.go

@@ -35,50 +35,45 @@ import (
 	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/auth"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
 )
 
-func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *http.Request) ObjectLayer {
+func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *http.Request) (auth.Credentials, ObjectLayer) {
 	// Get current object layer instance.
 	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return nil
+		return auth.Credentials{}, nil
 	}
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.ConfigUpdateAdminAction, "")
+	cred, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.ConfigUpdateAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
-		return nil
+		return cred, nil
 	}
 
-	return objectAPI
+	return cred, objectAPI
 }
 
-// DelConfigKVHandler - DELETE /minio/admin/v2/del-config-kv
+// DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
 func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DelConfigKVHandler")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
 		return
 	}
 
-	password := globalActiveCred.SecretKey
+	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		logger.LogIf(ctx, err, logger.Application)
@@ -103,28 +98,22 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	}
 }
 
-// SetConfigKVHandler - PUT /minio/admin/v2/set-config-kv
+// SetConfigKVHandler - PUT /minio/admin/v3/set-config-kv
 func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetConfigKVHandler")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
 		return
 	}
 
-	password := globalActiveCred.SecretKey
+	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		logger.LogIf(ctx, err, logger.Application)
@@ -162,17 +151,17 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 	// Make sure to write backend is encrypted
 	if globalConfigEncrypted {
-		saveConfig(context.Background(), objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
+		saveConfig(GlobalContext, objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
 	}
 
 	writeSuccessResponseHeadersOnly(w)
 }
 
-// GetConfigKVHandler - GET /minio/admin/v2/get-config-kv?key={key}
+// GetConfigKVHandler - GET /minio/admin/v3/get-config-kv?key={key}
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfigKVHandler")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
@@ -195,7 +184,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	password := globalActiveCred.SecretKey
+	password := cred.SecretKey
 	econfigData, err := madmin.EncryptData(password, buf.Bytes())
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -208,7 +197,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ClearConfigHistoryKVHandler")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
@@ -243,7 +232,7 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RestoreConfigHistoryKVHandler")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
@@ -289,7 +278,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListConfigHistoryKVHandler")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
@@ -313,7 +302,7 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	password := globalActiveCred.SecretKey
+	password := cred.SecretKey
 	econfigData, err := madmin.EncryptData(password, data)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -323,11 +312,11 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 	writeSuccessResponseJSON(w, econfigData)
 }
 
-// HelpConfigKVHandler - GET /minio/admin/v2/help-config-kv?subSys={subSys}&key={key}
+// HelpConfigKVHandler - GET /minio/admin/v3/help-config-kv?subSys={subSys}&key={key}
 func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HelpConfigKVHandler")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
@@ -349,28 +338,22 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
 	w.(http.Flusher).Flush()
 }
 
-// SetConfigHandler - PUT /minio/admin/v2/config
+// SetConfigHandler - PUT /minio/admin/v3/config
 func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetConfigHandler")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
 		return
 	}
 
-	password := globalActiveCred.SecretKey
+	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		logger.LogIf(ctx, err, logger.Application)
@@ -403,18 +386,18 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	// Make sure to write backend is encrypted
 	if globalConfigEncrypted {
-		saveConfig(context.Background(), objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
+		saveConfig(GlobalContext, objectAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
 	}
 
 	writeSuccessResponseHeadersOnly(w)
 }
 
-// GetConfigHandler - GET /minio/admin/v2/config
+// GetConfigHandler - GET /minio/admin/v3/config
 // Get config.json of this minio setup.
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfigHandler")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
@@ -471,7 +454,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 		}
 	}
 
-	password := globalActiveCred.SecretKey
+	password := cred.SecretKey
 	econfigData, err := madmin.EncryptData(password, []byte(s.String()))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)

cmd/admin-handlers-users.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -51,7 +51,7 @@ func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.R
 	return objectAPI, cred
 }
 
-// RemoveUser - DELETE /minio/admin/v2/remove-user?accessKey=<access_key>
+// RemoveUser - DELETE /minio/admin/v3/remove-user?accessKey=<access_key>
 func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveUser")
 
@@ -60,12 +60,6 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	vars := mux.Vars(r)
 	accessKey := vars["accessKey"]
 
@@ -93,7 +87,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// ListUsers - GET /minio/admin/v2/list-users
+// ListUsers - GET /minio/admin/v3/list-users
 func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListUsers")
 
@@ -125,7 +119,7 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseJSON(w, econfigData)
 }
 
-// GetUserInfo - GET /minio/admin/v2/user-info
+// GetUserInfo - GET /minio/admin/v3/user-info
 func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetUserInfo")
 
@@ -152,7 +146,7 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseJSON(w, data)
 }
 
-// UpdateGroupMembers - PUT /minio/admin/v2/update-group-members
+// UpdateGroupMembers - PUT /minio/admin/v3/update-group-members
 func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "UpdateGroupMembers")
 
@@ -195,7 +189,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 	}
 }
 
-// GetGroup - /minio/admin/v2/group?group=mygroup1
+// GetGroup - /minio/admin/v3/group?group=mygroup1
 func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetGroup")
 
@@ -222,7 +216,7 @@ func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseJSON(w, body)
 }
 
-// ListGroups - GET /minio/admin/v2/groups
+// ListGroups - GET /minio/admin/v3/groups
 func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListGroups")
 
@@ -246,7 +240,7 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseJSON(w, body)
 }
 
-// SetGroupStatus - PUT /minio/admin/v2/set-group-status?group=mygroup1&status=enabled
+// SetGroupStatus - PUT /minio/admin/v3/set-group-status?group=mygroup1&status=enabled
 func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetGroupStatus")
 
@@ -281,7 +275,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// SetUserStatus - PUT /minio/admin/v2/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
+// SetUserStatus - PUT /minio/admin/v3/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
 func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetUserStatus")
 
@@ -290,12 +284,6 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	vars := mux.Vars(r)
 	accessKey := vars["accessKey"]
 	status := vars["status"]
@@ -320,7 +308,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// AddUser - PUT /minio/admin/v2/add-user?accessKey=<access_key>
+// AddUser - PUT /minio/admin/v3/add-user?accessKey=<access_key>
 func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddUser")
 
@@ -329,12 +317,6 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	vars := mux.Vars(r)
 	accessKey := vars["accessKey"]
 
@@ -379,61 +361,55 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-// AddServiceAccount - PUT /minio/admin/v2/add-service-account?parentUser=<parent_user_accesskey>
+// AddServiceAccount - PUT /minio/admin/v3/add-service-account
 func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddServiceAccount")
 
-	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.CreateUserAdminAction)
-	if objectAPI == nil {
+	// Get current object layer instance.
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
-	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
-		// More than maxConfigSize bytes were available
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
+	cred, _, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
 	}
 
 	password := cred.SecretKey
-	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
 
 	var createReq madmin.AddServiceAccountReq
-	if err = json.Unmarshal(configBytes, &createReq); err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+	if err = json.Unmarshal(reqBytes, &createReq); err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
 
-	if createReq.Parent == "" {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
+	// Disallow creating service accounts by root user.
+	if owner {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminAccountNotEligible), r.URL)
 		return
 	}
 
-	// Disallow creating service accounts for the root user
-	if createReq.Parent == globalActiveCred.AccessKey {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddServiceAccountInvalidArgument), r.URL)
-		return
+	parentUser := cred.AccessKey
+	if cred.ParentUser != "" {
+		parentUser = cred.ParentUser
 	}
 
-	creds, err := globalIAMSys.NewServiceAccount(ctx, createReq.Parent, createReq.Policy)
+	newCred, err := globalIAMSys.NewServiceAccount(ctx, parentUser, createReq.Policy)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Notify all other Minio peers to reload user
-	for _, nerr := range globalNotificationSys.LoadUser(creds.AccessKey, false) {
+	// Notify all other Minio peers to reload user the service account
+	for _, nerr := range globalNotificationSys.LoadServiceAccount(newCred.AccessKey) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 			logger.LogIf(ctx, nerr.Err)
@@ -441,7 +417,10 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	}
 
 	var createResp = madmin.AddServiceAccountResp{
-		Credentials: creds,
+		Credentials: auth.Credentials{
+			AccessKey: newCred.AccessKey,
+			SecretKey: newCred.SecretKey,
+		},
 	}
 
 	data, err := json.Marshal(createResp)
@@ -450,60 +429,129 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	econfigData, err := madmin.EncryptData(password, data)
+	encryptedData, err := madmin.EncryptData(password, data)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	writeSuccessResponseJSON(w, econfigData)
+	writeSuccessResponseJSON(w, encryptedData)
 }
 
-// GetServiceAccount - GET /minio/admin/v2/get-service-account?accessKey=<access_key>
-func (a adminAPIHandlers) GetServiceAccount(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetServiceAccount")
+// ListServiceAccounts - GET /minio/admin/v3/list-service-accounts
+func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListServiceAccounts")
 
-	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.GetUserAdminAction)
-	if objectAPI == nil {
+	// Get current object layer instance.
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 
-	vars := mux.Vars(r)
-	accessKey := vars["accessKey"]
+	cred, _, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
 
-	password := cred.SecretKey
+	// Disallow creating service accounts by root user.
+	if owner {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminAccountNotEligible), r.URL)
+		return
+	}
 
-	creds, err := globalIAMSys.GetServiceAccount(ctx, accessKey)
+	serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, cred.AccessKey)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	data, err := json.Marshal(creds)
+	var listResp = madmin.ListServiceAccountsResp{
+		Accounts: serviceAccounts,
+	}
+
+	data, err := json.Marshal(listResp)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	econfigData, err := madmin.EncryptData(password, data)
+	encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	writeSuccessResponseJSON(w, econfigData)
+	writeSuccessResponseJSON(w, encryptedData)
 }
 
-// InfoCannedPolicy - GET /minio/admin/v2/info-canned-policy?name={policyName}
-func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "InfoCannedPolicy")
+// DeleteServiceAccount - DELETE /minio/admin/v3/delete-service-account
+func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteServiceAccount")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	cred, _, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	// Disallow creating service accounts by root user.
+	if owner {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminAccountNotEligible), r.URL)
+		return
+	}
+
+	serviceAccount := mux.Vars(r)["accessKey"]
+	if serviceAccount == "" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
+		return
+	}
+
+	user, err := globalIAMSys.GetServiceAccountParent(ctx, serviceAccount)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if cred.AccessKey != user || cred.ParentUser != user {
+		// The service account belongs to another user but return not found error to mitigate brute force attacks.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServiceAccountNotFound), r.URL)
+		return
+	}
+
+	err = globalIAMSys.DeleteServiceAccount(ctx, serviceAccount)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessNoContent(w)
+}
+
+// InfoCannedPolicyV2 - GET /minio/admin/v2/info-canned-policy?name={policyName}
+func (a adminAPIHandlers) InfoCannedPolicyV2(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "InfoCannedPolicyV2")
 
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
-	data, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
+	policy, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(policy)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -513,7 +561,58 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 	w.(http.Flusher).Flush()
 }
 
-// ListCannedPolicies - GET /minio/admin/v2/list-canned-policies
+// InfoCannedPolicy - GET /minio/admin/v3/info-canned-policy?name={policyName}
+func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "InfoCannedPolicy")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	policy, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	json.NewEncoder(w).Encode(policy)
+	w.(http.Flusher).Flush()
+}
+
+// ListCannedPoliciesV2 - GET /minio/admin/v2/list-canned-policies
+func (a adminAPIHandlers) ListCannedPoliciesV2(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListCannedPoliciesV2")
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	policies, err := globalIAMSys.ListPolicies()
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	policyMap := make(map[string][]byte, len(policies))
+	for k, p := range policies {
+		var err error
+		policyMap[k], err = json.Marshal(p)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+	}
+	if err = json.NewEncoder(w).Encode(policyMap); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// ListCannedPolicies - GET /minio/admin/v3/list-canned-policies
 func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListCannedPolicies")
 
@@ -528,7 +627,16 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	if err = json.NewEncoder(w).Encode(policies); err != nil {
+	var newPolicies = make(map[string]iampolicy.Policy)
+	for name, p := range policies {
+		_, err = json.Marshal(p)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		newPolicies[name] = p
+	}
+	if err = json.NewEncoder(w).Encode(newPolicies); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -536,7 +644,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 	w.(http.Flusher).Flush()
 }
 
-// RemoveCannedPolicy - DELETE /minio/admin/v2/remove-canned-policy?name=<policy_name>
+// RemoveCannedPolicy - DELETE /minio/admin/v3/remove-canned-policy?name=<policy_name>
 func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveCannedPolicy")
 
@@ -548,12 +656,6 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 	vars := mux.Vars(r)
 	policyName := vars["name"]
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	if err := globalIAMSys.DeletePolicy(policyName); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -568,7 +670,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 	}
 }
 
-// AddCannedPolicy - PUT /minio/admin/v2/add-canned-policy?name=<policy_name>
+// AddCannedPolicy - PUT /minio/admin/v3/add-canned-policy?name=<policy_name>
 func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddCannedPolicy")
 
@@ -580,12 +682,6 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 	vars := mux.Vars(r)
 	policyName := vars["name"]
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	// Error out if Content-Length is missing.
 	if r.ContentLength <= 0 {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
@@ -624,7 +720,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 	}
 }
 
-// SetPolicyForUserOrGroup - PUT /minio/admin/v2/set-policy?policy=xxx&user-or-group=?[&is-group]
+// SetPolicyForUserOrGroup - PUT /minio/admin/v3/set-policy?policy=xxx&user-or-group=?[&is-group]
 func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetPolicyForUserOrGroup")
 
@@ -638,15 +734,9 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 	entityName := vars["userOrGroup"]
 	isGroup := vars["isGroup"] == "true"
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	if !isGroup {
 		ok, err := globalIAMSys.IsTempUser(entityName)
-		if err != nil {
+		if err != nil && err != errNoSuchUser {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}

cmd/admin-handlers.go

@@ -21,7 +21,6 @@ import (
 	"crypto/subtle"
 	"crypto/tls"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -44,12 +43,9 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/message/log"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/cpu"
-	"github.com/minio/minio/pkg/event/target"
 	"github.com/minio/minio/pkg/handlers"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
-	"github.com/minio/minio/pkg/mem"
 	xnet "github.com/minio/minio/pkg/net"
 	trace "github.com/minio/minio/pkg/trace"
 )
@@ -99,7 +95,7 @@ func updateServer(updateURL, sha256Hex string, latestReleaseTime time.Time) (us
 	return us, nil
 }
 
-// ServerUpdateHandler - POST /minio/admin/v2/update?updateURL={updateURL}
+// ServerUpdateHandler - POST /minio/admin/v3/update?updateURL={updateURL}
 // ----------
 // updates all minio servers and restarts them gracefully.
 func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
@@ -178,7 +174,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 	}
 }
 
-// ServiceActionHandler - POST /minio/admin/v2/service?action={action}
+// ServiceActionHandler - POST /minio/admin/v3/service?action={action}
 // ----------
 // restarts/stops minio server gracefully. In a distributed setup,
 func (a adminAPIHandlers) ServiceActionHandler(w http.ResponseWriter, r *http.Request) {
@@ -267,7 +263,7 @@ type ServerInfo struct {
 	Data  *ServerInfoData `json:"data"`
 }
 
-// StorageInfoHandler - GET /minio/admin/v2/storageinfo
+// StorageInfoHandler - GET /minio/admin/v3/storageinfo
 // ----------
 // Get server information
 func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
@@ -292,7 +288,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 
 }
 
-// DataUsageInfoHandler - GET /minio/admin/v2/datausage
+// DataUsageInfoHandler - GET /minio/admin/v3/datausage
 // ----------
 // Get server/cluster data usage info
 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
@@ -378,155 +374,6 @@ func (a adminAPIHandlers) AccountingUsageInfoHandler(w http.ResponseWriter, r *h
 	writeSuccessResponseJSON(w, usageInfoJSON)
 }
 
-// ServerCPULoadInfo holds informantion about cpu utilization
-// of one minio node. It also reports any errors if encountered
-// while trying to reach this server.
-type ServerCPULoadInfo struct {
-	Addr         string     `json:"addr"`
-	Error        string     `json:"error,omitempty"`
-	Load         []cpu.Load `json:"load"`
-	HistoricLoad []cpu.Load `json:"historicLoad"`
-}
-
-// ServerMemUsageInfo holds informantion about memory utilization
-// of one minio node. It also reports any errors if encountered
-// while trying to reach this server.
-type ServerMemUsageInfo struct {
-	Addr          string      `json:"addr"`
-	Error         string      `json:"error,omitempty"`
-	Usage         []mem.Usage `json:"usage"`
-	HistoricUsage []mem.Usage `json:"historicUsage"`
-}
-
-// ServerNetReadPerfInfo network read performance information.
-type ServerNetReadPerfInfo struct {
-	Addr           string `json:"addr"`
-	ReadThroughput uint64 `json:"readThroughput"`
-	Error          string `json:"error,omitempty"`
-}
-
-// PerfInfoHandler - GET /minio/admin/v2/performance?perfType={perfType}
-// ----------
-// Get all performance information based on input type
-// Supported types = drive
-func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "PerfInfo")
-
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.PerfInfoAdminAction)
-	if objectAPI == nil {
-		return
-	}
-
-	vars := mux.Vars(r)
-	switch perfType := vars["perfType"]; perfType {
-	case "net":
-		var size int64 = defaultNetPerfSize
-		if sizeStr, found := vars["size"]; found {
-			var err error
-			if size, err = strconv.ParseInt(sizeStr, 10, 64); err != nil || size < 0 {
-				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
-				return
-			}
-		}
-
-		if !globalIsDistXL {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-			return
-		}
-
-		addr := r.Host
-		if globalIsDistXL {
-			addr = GetLocalPeer(globalEndpoints)
-		}
-
-		infos := map[string][]ServerNetReadPerfInfo{}
-		infos[addr] = globalNotificationSys.NetReadPerfInfo(size)
-		for peer, info := range globalNotificationSys.CollectNetPerfInfo(size) {
-			infos[peer] = info
-		}
-
-		// Marshal API response
-		jsonBytes, err := json.Marshal(infos)
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-
-		// Reply with performance information (across nodes in a
-		// distributed setup) as json.
-		writeSuccessResponseJSON(w, jsonBytes)
-
-	case "drive":
-		// Drive Perf is only implemented for Erasure coded backends
-		if !globalIsXL {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-			return
-		}
-
-		var size int64 = madmin.DefaultDrivePerfSize
-		if sizeStr, found := vars["size"]; found {
-			var err error
-			if size, err = strconv.ParseInt(sizeStr, 10, 64); err != nil || size <= 0 {
-				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
-				return
-			}
-		}
-		// Get drive performance details from local server's drive(s)
-		dp := getLocalDrivesPerf(globalEndpoints, size, r)
-
-		// Notify all other MinIO peers to report drive performance numbers
-		dps := globalNotificationSys.DrivePerfInfo(size)
-		dps = append(dps, dp)
-
-		// Marshal API response
-		jsonBytes, err := json.Marshal(dps)
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-
-		// Reply with performance information (across nodes in a
-		// distributed setup) as json.
-		writeSuccessResponseJSON(w, jsonBytes)
-	case "cpu":
-		// Get CPU load details from local server's cpu(s)
-		cpu := getLocalCPULoad(globalEndpoints, r)
-		// Notify all other MinIO peers to report cpu load numbers
-		cpus := globalNotificationSys.CPULoadInfo()
-		cpus = append(cpus, cpu)
-
-		// Marshal API response
-		jsonBytes, err := json.Marshal(cpus)
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-
-		// Reply with cpu load information (across nodes in a
-		// distributed setup) as json.
-		writeSuccessResponseJSON(w, jsonBytes)
-	case "mem":
-		// Get mem usage details from local server(s)
-		m := getLocalMemUsage(globalEndpoints, r)
-		// Notify all other MinIO peers to report mem usage numbers
-		mems := globalNotificationSys.MemUsageInfo()
-		mems = append(mems, m)
-
-		// Marshal API response
-		jsonBytes, err := json.Marshal(mems)
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-
-		// Reply with mem usage information (across nodes in a
-		// distributed setup) as json.
-		writeSuccessResponseJSON(w, jsonBytes)
-	default:
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-	}
-}
-
 func newLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
 	entry := &madmin.LockEntry{
 		Timestamp:  l.Timestamp,
@@ -622,7 +469,7 @@ type StartProfilingResult struct {
 	Error    string `json:"error"`
 }
 
-// StartProfilingHandler - POST /minio/admin/v2/profiling/start?profilerType={profilerType}
+// StartProfilingHandler - POST /minio/admin/v3/profiling/start?profilerType={profilerType}
 // ----------
 // Enable server profiling
 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
@@ -718,7 +565,7 @@ func (f dummyFileInfo) ModTime() time.Time { return f.modTime }
 func (f dummyFileInfo) IsDir() bool        { return f.isDir }
 func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 
-// DownloadProfilingHandler - POST /minio/admin/v2/profiling/download
+// DownloadProfilingHandler - POST /minio/admin/v3/profiling/download
 // ----------
 // Download profiling information of all nodes in a zip format
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
@@ -790,7 +637,7 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 	if hip.clientToken == "" {
 		jerr := json.NewDecoder(r).Decode(&hip.hs)
 		if jerr != nil {
-			logger.LogIf(context.Background(), jerr, logger.Application)
+			logger.LogIf(GlobalContext, jerr, logger.Application)
 			err = ErrRequestBodyParse
 			return
 		}
@@ -800,7 +647,7 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 	return
 }
 
-// HealHandler - POST /minio/admin/v2/heal/
+// HealHandler - POST /minio/admin/v3/heal/
 // -----------
 // Start heal processing and return heal status items.
 //
@@ -1093,7 +940,7 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 				HTTPStatusCode: http.StatusNotFound,
 			}
 		} else {
-			apiErr = errorCodes.ToAPIErr(toAdminAPIErrCode(ctx, err))
+			apiErr = errorCodes.ToAPIErrWithErr(toAdminAPIErrCode(ctx, err), err)
 		}
 	}
 	return apiErr
@@ -1116,7 +963,7 @@ func mustTrace(entry interface{}, trcAll, errOnly bool) bool {
 	return trace
 }
 
-// TraceHandler - POST /minio/admin/v2/trace
+// TraceHandler - POST /minio/admin/v3/trace
 // ----------
 // The handler sends http trace to the connected HTTP client.
 func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
@@ -1133,16 +980,13 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set(xhttp.ContentType, "text/event-stream")
 
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
 	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
 	traceCh := make(chan interface{}, 4000)
 
-	peers := getRestClients(globalEndpoints)
+	peers := newPeerRestClients(globalEndpoints)
 
-	globalHTTPTrace.Subscribe(traceCh, doneCh, func(entry interface{}) bool {
+	globalHTTPTrace.Subscribe(traceCh, ctx.Done(), func(entry interface{}) bool {
 		return mustTrace(entry, trcAll, trcErr)
 	})
 
@@ -1150,7 +994,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		if peer == nil {
 			continue
 		}
-		peer.Trace(traceCh, doneCh, trcAll, trcErr)
+		peer.Trace(traceCh, ctx.Done(), trcAll, trcErr)
 	}
 
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
@@ -1169,7 +1013,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			w.(http.Flusher).Flush()
-		case <-GlobalServiceDoneCh:
+		case <-ctx.Done():
 			return
 		}
 	}
@@ -1203,20 +1047,18 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 	w.Header().Add("Connection", "close")
 	w.Header().Set(xhttp.ContentType, "text/event-stream")
 
-	doneCh := make(chan struct{})
-	defer close(doneCh)
 	logCh := make(chan interface{}, 4000)
 
-	peers := getRestClients(globalEndpoints)
+	peers := newPeerRestClients(globalEndpoints)
 
-	globalConsoleSys.Subscribe(logCh, doneCh, node, limitLines, logKind, nil)
+	globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil)
 
 	for _, peer := range peers {
 		if peer == nil {
 			continue
 		}
 		if node == "" || strings.EqualFold(peer.host.Name, node) {
-			peer.ConsoleLog(logCh, doneCh)
+			peer.ConsoleLog(logCh, ctx.Done())
 		}
 	}
 
@@ -1240,13 +1082,13 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 				return
 			}
 			w.(http.Flusher).Flush()
-		case <-GlobalServiceDoneCh:
+		case <-ctx.Done():
 			return
 		}
 	}
 }
 
-// KMSKeyStatusHandler - GET /minio/admin/v2/kms/key/status?key-id=<master-key-id>
+// KMSKeyStatusHandler - GET /minio/admin/v3/kms/key/status?key-id=<master-key-id>
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSKeyStatusHandler")
 
@@ -1315,64 +1157,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseJSON(w, resp)
 }
 
-// ServerHardwareInfoHandler - GET /minio/admin/v2/hardwareinfo?Type={hwType}
-// ----------
-// Get all hardware information based on input type
-// Supported types = cpu
-func (a adminAPIHandlers) ServerHardwareInfoHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "HardwareInfo")
-
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerHardwareInfoAdminAction)
-	if objectAPI == nil {
-		return
-	}
-
-	vars := mux.Vars(r)
-	hardware := vars[madmin.HARDWARE]
-
-	switch madmin.HardwareType(hardware) {
-	case madmin.CPU:
-		// Get CPU hardware details from local server's cpu(s)
-		cpu := getLocalCPUInfo(globalEndpoints, r)
-		// Notify all other MinIO peers to report cpu hardware
-		cpus := globalNotificationSys.CPUInfo()
-		cpus = append(cpus, cpu)
-
-		// Marshal API response
-		jsonBytes, err := json.Marshal(cpus)
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-
-		// Reply with cpu hardware information (across nodes in a
-		// distributed setup) as json.
-		writeSuccessResponseJSON(w, jsonBytes)
-
-	case madmin.NETWORK:
-		// Get Network hardware details from local server's network(s)
-		network := getLocalNetworkInfo(globalEndpoints, r)
-		// Notify all other MinIO peers to report network hardware
-		networks := globalNotificationSys.NetworkInfo()
-		networks = append(networks, network)
-
-		// Marshal API response
-		jsonBytes, err := json.Marshal(networks)
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-
-		// Reply with cpu network information (across nodes in a
-		// distributed setup) as json.
-		writeSuccessResponseJSON(w, jsonBytes)
-
-	default:
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
-	}
-}
-
-// OBDInfoHandler - GET /minio/admin/v2/obdinfo
+// OBDInfoHandler - GET /minio/admin/v3/obdinfo
 // ----------
 // Get server on-board diagnostics
 func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) {
@@ -1381,127 +1166,160 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	if objectAPI == nil {
 		return
 	}
-	deadlinedCtx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Minute))
-	obdInfo := madmin.OBDInfo{}
 
-	setCommonHeaders(w)
-	w.Header().Set(xhttp.ContentType, string(mimeJSON))
-	w.WriteHeader(http.StatusOK)
+	vars := mux.Vars(r)
+	obdInfo := madmin.OBDInfo{}
+	obdInfoCh := make(chan madmin.OBDInfo)
 
 	enc := json.NewEncoder(w)
-	partialWrite := func() {
-		logger.LogIf(ctx, enc.Encode(obdInfo))
+	partialWrite := func(oinfo madmin.OBDInfo) {
+		obdInfoCh <- oinfo
 	}
 
-	finish := func() {
-		partialWrite()
-		w.(http.Flusher).Flush()
-		cancel()
-	}
+	setCommonHeaders(w)
+	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	w.WriteHeader(http.StatusOK)
 
 	errResp := func(err error) {
 		errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(),
 			w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
 		encodedErrorResponse := encodeResponse(errorResponse)
 		obdInfo.Error = string(encodedErrorResponse)
-		finish()
+		logger.LogIf(ctx, enc.Encode(obdInfo))
 	}
 
+	deadline := 3600 * time.Second
+	if dstr := r.URL.Query().Get("deadline"); dstr != "" {
+		var err error
+		deadline, err = time.ParseDuration(dstr)
+		if err != nil {
+			errResp(err)
+			return
+		}
+	}
+
+	deadlinedCtx, cancel := context.WithTimeout(ctx, deadline)
+
+	defer cancel()
+
 	nsLock := objectAPI.NewNSLock(deadlinedCtx, minioMetaBucket, "obd-in-progress")
-	if err := nsLock.GetLock(newDynamicTimeout(10*time.Minute, 600*time.Second)); err != nil { // returns a locked lock
+	if err := nsLock.GetLock(newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
 		errResp(err)
 		return
 	}
 	defer nsLock.Unlock()
 
-	vars := mux.Vars(r)
+	go func() {
+		defer close(obdInfoCh)
 
-	if cpu, ok := vars["syscpu"]; ok && cpu == "true" {
-		cpuInfo := getLocalCPUOBDInfo(deadlinedCtx)
+		if cpu, ok := vars["syscpu"]; ok && cpu == "true" {
+			cpuInfo := getLocalCPUOBDInfo(deadlinedCtx)
 
-		obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo)
-		obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, globalNotificationSys.CPUOBDInfo(deadlinedCtx)...)
-		partialWrite()
-	}
-
-	if diskHw, ok := vars["sysdiskhw"]; ok && diskHw == "true" {
-		diskHwInfo := getLocalDiskHwOBD(deadlinedCtx)
-
-		obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo)
-		obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, globalNotificationSys.DiskHwOBDInfo(deadlinedCtx)...)
-		partialWrite()
-	}
-
-	if osInfo, ok := vars["sysosinfo"]; ok && osInfo == "true" {
-		osInfo := getLocalOsInfoOBD(deadlinedCtx)
-
-		obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo)
-		obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, globalNotificationSys.OsOBDInfo(deadlinedCtx)...)
-		partialWrite()
-	}
-
-	if mem, ok := vars["sysmem"]; ok && mem == "true" {
-		memInfo := getLocalMemOBD(deadlinedCtx)
-
-		obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo)
-		obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, globalNotificationSys.MemOBDInfo(deadlinedCtx)...)
-		partialWrite()
-	}
-
-	if proc, ok := vars["sysprocess"]; ok && proc == "true" {
-		procInfo := getLocalProcOBD(deadlinedCtx)
-
-		obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo)
-		obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, globalNotificationSys.ProcOBDInfo(deadlinedCtx)...)
-		partialWrite()
-	}
-
-	if config, ok := vars["minioconfig"]; ok && config == "true" {
-		cfg, err := readServerConfig(ctx, objectAPI)
-		logger.LogIf(ctx, err)
-		obdInfo.Minio.Config = cfg
-		partialWrite()
-	}
-
-	if drive, ok := vars["perfdrive"]; ok && drive == "true" {
-		// Get drive obd details from local server's drive(s)
-		driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r)
-		driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r)
-
-		errStr := ""
-		if driveOBDSerial.Error != "" {
-			errStr = "serial: " + driveOBDSerial.Error
-		}
-		if driveOBDParallel.Error != "" {
-			errStr = errStr + " parallel: " + driveOBDParallel.Error
+			obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo)
+			obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, globalNotificationSys.CPUOBDInfo(deadlinedCtx)...)
+			partialWrite(obdInfo)
 		}
 
-		driveOBD := madmin.ServerDrivesOBDInfo{
-			Addr:     driveOBDSerial.Addr,
-			Serial:   driveOBDSerial.Serial,
-			Parallel: driveOBDParallel.Parallel,
-			Error:    errStr,
+		if diskHw, ok := vars["sysdiskhw"]; ok && diskHw == "true" {
+			diskHwInfo := getLocalDiskHwOBD(deadlinedCtx)
+
+			obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo)
+			obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, globalNotificationSys.DiskHwOBDInfo(deadlinedCtx)...)
+			partialWrite(obdInfo)
 		}
-		obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBD)
 
-		// Notify all other MinIO peers to report drive obd numbers
-		driveOBDs := globalNotificationSys.DriveOBDInfo(deadlinedCtx)
-		obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBDs...)
+		if osInfo, ok := vars["sysosinfo"]; ok && osInfo == "true" {
+			osInfo := getLocalOsInfoOBD(deadlinedCtx)
 
-		partialWrite()
+			obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo)
+			obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, globalNotificationSys.OsOBDInfo(deadlinedCtx)...)
+			partialWrite(obdInfo)
+		}
+
+		if mem, ok := vars["sysmem"]; ok && mem == "true" {
+			memInfo := getLocalMemOBD(deadlinedCtx)
+
+			obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo)
+			obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, globalNotificationSys.MemOBDInfo(deadlinedCtx)...)
+			partialWrite(obdInfo)
+		}
+
+		if proc, ok := vars["sysprocess"]; ok && proc == "true" {
+			procInfo := getLocalProcOBD(deadlinedCtx)
+
+			obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo)
+			obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, globalNotificationSys.ProcOBDInfo(deadlinedCtx)...)
+			partialWrite(obdInfo)
+		}
+
+		if config, ok := vars["minioconfig"]; ok && config == "true" {
+			cfg, err := readServerConfig(ctx, objectAPI)
+			logger.LogIf(ctx, err)
+			obdInfo.Minio.Config = cfg
+			partialWrite(obdInfo)
+		}
+
+		if drive, ok := vars["perfdrive"]; ok && drive == "true" {
+			// Get drive obd details from local server's drive(s)
+			driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r)
+			driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r)
+
+			errStr := ""
+			if driveOBDSerial.Error != "" {
+				errStr = "serial: " + driveOBDSerial.Error
+			}
+			if driveOBDParallel.Error != "" {
+				errStr = errStr + " parallel: " + driveOBDParallel.Error
+			}
+
+			driveOBD := madmin.ServerDrivesOBDInfo{
+				Addr:     driveOBDSerial.Addr,
+				Serial:   driveOBDSerial.Serial,
+				Parallel: driveOBDParallel.Parallel,
+				Error:    errStr,
+			}
+			obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBD)
+
+			// Notify all other MinIO peers to report drive obd numbers
+			driveOBDs := globalNotificationSys.DriveOBDInfo(deadlinedCtx)
+			obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBDs...)
+
+			partialWrite(obdInfo)
+		}
+
+		if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistXL {
+			obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx))
+			obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.DispatchNetOBDInfo(deadlinedCtx)...)
+			obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx)
+			partialWrite(obdInfo)
+		}
+	}()
+
+	ticker := time.NewTicker(30 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case oinfo, ok := <-obdInfoCh:
+			if !ok {
+				return
+			}
+			logger.LogIf(ctx, enc.Encode(oinfo))
+			w.(http.Flusher).Flush()
+		case <-ticker.C:
+			if _, err := w.Write([]byte(" ")); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-deadlinedCtx.Done():
+			w.(http.Flusher).Flush()
+			return
+		}
 	}
 
-	if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistXL {
-		obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx))
-		obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.DispatchNetOBDInfo(deadlinedCtx)...)
-		obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx)
-		partialWrite()
-	}
-
-	finish()
 }
 
-// ServerInfoHandler - GET /minio/admin/v2/info
+// ServerInfoHandler - GET /minio/admin/v3/info
 // ----------
 // Get server information
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
@@ -1637,7 +1455,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		Mode:         mode,
 		Domain:       domain,
 		Region:       globalServerRegion,
-		SQSARN:       globalNotificationSys.GetARNList(),
+		SQSARN:       globalNotificationSys.GetARNList(false),
 		DeploymentID: globalDeploymentID,
 		Buckets:      buckets,
 		Objects:      objects,
@@ -1661,11 +1479,15 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 
 func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
 
-	// Fetch the targets
-	targetList, err := notify.RegisterNotificationTargets(cfg, GlobalServiceDoneCh, NewGatewayHTTPTransport(), nil, true)
-	if err != nil {
+	// Fetch the configured targets
+	tr := NewGatewayHTTPTransport()
+	defer tr.CloseIdleConnections()
+	targetList, err := notify.FetchRegisteredTargets(cfg, GlobalContext.Done(), tr, true, false)
+	if err != nil && err != notify.ErrTargetsOffline {
+		logger.LogIf(GlobalContext, err)
 		return nil
 	}
+
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
 
 	for targetID, target := range targetList.TargetMap() {
@@ -1710,7 +1532,6 @@ func fetchVaultStatus(cfg config.Config) madmin.Vault {
 	}
 
 	if err := checkConnection(kmsInfo.Endpoint, 15*time.Second); err != nil {
-
 		vault.Status = "offline"
 	} else {
 		vault.Status = "online"
@@ -1778,25 +1599,23 @@ func fetchLoggerInfo(cfg config.Config) ([]madmin.Logger, []madmin.Audit) {
 
 // checkConnection - ping an endpoint , return err in case of no connection
 func checkConnection(endpointStr string, timeout time.Duration) error {
-	u, pErr := xnet.ParseURL(endpointStr)
-	if pErr != nil {
-		return pErr
+	tr := newCustomHTTPTransport(&tls.Config{RootCAs: globalRootCAs}, timeout)()
+	defer tr.CloseIdleConnections()
+
+	ctx, cancel := context.WithTimeout(GlobalContext, timeout)
+	defer cancel()
+
+	req, err := http.NewRequest(http.MethodHead, endpointStr, nil)
+	if err != nil {
+		return err
 	}
 
-	tr := newCustomHTTPTransport(
-		&tls.Config{RootCAs: globalRootCAs},
-		timeout,
-		0, /* Default value */
-	)()
-
-	if dErr := u.DialHTTP(tr); dErr != nil {
-		if urlErr, ok := dErr.(*url.Error); ok {
-			// To treat "connection refused" errors as un reachable endpoint.
-			if target.IsConnRefusedErr(urlErr.Err) {
-				return errors.New("endpoint unreachable, please check your endpoint")
-			}
-		}
-		return dErr
+	client := &http.Client{Transport: tr}
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		return err
 	}
+	defer xhttp.DrainBody(resp.Body)
+	resp.Body.Close()
 	return nil
 }

cmd/admin-handlers_test.go

@@ -43,7 +43,8 @@ type adminXLTestBed struct {
 
 // prepareAdminXLTestBed - helper function that setups a single-node
 // XL backend for admin-handler tests.
-func prepareAdminXLTestBed() (*adminXLTestBed, error) {
+func prepareAdminXLTestBed(ctx context.Context) (*adminXLTestBed, error) {
+
 	// reset global variables to start afresh.
 	resetTestGlobals()
 
@@ -52,7 +53,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	globalIsXL = true
 
 	// Initializing objectLayer for HealFormatHandler.
-	objLayer, xlDirs, xlErr := initTestXLObjLayer()
+	objLayer, xlDirs, xlErr := initTestXLObjLayer(ctx)
 	if xlErr != nil {
 		return nil, xlErr
 	}
@@ -70,9 +71,9 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	globalConfigSys = NewConfigSys()
 
 	globalIAMSys = NewIAMSys()
-	globalIAMSys.Init(objLayer)
+	globalIAMSys.Init(ctx, objLayer)
 
-	buckets, err := objLayer.ListBuckets(context.Background())
+	buckets, err := objLayer.ListBuckets(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -103,7 +104,7 @@ func (atb *adminXLTestBed) TearDown() {
 
 // initTestObjLayer - Helper function to initialize an XL-based object
 // layer and set globalObjectAPI.
-func initTestXLObjLayer() (ObjectLayer, []string, error) {
+func initTestXLObjLayer(ctx context.Context) (ObjectLayer, []string, error) {
 	xlDirs, err := getRandomDisks(16)
 	if err != nil {
 		return nil, nil, err
@@ -116,7 +117,7 @@ func initTestXLObjLayer() (ObjectLayer, []string, error) {
 	}
 
 	globalPolicySys = NewPolicySys()
-	objLayer, err := newXLSets(endpoints, storageDisks, format, 1, 16)
+	objLayer, err := newXLSets(ctx, endpoints, storageDisks, format)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -191,7 +192,10 @@ func getServiceCmdRequest(cmd cmdType, cred auth.Credentials) (*http.Request, er
 // testServicesCmdHandler - parametrizes service subcommand tests on
 // cmdType value.
 func testServicesCmdHandler(cmd cmdType, t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	adminTestBed, err := prepareAdminXLTestBed(ctx)
 	if err != nil {
 		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
 	}
@@ -259,7 +263,10 @@ func buildAdminRequest(queryVal url.Values, method, path string,
 }
 
 func TestAdminServerInfo(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	adminTestBed, err := prepareAdminXLTestBed(ctx)
 	if err != nil {
 		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
 	}
@@ -314,12 +321,12 @@ func TestToAdminAPIErrCode(t *testing.T) {
 		// 3. Non-admin API specific error.
 		{
 			err:            errDiskNotFound,
-			expectedAPIErr: toAPIErrorCode(context.Background(), errDiskNotFound),
+			expectedAPIErr: toAPIErrorCode(GlobalContext, errDiskNotFound),
 		},
 	}
 
 	for i, test := range testCases {
-		actualErr := toAdminAPIErrCode(context.Background(), test.err)
+		actualErr := toAdminAPIErrCode(GlobalContext, test.err)
 		if actualErr != test.expectedAPIErr {
 			t.Errorf("Test %d: Expected %v but received %v",
 				i+1, test.expectedAPIErr, actualErr)

cmd/admin-heal-ops.go

@@ -183,7 +183,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
 	}
 
 	b, err := json.Marshal(&hsp)
-	return b, toAdminAPIErr(context.Background(), err)
+	return b, toAdminAPIErr(GlobalContext, err)
 }
 
 // LaunchNewHealSequence - launches a background routine that performs

cmd/admin-router.go

@@ -24,9 +24,11 @@ import (
 )
 
 const (
-	adminPathPrefix       = minioReservedBucketPath + "/admin"
-	adminAPIVersion       = madmin.AdminAPIVersion
-	adminAPIVersionPrefix = SlashSeparator + madmin.AdminAPIVersion
+	adminPathPrefix         = minioReservedBucketPath + "/admin"
+	adminAPIVersionV2       = madmin.AdminAPIVersionV2
+	adminAPIVersion         = madmin.AdminAPIVersion
+	adminAPIVersionPrefix   = SlashSeparator + adminAPIVersion
+	adminAPIVersionV2Prefix = SlashSeparator + adminAPIVersionV2
 )
 
 // adminAPIHandlers provides HTTP handlers for MinIO admin API.
@@ -41,135 +43,163 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 	/// Service operations
 
-	// Restart and stop MinIO service.
-	adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix+"/service").HandlerFunc(httpTraceAll(adminAPI.ServiceActionHandler)).Queries("action", "{action:.*}")
-	// Update MinIO servers.
-	adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix+"/update").HandlerFunc(httpTraceAll(adminAPI.ServerUpdateHandler)).Queries("updateURL", "{updateURL:.*}")
-
-	// Info operations
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
-	// Harware Info operations
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/hardware").HandlerFunc(httpTraceAll(adminAPI.ServerHardwareInfoHandler)).Queries("hwType", "{hwType:.*}")
-
-	// StorageInfo operations
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/storageinfo").HandlerFunc(httpTraceAll(adminAPI.StorageInfoHandler))
-	// DataUsageInfo operations
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/datausageinfo").HandlerFunc(httpTraceAll(adminAPI.DataUsageInfoHandler))
-
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/accountingusageinfo").HandlerFunc(httpTraceAll(adminAPI.AccountingUsageInfoHandler))
-
-	if globalIsDistXL || globalIsXL {
-		/// Heal operations
-
-		// Heal processing endpoint.
-		adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix + "/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-		adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix + "/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-		adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix + "/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-
-		adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix + "/background-heal/status").HandlerFunc(httpTraceAll(adminAPI.BackgroundHealStatusHandler))
-
-		/// Health operations
-
-	}
-	// Performance command - return performance details based on input type
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}")
-
-	// Profiling operations
-	adminRouter.Methods(http.MethodPost).Path(adminAPIVersionPrefix+"/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
-		Queries("profilerType", "{profilerType:.*}")
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
-
-	// Config KV operations.
-	if enableConfigOps {
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/get-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKVHandler)).Queries("key", "{key:.*}")
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/set-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKVHandler))
-		adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix + "/del-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.DelConfigKVHandler))
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/help-config-kv").HandlerFunc(httpTraceAll(adminAPI.HelpConfigKVHandler)).Queries("subSys", "{subSys:.*}", "key", "{key:.*}")
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/list-config-history-kv").HandlerFunc(httpTraceAll(adminAPI.ListConfigHistoryKVHandler)).Queries("count", "{count:[0-9]+}")
-		adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix+"/clear-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.ClearConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/restore-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.RestoreConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
+	adminVersions := []string{
+		adminAPIVersionPrefix,
+		adminAPIVersionV2Prefix,
 	}
 
-	/// Config operations
-	if enableConfigOps {
-		// Get config
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
-		// Set config
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
-	}
+	for _, adminVersion := range adminVersions {
+		// Restart and stop MinIO service.
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(httpTraceAll(adminAPI.ServiceActionHandler)).Queries("action", "{action:.*}")
+		// Update MinIO servers.
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(httpTraceAll(adminAPI.ServerUpdateHandler)).Queries("updateURL", "{updateURL:.*}")
 
-	if enableIAMOps {
-		// -- IAM APIs --
+		// Info operations
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
 
-		// Add policy IAM
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name",
-			"{name:.*}")
+		// StorageInfo operations
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(httpTraceAll(adminAPI.StorageInfoHandler))
+		// DataUsageInfo operations
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(httpTraceAll(adminAPI.DataUsageInfoHandler))
 
-		// Add user IAM
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
-			Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountingusageinfo").HandlerFunc(httpTraceAll(adminAPI.AccountingUsageInfoHandler))
 
-		// Service accounts ops
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/add-service-account").HandlerFunc(httpTraceHdrs(adminAPI.AddServiceAccount))
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/get-service-account").HandlerFunc(httpTraceHdrs(adminAPI.GetServiceAccount)).Queries("accessKey", "{accessKey:.*}")
+		if globalIsDistXL || globalIsXL {
+			/// Heal operations
 
-		// Info policy IAM
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
+			// Heal processing endpoint.
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 
-		// Remove policy IAM
-		adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix+"/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/background-heal/status").HandlerFunc(httpTraceAll(adminAPI.BackgroundHealStatusHandler))
 
-		// Set user or group policy
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/set-user-or-group-policy").
-			HandlerFunc(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup)).
-			Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
+			/// Health operations
 
-		// Remove user IAM
-		adminRouter.Methods(http.MethodDelete).Path(adminAPIVersionPrefix+"/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
+		}
 
-		// List users
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
+		// Profiling operations
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
+			Queries("profilerType", "{profilerType:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
 
-		// User info
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/user-info").HandlerFunc(httpTraceHdrs(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}")
+		// Config KV operations.
+		if enableConfigOps {
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKVHandler)).Queries("key", "{key:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/set-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKVHandler))
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/del-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.DelConfigKVHandler))
+		}
 
-		// Add/Remove members from group
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix + "/update-group-members").HandlerFunc(httpTraceHdrs(adminAPI.UpdateGroupMembers))
+		// Enable config help in all modes.
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/help-config-kv").HandlerFunc(httpTraceAll(adminAPI.HelpConfigKVHandler)).Queries("subSys", "{subSys:.*}", "key", "{key:.*}")
 
-		// Get Group
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/group").HandlerFunc(httpTraceHdrs(adminAPI.GetGroup)).Queries("group", "{group:.*}")
+		// Config KV history operations.
+		if enableConfigOps {
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-config-history-kv").HandlerFunc(httpTraceAll(adminAPI.ListConfigHistoryKVHandler)).Queries("count", "{count:[0-9]+}")
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/clear-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.ClearConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/restore-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.RestoreConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
+		}
 
-		// List Groups
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/groups").HandlerFunc(httpTraceHdrs(adminAPI.ListGroups))
+		/// Config import/export bulk operations
+		if enableConfigOps {
+			// Get config
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
+			// Set config
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
+		}
 
-		// Set Group Status
-		adminRouter.Methods(http.MethodPut).Path(adminAPIVersionPrefix+"/set-group-status").HandlerFunc(httpTraceHdrs(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
+		if enableIAMOps {
+			// -- IAM APIs --
 
-		// List policies
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
-	}
+			// Add policy IAM
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
 
-	// -- Top APIs --
-	// Top locks
-	if globalIsDistXL {
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
-	}
+			// Add user IAM
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
 
-	// HTTP Trace
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/trace").HandlerFunc(adminAPI.TraceHandler)
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
 
-	// Console Logs
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/log").HandlerFunc(httpTraceAll(adminAPI.ConsoleLogHandler))
+			// Service accounts ops
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/add-service-account").HandlerFunc(httpTraceHdrs(adminAPI.AddServiceAccount))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-service-accounts").HandlerFunc(httpTraceHdrs(adminAPI.ListServiceAccounts))
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/delete-service-account").HandlerFunc(httpTraceHdrs(adminAPI.DeleteServiceAccount)).Queries("accessKey", "{accessKey:.*}")
 
-	// -- KMS APIs --
-	//
-	adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix + "/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
+			if adminVersion == adminAPIVersionV2Prefix {
+				// Info policy IAM v2
+				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicyV2)).Queries("name", "{name:.*}")
 
-	if !globalIsGateway {
+				// List policies v2
+				adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPoliciesV2))
+			} else {
+				// Info policy IAM latest
+				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
 
-		// -- OBD API --
-		adminRouter.Methods(http.MethodGet).Path(adminAPIVersionPrefix+"/obdinfo").HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)).Queries("perfdrive", "{perfdrive:true|false}", "perfnet", "{perfnet:true|false}", "minioinfo", "{minioinfo:true|false}", "minioconfig", "{minioconfig:true|false}", "syscpu", "{syscpu:true|false}", "sysdiskhw", "{sysdiskhw:true|false}", "sysosinfo", "{sysosinfo:true|false}", "sysmem", "{sysmem:true|false}", "sysprocess", "{sysprocess:true|false}")
+				// List policies latest
+				adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
+			}
+
+			// Remove policy IAM
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
+
+			// Set user or group policy
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-or-group-policy").
+				HandlerFunc(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup)).
+				Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
+
+			// Remove user IAM
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
+
+			// List users
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
+
+			// User info
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/user-info").HandlerFunc(httpTraceHdrs(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}")
+
+			// Add/Remove members from group
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/update-group-members").HandlerFunc(httpTraceHdrs(adminAPI.UpdateGroupMembers))
+
+			// Get Group
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/group").HandlerFunc(httpTraceHdrs(adminAPI.GetGroup)).Queries("group", "{group:.*}")
+
+			// List Groups
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/groups").HandlerFunc(httpTraceHdrs(adminAPI.ListGroups))
+
+			// Set Group Status
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(httpTraceHdrs(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
+
+		}
+
+		// -- Top APIs --
+		// Top locks
+		if globalIsDistXL {
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
+		}
+
+		// HTTP Trace
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(adminAPI.TraceHandler)
+
+		// Console Logs
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/log").HandlerFunc(httpTraceAll(adminAPI.ConsoleLogHandler))
+
+		// -- KMS APIs --
+		//
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
+
+		if !globalIsGateway {
+			// -- OBD API --
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/obdinfo").
+				HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)).
+				Queries("perfdrive", "{perfdrive:true|false}",
+					"perfnet", "{perfnet:true|false}",
+					"minioinfo", "{minioinfo:true|false}",
+					"minioconfig", "{minioconfig:true|false}",
+					"syscpu", "{syscpu:true|false}",
+					"sysdiskhw", "{sysdiskhw:true|false}",
+					"sysosinfo", "{sysosinfo:true|false}",
+					"sysmem", "{sysmem:true|false}",
+					"sysprocess", "{sysprocess:true|false}",
+				)
+		}
 	}
 
 	// If none of the routes match add default error handler routes

cmd/admin-server-info.go

@@ -17,176 +17,14 @@
 package cmd
 
 import (
-	"net"
 	"net/http"
 	"os"
 
-	"github.com/minio/minio-go/v6/pkg/set"
-	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/madmin"
-	"github.com/minio/minio/pkg/mem"
-
-	cpuhw "github.com/shirou/gopsutil/cpu"
 )
 
-// getLocalMemUsage - returns ServerMemUsageInfo for all zones, endpoints.
-func getLocalMemUsage(endpointZones EndpointZones, r *http.Request) ServerMemUsageInfo {
-	var memUsages []mem.Usage
-	var historicUsages []mem.Usage
-	seenHosts := set.NewStringSet()
-	for _, ep := range endpointZones {
-		for _, endpoint := range ep.Endpoints {
-			if seenHosts.Contains(endpoint.Host) {
-				continue
-			}
-			seenHosts.Add(endpoint.Host)
-
-			// Only proceed for local endpoints
-			if endpoint.IsLocal {
-				memUsages = append(memUsages, mem.GetUsage())
-				historicUsages = append(historicUsages, mem.GetHistoricUsage())
-			}
-		}
-	}
-	addr := r.Host
-	if globalIsDistXL {
-		addr = GetLocalPeer(endpointZones)
-	}
-	return ServerMemUsageInfo{
-		Addr:          addr,
-		Usage:         memUsages,
-		HistoricUsage: historicUsages,
-	}
-}
-
-// getLocalCPULoad - returns ServerCPULoadInfo for all zones, endpoints.
-func getLocalCPULoad(endpointZones EndpointZones, r *http.Request) ServerCPULoadInfo {
-	var cpuLoads []cpu.Load
-	var historicLoads []cpu.Load
-	seenHosts := set.NewStringSet()
-	for _, ep := range endpointZones {
-		for _, endpoint := range ep.Endpoints {
-			if seenHosts.Contains(endpoint.Host) {
-				continue
-			}
-			seenHosts.Add(endpoint.Host)
-
-			// Only proceed for local endpoints
-			if endpoint.IsLocal {
-				cpuLoads = append(cpuLoads, cpu.GetLoad())
-				historicLoads = append(historicLoads, cpu.GetHistoricLoad())
-			}
-		}
-	}
-	addr := r.Host
-	if globalIsDistXL {
-		addr = GetLocalPeer(endpointZones)
-	}
-	return ServerCPULoadInfo{
-		Addr:         addr,
-		Load:         cpuLoads,
-		HistoricLoad: historicLoads,
-	}
-}
-
-// getLocalDrivesPerf - returns ServerDrivesPerfInfo for all zones, endpoints.
-func getLocalDrivesPerf(endpointZones EndpointZones, size int64, r *http.Request) madmin.ServerDrivesPerfInfo {
-	var dps []disk.Performance
-	for _, ep := range endpointZones {
-		for _, endpoint := range ep.Endpoints {
-			// Only proceed for local endpoints
-			if endpoint.IsLocal {
-				if _, err := os.Stat(endpoint.Path); err != nil {
-					// Since this drive is not available, add relevant details and proceed
-					dps = append(dps, disk.Performance{Path: endpoint.Path, Error: err.Error()})
-					continue
-				}
-				dp := disk.GetPerformance(pathJoin(endpoint.Path, minioMetaTmpBucket, mustGetUUID()), size)
-				dp.Path = endpoint.Path
-				dps = append(dps, dp)
-			}
-		}
-	}
-	addr := r.Host
-	if globalIsDistXL {
-		addr = GetLocalPeer(endpointZones)
-	}
-	return madmin.ServerDrivesPerfInfo{
-		Addr: addr,
-		Perf: dps,
-	}
-}
-
-// getLocalCPUInfo - returns ServerCPUHardwareInfo for all zones, endpoints.
-func getLocalCPUInfo(endpointZones EndpointZones, r *http.Request) madmin.ServerCPUHardwareInfo {
-	var cpuHardwares []cpuhw.InfoStat
-	seenHosts := set.NewStringSet()
-	for _, ep := range endpointZones {
-		for _, endpoint := range ep.Endpoints {
-			if seenHosts.Contains(endpoint.Host) {
-				continue
-			}
-			// Add to the list of visited hosts
-			seenHosts.Add(endpoint.Host)
-			// Only proceed for local endpoints
-			if endpoint.IsLocal {
-				cpuHardware, err := cpuhw.Info()
-				if err != nil {
-					return madmin.ServerCPUHardwareInfo{
-						Error: err.Error(),
-					}
-				}
-				cpuHardwares = append(cpuHardwares, cpuHardware...)
-			}
-		}
-	}
-	addr := r.Host
-	if globalIsDistXL {
-		addr = GetLocalPeer(endpointZones)
-	}
-
-	return madmin.ServerCPUHardwareInfo{
-		Addr:    addr,
-		CPUInfo: cpuHardwares,
-	}
-}
-
-// getLocalNetworkInfo - returns ServerNetworkHardwareInfo for all zones, endpoints.
-func getLocalNetworkInfo(endpointZones EndpointZones, r *http.Request) madmin.ServerNetworkHardwareInfo {
-	var networkHardwares []net.Interface
-	seenHosts := set.NewStringSet()
-	for _, ep := range endpointZones {
-		for _, endpoint := range ep.Endpoints {
-			if seenHosts.Contains(endpoint.Host) {
-				continue
-			}
-			// Add to the list of visited hosts
-			seenHosts.Add(endpoint.Host)
-			// Only proceed for local endpoints
-			if endpoint.IsLocal {
-				networkHardware, err := net.Interfaces()
-				if err != nil {
-					return madmin.ServerNetworkHardwareInfo{
-						Error: err.Error(),
-					}
-				}
-				networkHardwares = append(networkHardwares, networkHardware...)
-			}
-		}
-	}
-	addr := r.Host
-	if globalIsDistXL {
-		addr = GetLocalPeer(endpointZones)
-	}
-
-	return madmin.ServerNetworkHardwareInfo{
-		Addr:        addr,
-		NetworkInfo: networkHardwares,
-	}
-}
-
-// getLocalServerProperty - returns ServerDrivesPerfInfo for only the
+// getLocalServerProperty - returns madmin.ServerProperties for only the
 // local endpoints from given list of endpoints
 func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin.ServerProperties {
 	var disks []madmin.Disk

cmd/api-errors.go

@@ -25,7 +25,6 @@ import (
 	"strings"
 
 	"github.com/Azure/azure-storage-blob-go/azblob"
-	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 	"google.golang.org/api/googleapi"
 
 	minio "github.com/minio/minio-go/v6"
@@ -151,6 +150,7 @@ const (
 	ErrBadRequest
 	ErrKeyTooLongError
 	ErrInvalidBucketObjectLockConfiguration
+	ErrObjectLockConfigurationNotFound
 	ErrObjectLockConfigurationNotAllowed
 	ErrNoSuchObjectLockConfiguration
 	ErrObjectLocked
@@ -334,20 +334,28 @@ const (
 	ErrAdminProfilerNotEnabled
 	ErrInvalidDecompressedSize
 	ErrAddUserInvalidArgument
-	ErrAddServiceAccountInvalidArgument
+	ErrAdminAccountNotEligible
+	ErrServiceAccountNotFound
 	ErrPostPolicyConditionInvalidFormat
 )
 
 type errorCodeMap map[APIErrorCode]APIError
 
-func (e errorCodeMap) ToAPIErr(errCode APIErrorCode) APIError {
+func (e errorCodeMap) ToAPIErrWithErr(errCode APIErrorCode, err error) APIError {
 	apiErr, ok := e[errCode]
 	if !ok {
-		return e[ErrInternalError]
+		apiErr = e[ErrInternalError]
+	}
+	if err != nil {
+		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
 	}
 	return apiErr
 }
 
+func (e errorCodeMap) ToAPIErr(errCode APIErrorCode) APIError {
+	return e.ToAPIErrWithErr(errCode, nil)
+}
+
 // error code to APIError structure, these fields carry respective
 // descriptions for all the error responses.
 var errorCodes = errorCodeMap{
@@ -765,9 +773,14 @@ var errorCodes = errorCodeMap{
 		Description:    "Bucket is missing ObjectLockConfiguration",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrObjectLockConfigurationNotFound: {
+		Code:           "ObjectLockConfigurationNotFoundError",
+		Description:    "Object Lock configuration does not exist for this bucket",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrObjectLockConfigurationNotAllowed: {
 		Code:           "InvalidBucketState",
-		Description:    "Object Lock configuration cannot be enabled on existing buckets.",
+		Description:    "Object Lock configuration cannot be enabled on existing buckets",
 		HTTPStatusCode: http.StatusConflict,
 	},
 	ErrNoSuchObjectLockConfiguration: {
@@ -1586,12 +1599,16 @@ var errorCodes = errorCodeMap{
 		Description:    "User is not allowed to be same as admin access key",
 		HTTPStatusCode: http.StatusConflict,
 	},
-	ErrAddServiceAccountInvalidArgument: {
-		Code:           "XMinioInvalidArgument",
-		Description:    "New service accounts for admin access key is not allowed",
+	ErrAdminAccountNotEligible: {
+		Code:           "XMinioInvalidIAMCredentials",
+		Description:    "The administrator key is not eligible for this operation",
 		HTTPStatusCode: http.StatusConflict,
 	},
-
+	ErrServiceAccountNotFound: {
+		Code:           "XMinioInvalidIAMCredentials",
+		Description:    "The specified service account is not found",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrPostPolicyConditionInvalidFormat: {
 		Code:           "PostPolicyInvalidKeyName",
 		Description:    "Invalid according to Policy: Policy Condition failed",
@@ -1898,12 +1915,6 @@ func toAPIError(ctx context.Context, err error) APIError {
 				Description:    e.Error(),
 				HTTPStatusCode: e.Response().StatusCode,
 			}
-		case oss.ServiceError:
-			apiErr = APIError{
-				Code:           e.Code,
-				Description:    e.Message,
-				HTTPStatusCode: e.StatusCode,
-			}
 			// Add more Gateway SDKs here if any in future.
 		}
 	}

cmd/api-headers.go

@@ -97,6 +97,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	if !objInfo.Expires.IsZero() {
 		w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
 	}
+
 	if globalCacheConfig.Enabled {
 		w.Header().Set(xhttp.XCache, objInfo.CacheStatus.String())
 		w.Header().Set(xhttp.XCacheLookup, objInfo.CacheLookupStatus.String())

cmd/api-router.go

@@ -18,14 +18,9 @@ package cmd
 
 import (
 	"net/http"
-	"strconv"
-	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/config"
 	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/env"
 )
 
 func newHTTPServerFn() *xhttp.Server {
@@ -92,229 +87,200 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
-	requestsMax, _ := strconv.Atoi(env.Get(config.EnvAPIRequestsMax, "0"))
-	if requestsMax < 0 {
-		requestsMax = 0 // 0 means unlimited
-	}
-
-	var (
-		enabled              bool
-		requestsMaxPerServer int
-		requestsDeadline     time.Duration
-	)
-
-	if len(globalEndpoints.Hosts()) == 0 {
-		requestsMaxPerServer = requestsMax
-	} else {
-		requestsMaxPerServer = requestsMax / len(globalEndpoints.Hosts())
-	}
-
-	enabled = requestsMaxPerServer > 0
-	if enabled {
-		var err error
-		requestsDeadline, err = time.ParseDuration(env.Get(config.EnvAPIRequestsDeadline, "10s"))
-		if err != nil {
-			logger.Info("(%s) parsing environment value MINIO_API_REQUESTS_DEADLINE, defaulting to 10 seconds", err)
-			requestsDeadline = 10 * time.Second
-		}
-	}
-
-	requestsMaxCh := make(chan struct{}, requestsMaxPerServer)
-
 	for _, bucket := range routers {
 		// Object operations
 		// HeadObject
 		bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("headobject", httpTraceAll(api.HeadObjectHandler)), enabled, requestsMaxCh, requestsDeadline))
+			maxClients(collectAPIStats("headobject", httpTraceAll(api.HeadObjectHandler))))
 		// CopyObjectPart
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(maxClients(collectAPIStats("copyobjectpart", httpTraceAll(api.CopyObjectPartHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(maxClients(collectAPIStats("copyobjectpart", httpTraceAll(api.CopyObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobjectpart", httpTraceHdrs(api.PutObjectPartHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+			maxClients(collectAPIStats("putobjectpart", httpTraceHdrs(api.PutObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// ListObjectParts
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("listobjectparts", httpTraceAll(api.ListObjectPartsHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("uploadId", "{uploadId:.*}")
+			maxClients(collectAPIStats("listobjectparts", httpTraceAll(api.ListObjectPartsHandler)))).Queries("uploadId", "{uploadId:.*}")
 		// CompleteMultipartUpload
 		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("completemutipartupload", httpTraceAll(api.CompleteMultipartUploadHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("uploadId", "{uploadId:.*}")
+			maxClients(collectAPIStats("completemutipartupload", httpTraceAll(api.CompleteMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
 		// NewMultipartUpload
 		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("newmultipartupload", httpTraceAll(api.NewMultipartUploadHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("uploads", "")
+			maxClients(collectAPIStats("newmultipartupload", httpTraceAll(api.NewMultipartUploadHandler)))).Queries("uploads", "")
 		// AbortMultipartUpload
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("abortmultipartupload", httpTraceAll(api.AbortMultipartUploadHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("uploadId", "{uploadId:.*}")
+			maxClients(collectAPIStats("abortmultipartupload", httpTraceAll(api.AbortMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("getobjectacl", httpTraceHdrs(api.GetObjectACLHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("acl", "")
+			maxClients(collectAPIStats("getobjectacl", httpTraceHdrs(api.GetObjectACLHandler)))).Queries("acl", "")
 		// PutObjectACL - this is a dummy call.
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobjectacl", httpTraceHdrs(api.PutObjectACLHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("acl", "")
+			maxClients(collectAPIStats("putobjectacl", httpTraceHdrs(api.PutObjectACLHandler)))).Queries("acl", "")
 		// GetObjectTagging
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("getobjecttagging", httpTraceHdrs(api.GetObjectTaggingHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("tagging", "")
+			maxClients(collectAPIStats("getobjecttagging", httpTraceHdrs(api.GetObjectTaggingHandler)))).Queries("tagging", "")
 		// PutObjectTagging
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobjecttagging", httpTraceHdrs(api.PutObjectTaggingHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("tagging", "")
+			maxClients(collectAPIStats("putobjecttagging", httpTraceHdrs(api.PutObjectTaggingHandler)))).Queries("tagging", "")
 		// DeleteObjectTagging
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("deleteobjecttagging", httpTraceHdrs(api.DeleteObjectTaggingHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("tagging", "")
+			maxClients(collectAPIStats("deleteobjecttagging", httpTraceHdrs(api.DeleteObjectTaggingHandler)))).Queries("tagging", "")
 		// SelectObjectContent
 		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("selectobjectcontent", httpTraceHdrs(api.SelectObjectContentHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("select", "").Queries("select-type", "2")
+			maxClients(collectAPIStats("selectobjectcontent", httpTraceHdrs(api.SelectObjectContentHandler)))).Queries("select", "").Queries("select-type", "2")
 		// GetObjectRetention
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("getobjectretention", httpTraceAll(api.GetObjectRetentionHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("retention", "")
+			maxClients(collectAPIStats("getobjectretention", httpTraceAll(api.GetObjectRetentionHandler)))).Queries("retention", "")
 		// GetObjectLegalHold
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("getobjectlegalhold", httpTraceAll(api.GetObjectLegalHoldHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("legal-hold", "")
+			maxClients(collectAPIStats("getobjectlegalhold", httpTraceAll(api.GetObjectLegalHoldHandler)))).Queries("legal-hold", "")
 		// GetObject
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler)), enabled, requestsMaxCh, requestsDeadline))
+			maxClients(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler))))
 		// CopyObject
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(maxClients(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler)), enabled, requestsMaxCh, requestsDeadline))
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(maxClients(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler))))
 		// PutObjectRetention
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobjectretention", httpTraceAll(api.PutObjectRetentionHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("retention", "")
+			maxClients(collectAPIStats("putobjectretention", httpTraceAll(api.PutObjectRetentionHandler)))).Queries("retention", "")
 		// PutObjectLegalHold
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobjectlegalhold", httpTraceAll(api.PutObjectLegalHoldHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("legal-hold", "")
+			maxClients(collectAPIStats("putobjectlegalhold", httpTraceAll(api.PutObjectLegalHoldHandler)))).Queries("legal-hold", "")
 
 		// PutObject
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("putobject", httpTraceHdrs(api.PutObjectHandler)), enabled, requestsMaxCh, requestsDeadline))
+			maxClients(collectAPIStats("putobject", httpTraceHdrs(api.PutObjectHandler))))
 		// DeleteObject
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			maxClients(collectAPIStats("deleteobject", httpTraceAll(api.DeleteObjectHandler)), enabled, requestsMaxCh, requestsDeadline))
+			maxClients(collectAPIStats("deleteobject", httpTraceAll(api.DeleteObjectHandler))))
 
 		/// Bucket operations
 		// GetBucketLocation
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketlocation", httpTraceAll(api.GetBucketLocationHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("location", "")
+			maxClients(collectAPIStats("getbucketlocation", httpTraceAll(api.GetBucketLocationHandler)))).Queries("location", "")
 		// GetBucketPolicy
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketpolicy", httpTraceAll(api.GetBucketPolicyHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("policy", "")
+			maxClients(collectAPIStats("getbucketpolicy", httpTraceAll(api.GetBucketPolicyHandler)))).Queries("policy", "")
 		// GetBucketLifecycle
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("lifecycle", "")
+			maxClients(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
 		// GetBucketEncryption
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketencryption", httpTraceAll(api.GetBucketEncryptionHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("encryption", "")
+			maxClients(collectAPIStats("getbucketencryption", httpTraceAll(api.GetBucketEncryptionHandler)))).Queries("encryption", "")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketacl", httpTraceAll(api.GetBucketACLHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("acl", "")
+			maxClients(collectAPIStats("getbucketacl", httpTraceAll(api.GetBucketACLHandler)))).Queries("acl", "")
 		// PutBucketACL -- this is a dummy call.
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketacl", httpTraceAll(api.PutBucketACLHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("acl", "")
+			maxClients(collectAPIStats("putbucketacl", httpTraceAll(api.PutBucketACLHandler)))).Queries("acl", "")
 		// GetBucketCors - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketcors", httpTraceAll(api.GetBucketCorsHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("cors", "")
+			maxClients(collectAPIStats("getbucketcors", httpTraceAll(api.GetBucketCorsHandler)))).Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketwebsite", httpTraceAll(api.GetBucketWebsiteHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("website", "")
+			maxClients(collectAPIStats("getbucketwebsite", httpTraceAll(api.GetBucketWebsiteHandler)))).Queries("website", "")
 		// GetBucketAccelerateHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketaccelerate", httpTraceAll(api.GetBucketAccelerateHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("accelerate", "")
+			maxClients(collectAPIStats("getbucketaccelerate", httpTraceAll(api.GetBucketAccelerateHandler)))).Queries("accelerate", "")
 		// GetBucketRequestPaymentHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketrequestpayment", httpTraceAll(api.GetBucketRequestPaymentHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("requestPayment", "")
+			maxClients(collectAPIStats("getbucketrequestpayment", httpTraceAll(api.GetBucketRequestPaymentHandler)))).Queries("requestPayment", "")
 		// GetBucketLoggingHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketlogging", httpTraceAll(api.GetBucketLoggingHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("logging", "")
+			maxClients(collectAPIStats("getbucketlogging", httpTraceAll(api.GetBucketLoggingHandler)))).Queries("logging", "")
 		// GetBucketLifecycleHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("lifecycle", "")
+			maxClients(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
 		// GetBucketReplicationHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketreplication", httpTraceAll(api.GetBucketReplicationHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("replication", "")
+			maxClients(collectAPIStats("getbucketreplication", httpTraceAll(api.GetBucketReplicationHandler)))).Queries("replication", "")
 		// GetBucketTaggingHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbuckettagging", httpTraceAll(api.GetBucketTaggingHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("tagging", "")
+			maxClients(collectAPIStats("getbuckettagging", httpTraceAll(api.GetBucketTaggingHandler)))).Queries("tagging", "")
 		//DeleteBucketWebsiteHandler
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucketwebsite", httpTraceAll(api.DeleteBucketWebsiteHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("website", "")
+			maxClients(collectAPIStats("deletebucketwebsite", httpTraceAll(api.DeleteBucketWebsiteHandler)))).Queries("website", "")
 		// DeleteBucketTaggingHandler
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebuckettagging", httpTraceAll(api.DeleteBucketTaggingHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("tagging", "")
+			maxClients(collectAPIStats("deletebuckettagging", httpTraceAll(api.DeleteBucketTaggingHandler)))).Queries("tagging", "")
 
 		// GetBucketObjectLockConfig
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketobjectlockconfiguration", httpTraceAll(api.GetBucketObjectLockConfigHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("object-lock", "")
+			maxClients(collectAPIStats("getbucketobjectlockconfiguration", httpTraceAll(api.GetBucketObjectLockConfigHandler)))).Queries("object-lock", "")
 		// GetBucketVersioning
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketversioning", httpTraceAll(api.GetBucketVersioningHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("versioning", "")
+			maxClients(collectAPIStats("getbucketversioning", httpTraceAll(api.GetBucketVersioningHandler)))).Queries("versioning", "")
 		// GetBucketNotification
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("getbucketnotification", httpTraceAll(api.GetBucketNotificationHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("notification", "")
+			maxClients(collectAPIStats("getbucketnotification", httpTraceAll(api.GetBucketNotificationHandler)))).Queries("notification", "")
 		// ListenBucketNotification
 		bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listenbucketnotification", httpTraceAll(api.ListenBucketNotificationHandler))).Queries("events", "{events:.*}")
 		// ListMultipartUploads
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listmultipartuploads", httpTraceAll(api.ListMultipartUploadsHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("uploads", "")
+			maxClients(collectAPIStats("listmultipartuploads", httpTraceAll(api.ListMultipartUploadsHandler)))).Queries("uploads", "")
 		// ListObjectsV2M
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listobjectsv2M", httpTraceAll(api.ListObjectsV2MHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("list-type", "2", "metadata", "true")
+			maxClients(collectAPIStats("listobjectsv2M", httpTraceAll(api.ListObjectsV2MHandler)))).Queries("list-type", "2", "metadata", "true")
 		// ListObjectsV2
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listobjectsv2", httpTraceAll(api.ListObjectsV2Handler)), enabled, requestsMaxCh, requestsDeadline)).Queries("list-type", "2")
+			maxClients(collectAPIStats("listobjectsv2", httpTraceAll(api.ListObjectsV2Handler)))).Queries("list-type", "2")
 		// ListBucketVersions
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listbucketversions", httpTraceAll(api.ListBucketObjectVersionsHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("versions", "")
+			maxClients(collectAPIStats("listbucketversions", httpTraceAll(api.ListBucketObjectVersionsHandler)))).Queries("versions", "")
 		// ListObjectsV1 (Legacy)
 		bucket.Methods(http.MethodGet).HandlerFunc(
-			maxClients(collectAPIStats("listobjectsv1", httpTraceAll(api.ListObjectsV1Handler)), enabled, requestsMaxCh, requestsDeadline))
+			maxClients(collectAPIStats("listobjectsv1", httpTraceAll(api.ListObjectsV1Handler))))
 		// PutBucketLifecycle
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketlifecycle", httpTraceAll(api.PutBucketLifecycleHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("lifecycle", "")
+			maxClients(collectAPIStats("putbucketlifecycle", httpTraceAll(api.PutBucketLifecycleHandler)))).Queries("lifecycle", "")
 		// PutBucketEncryption
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketencryption", httpTraceAll(api.PutBucketEncryptionHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("encryption", "")
+			maxClients(collectAPIStats("putbucketencryption", httpTraceAll(api.PutBucketEncryptionHandler)))).Queries("encryption", "")
 
 		// PutBucketPolicy
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketpolicy", httpTraceAll(api.PutBucketPolicyHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("policy", "")
+			maxClients(collectAPIStats("putbucketpolicy", httpTraceAll(api.PutBucketPolicyHandler)))).Queries("policy", "")
 
 		// PutBucketObjectLockConfig
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketobjectlockconfig", httpTraceAll(api.PutBucketObjectLockConfigHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("object-lock", "")
+			maxClients(collectAPIStats("putbucketobjectlockconfig", httpTraceAll(api.PutBucketObjectLockConfigHandler)))).Queries("object-lock", "")
 		// PutBucketVersioning
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketversioning", httpTraceAll(api.PutBucketVersioningHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("versioning", "")
+			maxClients(collectAPIStats("putbucketversioning", httpTraceAll(api.PutBucketVersioningHandler)))).Queries("versioning", "")
 		// PutBucketNotification
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucketnotification", httpTraceAll(api.PutBucketNotificationHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("notification", "")
+			maxClients(collectAPIStats("putbucketnotification", httpTraceAll(api.PutBucketNotificationHandler)))).Queries("notification", "")
 		// PutBucket
 		bucket.Methods(http.MethodPut).HandlerFunc(
-			maxClients(collectAPIStats("putbucket", httpTraceAll(api.PutBucketHandler)), enabled, requestsMaxCh, requestsDeadline))
+			maxClients(collectAPIStats("putbucket", httpTraceAll(api.PutBucketHandler))))
 		// HeadBucket
 		bucket.Methods(http.MethodHead).HandlerFunc(
-			maxClients(collectAPIStats("headbucket", httpTraceAll(api.HeadBucketHandler)), enabled, requestsMaxCh, requestsDeadline))
+			maxClients(collectAPIStats("headbucket", httpTraceAll(api.HeadBucketHandler))))
 		// PostPolicy
 		bucket.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(
-			maxClients(collectAPIStats("postpolicybucket", httpTraceHdrs(api.PostPolicyBucketHandler)), enabled, requestsMaxCh, requestsDeadline))
+			maxClients(collectAPIStats("postpolicybucket", httpTraceHdrs(api.PostPolicyBucketHandler))))
 		// DeleteMultipleObjects
 		bucket.Methods(http.MethodPost).HandlerFunc(
-			maxClients(collectAPIStats("deletemultipleobjects", httpTraceAll(api.DeleteMultipleObjectsHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("delete", "")
+			maxClients(collectAPIStats("deletemultipleobjects", httpTraceAll(api.DeleteMultipleObjectsHandler)))).Queries("delete", "")
 		// DeleteBucketPolicy
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucketpolicy", httpTraceAll(api.DeleteBucketPolicyHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("policy", "")
+			maxClients(collectAPIStats("deletebucketpolicy", httpTraceAll(api.DeleteBucketPolicyHandler)))).Queries("policy", "")
 		// DeleteBucketLifecycle
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucketlifecycle", httpTraceAll(api.DeleteBucketLifecycleHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("lifecycle", "")
+			maxClients(collectAPIStats("deletebucketlifecycle", httpTraceAll(api.DeleteBucketLifecycleHandler)))).Queries("lifecycle", "")
 		// DeleteBucketEncryption
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucketencryption", httpTraceAll(api.DeleteBucketEncryptionHandler)), enabled, requestsMaxCh, requestsDeadline)).Queries("encryption", "")
+			maxClients(collectAPIStats("deletebucketencryption", httpTraceAll(api.DeleteBucketEncryptionHandler)))).Queries("encryption", "")
 		// DeleteBucket
 		bucket.Methods(http.MethodDelete).HandlerFunc(
-			maxClients(collectAPIStats("deletebucket", httpTraceAll(api.DeleteBucketHandler)), enabled, requestsMaxCh, requestsDeadline))
+			maxClients(collectAPIStats("deletebucket", httpTraceAll(api.DeleteBucketHandler))))
 	}
 
 	/// Root operation
 
 	// ListBuckets
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
-		maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler)), enabled, requestsMaxCh, requestsDeadline))
+		maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler))))
 
 	// If none of the routes match add default error handler routes
 	apiRouter.NotFoundHandler = http.HandlerFunc(collectAPIStats("notfound", httpTraceAll(errorResponseHandler)))

cmd/auth-handler.go

@@ -26,12 +26,15 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strconv"
 	"strings"
+	"time"
 
 	xhttp "github.com/minio/minio/cmd/http"
 	xjwt "github.com/minio/minio/cmd/jwt"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/hash"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
@@ -118,9 +121,7 @@ func getRequestAuthType(r *http.Request) authType {
 	return authTypeUnknown
 }
 
-// checkAdminRequestAuthType checks whether the request is a valid signature V2 or V4 request.
-// It does not accept presigned or JWT or anonymous requests.
-func checkAdminRequestAuthType(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
+func validateAdminSignature(ctx context.Context, r *http.Request, region string) (auth.Credentials, map[string]interface{}, bool, APIErrorCode) {
 	var cred auth.Credentials
 	var owner bool
 	s3Err := ErrAccessDenied
@@ -129,7 +130,7 @@ func checkAdminRequestAuthType(ctx context.Context, r *http.Request, action iamp
 		// We only support admin credentials to access admin APIs.
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 		if s3Err != ErrNone {
-			return cred, s3Err
+			return cred, nil, owner, s3Err
 		}
 
 		// we only support V4 (no presign) with auth body
@@ -139,14 +140,24 @@ func checkAdminRequestAuthType(ctx context.Context, r *http.Request, action iamp
 		reqInfo := (&logger.ReqInfo{}).AppendTags("requestHeaders", dumpRequest(r))
 		ctx := logger.SetReqInfo(ctx, reqInfo)
 		logger.LogIf(ctx, errors.New(getAPIError(s3Err).Description), logger.Application)
+		return cred, nil, owner, s3Err
 	}
 
-	var claims map[string]interface{}
-	claims, s3Err = checkClaimsFromToken(r, cred)
+	claims, s3Err := checkClaimsFromToken(r, cred)
+	if s3Err != ErrNone {
+		return cred, nil, owner, s3Err
+	}
+
+	return cred, claims, owner, ErrNone
+}
+
+// checkAdminRequestAuthType checks whether the request is a valid signature V2 or V4 request.
+// It does not accept presigned or JWT or anonymous requests.
+func checkAdminRequestAuthType(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
+	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, region)
 	if s3Err != ErrNone {
 		return cred, s3Err
 	}
-
 	if globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
 		Action:          iampolicy.Action(action),
@@ -173,15 +184,13 @@ func getSessionToken(r *http.Request) (token string) {
 // Fetch claims in the security token returned by the client, doesn't return
 // errors - upon errors the returned claims map will be empty.
 func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
-	claims, _ := getClaimsFromToken(r)
+	claims, _ := getClaimsFromToken(r, getSessionToken(r))
 	return claims
 }
 
 // Fetch claims in the security token returned by the client.
-func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
+func getClaimsFromToken(r *http.Request, token string) (map[string]interface{}, error) {
 	claims := xjwt.NewMapClaims()
-
-	token := getSessionToken(r)
 	if token == "" {
 		return claims.Map(), nil
 	}
@@ -242,10 +251,13 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 	if token != "" && cred.AccessKey == "" {
 		return nil, ErrNoAccessKey
 	}
+	if cred.IsServiceAccount() && token == "" {
+		token = cred.SessionToken
+	}
 	if subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
 		return nil, ErrInvalidToken
 	}
-	claims, err := getClaimsFromToken(r)
+	claims, err := getClaimsFromToken(r, token)
 	if err != nil {
 		return nil, toAPIErrorCode(r.Context(), err)
 	}
@@ -464,6 +476,106 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL, guessIsBrowserReq(r))
 }
 
+func validateSignature(atype authType, r *http.Request) (auth.Credentials, bool, map[string]interface{}, APIErrorCode) {
+	var cred auth.Credentials
+	var owner bool
+	var s3Err APIErrorCode
+	switch atype {
+	case authTypeUnknown, authTypeStreamingSigned:
+		return cred, owner, nil, ErrSignatureVersionNotSupported
+	case authTypeSignedV2, authTypePresignedV2:
+		if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
+			return cred, owner, nil, s3Err
+		}
+		cred, owner, s3Err = getReqAccessKeyV2(r)
+	case authTypePresigned, authTypeSigned:
+		region := globalServerRegion
+		if s3Err = isReqAuthenticated(GlobalContext, r, region, serviceS3); s3Err != ErrNone {
+			return cred, owner, nil, s3Err
+		}
+		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
+	}
+	if s3Err != ErrNone {
+		return cred, owner, nil, s3Err
+	}
+
+	claims, s3Err := checkClaimsFromToken(r, cred)
+	if s3Err != ErrNone {
+		return cred, owner, nil, s3Err
+	}
+
+	return cred, owner, claims, ErrNone
+}
+
+func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate time.Time, retMode objectlock.RetMode, byPassSet bool, r *http.Request, cred auth.Credentials, owner bool, claims map[string]interface{}) (s3Err APIErrorCode) {
+	var retSet bool
+	if cred.AccessKey == "" {
+		conditions := getConditionValues(r, "", "", nil)
+		conditions["object-lock-mode"] = []string{string(retMode)}
+		conditions["object-lock-retain-until-date"] = []string{retDate.Format(time.RFC3339)}
+		if retDays > 0 {
+			conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
+		}
+		if retMode == objectlock.RetGovernance && byPassSet {
+			byPassSet = globalPolicySys.IsAllowed(policy.Args{
+				AccountName:     cred.AccessKey,
+				Action:          policy.Action(policy.BypassGovernanceRetentionAction),
+				BucketName:      bucketName,
+				ConditionValues: conditions,
+				IsOwner:         false,
+				ObjectName:      objectName,
+			})
+		}
+		if globalPolicySys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Action:          policy.Action(policy.PutObjectRetentionAction),
+			BucketName:      bucketName,
+			ConditionValues: conditions,
+			IsOwner:         false,
+			ObjectName:      objectName,
+		}) {
+			retSet = true
+		}
+		if byPassSet || retSet {
+			return ErrNone
+		}
+		return ErrAccessDenied
+	}
+
+	conditions := getConditionValues(r, "", cred.AccessKey, claims)
+	conditions["object-lock-mode"] = []string{string(retMode)}
+	conditions["object-lock-retain-until-date"] = []string{retDate.Format(time.RFC3339)}
+	if retDays > 0 {
+		conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
+	}
+	if retMode == objectlock.RetGovernance && byPassSet {
+		byPassSet = globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     cred.AccessKey,
+			Action:          policy.BypassGovernanceRetentionAction,
+			BucketName:      bucketName,
+			ObjectName:      objectName,
+			ConditionValues: conditions,
+			IsOwner:         owner,
+			Claims:          claims,
+		})
+	}
+	if globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     cred.AccessKey,
+		Action:          policy.PutObjectRetentionAction,
+		BucketName:      bucketName,
+		ConditionValues: conditions,
+		ObjectName:      objectName,
+		IsOwner:         owner,
+		Claims:          claims,
+	}) {
+		retSet = true
+	}
+	if byPassSet || retSet {
+		return ErrNone
+	}
+	return ErrAccessDenied
+}
+
 // isPutActionAllowed - check if PUT operation is allowed on the resource, this
 // call verifies bucket policies and IAM policies, supports multi user
 // checks etc.
@@ -488,6 +600,15 @@ func isPutActionAllowed(atype authType, bucketName, objectName string, r *http.R
 		return s3Err
 	}
 
+	// Do not check for PutObjectRetentionAction permission,
+	// if mode and retain until date are not set.
+	// Can happen when bucket has default lock config set
+	if action == iampolicy.PutObjectRetentionAction &&
+		r.Header.Get(xhttp.AmzObjectLockMode) == "" &&
+		r.Header.Get(xhttp.AmzObjectLockRetainUntilDate) == "" {
+		return ErrNone
+	}
+
 	if cred.AccessKey == "" {
 		if globalPolicySys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,

cmd/auth-handler_test.go

@@ -391,6 +391,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 		}
 	}
 }
+
 func TestCheckAdminRequestAuthType(t *testing.T) {
 	objLayer, fsDir, err := prepareFS()
 	if err != nil {
@@ -425,3 +426,48 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 		}
 	}
 }
+
+func TestValidateAdminSignature(t *testing.T) {
+
+	ctx := context.Background()
+
+	objLayer, fsDir, err := prepareFS()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(fsDir)
+
+	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+
+	creds, err := auth.CreateCredentials("admin", "mypassword")
+	if err != nil {
+		t.Fatalf("unable create credential, %s", err)
+	}
+	globalActiveCred = creds
+
+	testCases := []struct {
+		AccessKey string
+		SecretKey string
+		ErrCode   APIErrorCode
+	}{
+		{"", "", ErrInvalidAccessKeyID},
+		{"admin", "", ErrSignatureDoesNotMatch},
+		{"admin", "wrongpassword", ErrSignatureDoesNotMatch},
+		{"wronguser", "mypassword", ErrInvalidAccessKeyID},
+		{"", "mypassword", ErrInvalidAccessKeyID},
+		{"admin", "mypassword", ErrNone},
+	}
+
+	for i, testCase := range testCases {
+		req := mustNewRequest("GET", "http://localhost:9000/", 0, nil, t)
+		if err := signRequestV4(req, testCase.AccessKey, testCase.SecretKey); err != nil {
+			t.Fatalf("Unable to inititalized new signed http request %s", err)
+		}
+		_, _, _, s3Error := validateAdminSignature(ctx, req, globalMinioDefaultRegion)
+		if s3Error != testCase.ErrCode {
+			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i+1, testCase.ErrCode, s3Error)
+		}
+	}
+}

cmd/background-heal-ops.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"path"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
@@ -88,6 +89,7 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 			case bucket != "" && object != "":
 				res, err = objAPI.HealObject(ctx, bucket, object, task.opts)
 			}
+			ObjectPathUpdated(path.Join(bucket, object))
 			if task.responseCh != nil {
 				task.responseCh <- healResult{result: res, err: err}
 			}

cmd/benchmark-utils_test.go

@@ -28,11 +28,6 @@ import (
 	humanize "github.com/dustin/go-humanize"
 )
 
-// Prepare XL/FS backend for benchmark.
-func prepareBenchmarkBackend(instanceType string) (ObjectLayer, []string, error) {
-	return prepareTestBackend(instanceType)
-}
-
 // Benchmark utility functions for ObjectLayer.PutObject().
 // Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark.
 func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
@@ -135,7 +130,9 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 // creates XL/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function.
 func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
 	// create a temp XL/FS backend.
-	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	objLayer, disks, err := prepareTestBackend(ctx, instanceType)
 	if err != nil {
 		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
 	}
@@ -149,7 +146,9 @@ func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
 // creates XL/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function.
 func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
 	// create a temp XL/FS backend.
-	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	objLayer, disks, err := prepareTestBackend(ctx, instanceType)
 	if err != nil {
 		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
 	}
@@ -163,7 +162,9 @@ func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
 // creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for put object.
 func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int) {
 	// create a temp XL/FS backend.
-	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	objLayer, disks, err := prepareTestBackend(ctx, instanceType)
 	if err != nil {
 		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
 	}
@@ -242,7 +243,9 @@ func generateBytesData(size int) []byte {
 // creates XL/FS backend setup, obtains the object layer and calls the runGetObjectBenchmark function.
 func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
 	// create a temp XL/FS backend.
-	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	objLayer, disks, err := prepareTestBackend(ctx, instanceType)
 	if err != nil {
 		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
 	}
@@ -256,7 +259,9 @@ func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
 // creates XL/FS backend setup, obtains the object layer and runs parallel benchmark for ObjectLayer.GetObject() .
 func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int) {
 	// create a temp XL/FS backend.
-	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	objLayer, disks, err := prepareTestBackend(ctx, instanceType)
 	if err != nil {
 		b.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
 	}

cmd/bitrot-streaming.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"encoding/hex"
 	"fmt"
 	"hash"
@@ -142,7 +141,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
 		err := &errHashMismatch{fmt.Sprintf("hashes do not match expected %s, got %s",
 			hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))}
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(GlobalContext, err)
 		return 0, err
 	}
 	b.currOffset += int64(len(buf))

cmd/bitrot-whole.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"hash"
 	"io"
 
@@ -36,12 +35,12 @@ type wholeBitrotWriter struct {
 func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
 	err := b.disk.AppendFile(b.volume, b.filePath, p)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(GlobalContext, err)
 		return 0, err
 	}
 	_, err = b.Hash.Write(p)
 	if err != nil {
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(GlobalContext, err)
 		return 0, err
 	}
 	return len(p), nil
@@ -70,14 +69,14 @@ func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error)
 	if b.buf == nil {
 		b.buf = make([]byte, b.tillOffset-offset)
 		if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
-			ctx := context.Background()
+			ctx := GlobalContext
 			logger.GetReqInfo(ctx).AppendTags("disk", b.disk.String())
 			logger.LogIf(ctx, err)
 			return 0, err
 		}
 	}
 	if len(b.buf) < len(buf) {
-		logger.LogIf(context.Background(), errLessData)
+		logger.LogIf(GlobalContext, errLessData)
 		return 0, errLessData
 	}
 	n = copy(buf, b.buf)

cmd/bitrot.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"errors"
 	"hash"
 	"io"
@@ -72,7 +71,7 @@ func (a BitrotAlgorithm) New() hash.Hash {
 		hh, _ := highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit
 		return hh
 	default:
-		logger.CriticalIf(context.Background(), errors.New("Unsupported bitrot algorithm"))
+		logger.CriticalIf(GlobalContext, errors.New("Unsupported bitrot algorithm"))
 		return nil
 	}
 }
@@ -88,7 +87,7 @@ func (a BitrotAlgorithm) Available() bool {
 func (a BitrotAlgorithm) String() string {
 	name, ok := bitrotAlgorithms[a]
 	if !ok {
-		logger.CriticalIf(context.Background(), errors.New("Unsupported bitrot algorithm"))
+		logger.CriticalIf(GlobalContext, errors.New("Unsupported bitrot algorithm"))
 	}
 	return name
 }

cmd/bootstrap-peer-server.go

@@ -126,7 +126,7 @@ func (client *bootstrapRESTClient) reConnect() {
 // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
 func (client *bootstrapRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
-	return client.callWithContext(context.Background(), method, values, body, length)
+	return client.callWithContext(GlobalContext, method, values, body, length)
 }
 
 // Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
@@ -247,7 +247,7 @@ func newBootstrapRESTClient(endpoint Endpoint) (*bootstrapRESTClient, error) {
 		}
 	}
 
-	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout, rest.DefaultRESTTimeout)
+	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
 	restClient, err := rest.NewClient(serverURL, trFn, newAuthToken)
 	if err != nil {
 		return nil, err

cmd/bucket-encryption.go

@@ -44,7 +44,7 @@ func NewBucketSSEConfigSys() *BucketSSEConfigSys {
 // load - Loads the bucket encryption configuration for the given list of buckets
 func (sys *BucketSSEConfigSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
 	for _, bucket := range buckets {
-		config, err := objAPI.GetBucketSSEConfig(context.Background(), bucket.Name)
+		config, err := objAPI.GetBucketSSEConfig(GlobalContext, bucket.Name)
 		if err != nil {
 			if _, ok := err.(BucketSSEConfigNotFound); ok {
 				sys.Remove(bucket.Name)
@@ -81,7 +81,7 @@ func (sys *BucketSSEConfigSys) Get(bucket string) (config bucketsse.BucketSSECon
 			return
 		}
 
-		cfg, err := objAPI.GetBucketSSEConfig(context.Background(), bucket)
+		cfg, err := objAPI.GetBucketSSEConfig(GlobalContext, bucket)
 		if err != nil {
 			return
 		}
@@ -130,7 +130,7 @@ func saveBucketSSEConfig(ctx context.Context, objAPI ObjectLayer, bucket string,
 func getBucketSSEConfig(objAPI ObjectLayer, bucket string) (*bucketsse.BucketSSEConfig, error) {
 	// Path to bucket-encryption.xml for the given bucket.
 	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
-	configData, err := readConfig(context.Background(), objAPI, configFile)
+	configData, err := readConfig(GlobalContext, objAPI, configFile)
 	if err != nil {
 		if err == errConfigNotFound {
 			err = BucketSSEConfigNotFound{Bucket: bucket}

cmd/bucket-handlers.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"encoding/base64"
 	"encoding/xml"
 	"fmt"
@@ -70,7 +69,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	// Get buckets in the DNS
 	dnsBuckets, err := globalDNSConfig.List()
 	if err != nil && err != dns.ErrNoEntriesFound {
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(GlobalContext, err)
 		return
 	}
 
@@ -118,12 +117,12 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 
 	for _, err := range g.Wait() {
 		if err != nil {
-			logger.LogIf(context.Background(), err)
+			logger.LogIf(GlobalContext, err)
 		}
 	}
 
 	for _, bucket := range bucketsInConflict.ToSlice() {
-		logger.LogIf(context.Background(), fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", bucket, globalDomainIPs.ToSlice()))
+		logger.LogIf(GlobalContext, fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket. Use one of these IP addresses %v to access the bucket", bucket, globalDomainIPs.ToSlice()))
 	}
 
 	// Remove buckets that are in DNS for this server, but aren't local
@@ -140,7 +139,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 		// We go to here, so we know the bucket no longer exists,
 		// but is registered in DNS to this server
 		if err = globalDNSConfig.Delete(bucket); err != nil {
-			logger.LogIf(context.Background(), fmt.Errorf("Failed to remove DNS entry for %s due to %w",
+			logger.LogIf(GlobalContext, fmt.Errorf("Failed to remove DNS entry for %s due to %w",
 				bucket, err))
 		}
 	}
@@ -304,7 +303,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 
 		// err will be nil here as we already called this function
 		// earlier in this request.
-		claims, _ := getClaimsFromToken(r)
+		claims, _ := getClaimsFromToken(r, getSessionToken(r))
 		n := 0
 		// Use the following trick to filter in place
 		// https://github.com/golang/go/wiki/SliceTricks#filter-in-place
@@ -399,11 +398,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			}
 			continue
 		}
-		govBypassPerms := checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName)
-		if _, err := enforceRetentionBypassForDelete(ctx, r, bucket, object.ObjectName, getObjectInfoFn, govBypassPerms); err != ErrNone {
-			dErrs[index] = err
-			continue
+
+		if _, ok := globalBucketObjectLockConfig.Get(bucket); ok {
+			if err := enforceRetentionBypassForDelete(ctx, r, bucket, object.ObjectName, getObjectInfoFn); err != ErrNone {
+				dErrs[index] = err
+				continue
+			}
 		}
+
 		// Avoid duplicate objects, we use map to filter them out.
 		if _, ok := objectsToDelete[object.ObjectName]; !ok {
 			objectsToDelete[object.ObjectName] = index
@@ -753,7 +755,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 	rawReader := hashReader
 	pReader := NewPutObjReader(rawReader, nil, nil)
-	var objectEncryptionKey []byte
+	var objectEncryptionKey crypto.ObjectKey
 
 	// Check if bucket encryption is enabled
 	_, encEnabled := globalBucketSSEConfigSys.Get(bucket)
@@ -795,7 +797,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
-			pReader = NewPutObjReader(rawReader, hashReader, objectEncryptionKey)
+			pReader = NewPutObjReader(rawReader, hashReader, &objectEncryptionKey)
 		}
 	}
 
@@ -888,9 +890,15 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	forceDelete := false
-	if vs, found := r.Header[http.CanonicalHeaderKey("x-minio-force-delete")]; found {
-		switch strings.ToLower(strings.Join(vs, "")) {
+	if value := r.Header.Get(xhttp.MinIOForceDelete); value != "" {
+		switch value {
 		case "true":
 			forceDelete = true
 		case "false":
@@ -900,14 +908,20 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		}
 	}
 
-	objectAPI := api.ObjectAPI()
-	if objectAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
-		return
+	if forceDelete {
+		if s3Error := checkRequestAuthType(ctx, r, policy.ForceDeleteBucketAction, bucket, ""); s3Error != ErrNone {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+			return
+		}
+	} else {
+		if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+			return
+		}
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+	if _, ok := globalBucketObjectLockConfig.Get(bucket); ok && forceDelete {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -1007,11 +1021,6 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
-		return
-	}
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketObjectLockConfigurationAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
@@ -1089,7 +1098,7 @@ func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWri
 	if err != nil {
 		var aerr APIError
 		if err == errConfigNotFound {
-			aerr = errorCodes.ToAPIErr(ErrMethodNotAllowed)
+			aerr = errorCodes.ToAPIErr(ErrObjectLockConfigurationNotFound)
 		} else {
 			aerr = toAPIError(ctx, err)
 		}

cmd/bucket-handlers_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"encoding/xml"
 	"io/ioutil"
 	"net/http"
@@ -36,7 +35,7 @@ func TestRemoveBucketHandler(t *testing.T) {
 
 func testRemoveBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
 	credentials auth.Credentials, t *testing.T) {
-	_, err := obj.PutObject(context.Background(), bucketName, "test-object", mustGetPutObjReader(t, bytes.NewBuffer([]byte{}), int64(0), "", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), ObjectOptions{})
+	_, err := obj.PutObject(GlobalContext, bucketName, "test-object", mustGetPutObjReader(t, bytes.NewBuffer([]byte{}), int64(0), "", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), ObjectOptions{})
 	// if object upload fails stop the test.
 	if err != nil {
 		t.Fatalf("Error uploading object: <ERROR> %v", err)
@@ -670,7 +669,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := 0; i < 10; i++ {
 		objectName := "test-object-" + strconv.Itoa(i)
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
+		_, err = obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object %d:  Error uploading object: <ERROR> %v", i, err)

cmd/bucket-notification-handlers.go

@@ -22,7 +22,6 @@ import (
 	"errors"
 	"io"
 	"net/http"
-	"net/url"
 	"path"
 	"reflect"
 	"time"
@@ -116,7 +115,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 			// With newer config disallowing changing / turning off
 			// notification targets without removing ARN in notification
 			// configuration we won't see this problem anymore.
-			if reflect.DeepEqual(queue.ARN, arnErr.ARN) {
+			if reflect.DeepEqual(queue.ARN, arnErr.ARN) && i < len(config.QueueList) {
 				config.QueueList = append(config.QueueList[:i],
 					config.QueueList[i+1:]...)
 			}
@@ -282,16 +281,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	w.Header().Set(xhttp.ContentType, "text/event-stream")
 
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
 	// Listen Publisher and peer-listen-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
 	listenCh := make(chan interface{}, 4000)
 
-	peers := getRestClients(globalEndpoints)
+	peers := newPeerRestClients(globalEndpoints)
 
-	globalHTTPListen.Subscribe(listenCh, doneCh, func(evI interface{}) bool {
+	globalHTTPListen.Subscribe(listenCh, ctx.Done(), func(evI interface{}) bool {
 		ev, ok := evI.(event.Event)
 		if !ok {
 			return false
@@ -299,18 +295,14 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		if ev.S3.Bucket.Name != values.Get(peerRESTListenBucket) {
 			return false
 		}
-		objectName, uerr := url.QueryUnescape(ev.S3.Object.Key)
-		if uerr != nil {
-			objectName = ev.S3.Object.Key
-		}
-		return len(rulesMap.Match(ev.EventName, objectName).ToSlice()) != 0
+		return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key)
 	})
 
 	for _, peer := range peers {
 		if peer == nil {
 			continue
 		}
-		peer.Listen(listenCh, doneCh, values)
+		peer.Listen(listenCh, ctx.Done(), values)
 	}
 
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
@@ -336,7 +328,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 				return
 			}
 			w.(http.Flusher).Flush()
-		case <-GlobalServiceDoneCh:
+		case <-ctx.Done():
 			return
 		}
 	}

cmd/bucket-policy-handlers_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -103,7 +102,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	credentials auth.Credentials, t *testing.T) {
 
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
-	if err := obj.MakeBucketWithLocation(context.Background(), bucketName1, ""); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, ""); err != nil {
 		t.Fatal(err)
 	}
 

cmd/common-main.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"crypto/x509"
 	"encoding/gob"
 	"errors"
@@ -42,7 +41,7 @@ func init() {
 	logger.RegisterError(config.FmtError)
 
 	// Initialize globalConsoleSys system
-	globalConsoleSys = NewConsoleLogger(context.Background())
+	globalConsoleSys = NewConsoleLogger(GlobalContext)
 	logger.AddTarget(globalConsoleSys)
 
 	gob.Register(StorageErr(""))
@@ -150,6 +149,12 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 		globalCLIContext.Addr = ctx.String("address")
 	}
 
+	// Check "no-compat" flag from command line argument.
+	globalCLIContext.StrictS3Compat = true
+	if ctx.IsSet("no-compat") || ctx.GlobalIsSet("no-compat") {
+		globalCLIContext.StrictS3Compat = false
+	}
+
 	// Set all config, certs and CAs directories.
 	var configSet, certsSet bool
 	globalConfigDir, configSet = newConfigDirFromCtx(ctx, "config-dir", defaultConfigDir.Get)
@@ -165,13 +170,17 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	globalCertsCADir = &ConfigDir{path: filepath.Join(globalCertsDir.Get(), certsCADir)}
 
 	logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get())
-
-	// Check "compat" flag from command line argument.
-	globalCLIContext.StrictS3Compat = ctx.IsSet("compat") || ctx.GlobalIsSet("compat")
 }
 
 func handleCommonEnvVars() {
-	var err error
+	wormEnabled, err := config.LookupWorm()
+	if err != nil {
+		logger.Fatal(config.ErrInvalidWormValue(err), "Invalid worm configuration")
+	}
+	if wormEnabled {
+		logger.Fatal(errors.New("WORM is deprecated"), "global MINIO_WORM support is removed, please downgrade your server or migrate to https://github.com/minio/minio/tree/master/docs/retention")
+	}
+
 	globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.EnableOn))
 	if err != nil {
 		logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
@@ -237,12 +246,6 @@ func handleCommonEnvVars() {
 		os.Unsetenv(config.EnvAccessKeyOld)
 		os.Unsetenv(config.EnvSecretKeyOld)
 	}
-
-	globalWORMEnabled, err = config.LookupWorm()
-	if err != nil {
-		logger.Fatal(config.ErrInvalidWormValue(err), "Invalid worm configuration")
-	}
-
 }
 
 func logStartupMessage(msg string) {

cmd/config-common.go

@@ -50,7 +50,11 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
 }
 
 func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
-	return objAPI.DeleteObject(ctx, minioMetaBucket, configFile)
+	err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile)
+	if err != nil && isErrObjectNotFound(err) {
+		return errConfigNotFound
+	}
+	return err
 }
 
 func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {

cmd/config-current.go

@@ -17,12 +17,12 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
 	"strings"
 	"sync"
 
 	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/api"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
 	"github.com/minio/minio/cmd/config/etcd"
@@ -49,6 +49,7 @@ func initHelp() {
 		config.IdentityOpenIDSubSys: openid.DefaultKVS,
 		config.PolicyOPASubSys:      opa.DefaultKVS,
 		config.RegionSubSys:         config.DefaultRegionKVS,
+		config.APISubSys:            api.DefaultKVS,
 		config.CredentialsSubSys:    config.DefaultCredentialKVS,
 		config.KmsVaultSubSys:       crypto.DefaultVaultKVS,
 		config.KmsKesSubSys:         crypto.DefaultKesKVS,
@@ -101,6 +102,10 @@ func initHelp() {
 			Key:         config.KmsKesSubSys,
 			Description: "enable external MinIO key encryption service",
 		},
+		config.HelpKV{
+			Key:         config.APISubSys,
+			Description: "manage global HTTP API call specific features, such as throttling, authentication types, etc.",
+		},
 		config.HelpKV{
 			Key:             config.LoggerWebhookSubSys,
 			Description:     "send server logs to webhook endpoints",
@@ -175,6 +180,7 @@ func initHelp() {
 	var helpMap = map[string]config.HelpKVS{
 		"":                          helpSubSys, // Help for all sub-systems.
 		config.RegionSubSys:         config.RegionHelp,
+		config.APISubSys:            api.Help,
 		config.StorageClassSubSys:   storageclass.Help,
 		config.EtcdSubSys:           etcd.Help,
 		config.CacheSubSys:          cache.Help,
@@ -222,6 +228,10 @@ func validateConfig(s config.Config) error {
 		return err
 	}
 
+	if _, err := api.LookupConfig(s[config.APISubSys][config.Default]); err != nil {
+		return err
+	}
+
 	if globalIsXL {
 		if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],
 			globalXLSetDriveCount); err != nil {
@@ -297,12 +307,12 @@ func validateConfig(s config.Config) error {
 		return err
 	}
 
-	return notify.TestNotificationTargets(s, GlobalServiceDoneCh, NewGatewayHTTPTransport(),
+	return notify.TestNotificationTargets(s, GlobalContext.Done(), NewGatewayHTTPTransport(),
 		globalNotificationSys.ConfiguredTargetIDs())
 }
 
 func lookupConfigs(s config.Config) {
-	ctx := context.Background()
+	ctx := GlobalContext
 
 	var err error
 	if !globalActiveCred.IsValid() {
@@ -350,6 +360,18 @@ func lookupConfigs(s config.Config) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid region configuration: %w", err))
 	}
 
+	apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
+	}
+
+	apiRequestsMax := apiConfig.APIRequestsMax
+	if len(globalEndpoints.Hosts()) > 0 {
+		apiRequestsMax /= len(globalEndpoints.Hosts())
+	}
+
+	globalAPIThrottling.init(apiRequestsMax, apiConfig.APIRequestsDeadline)
+
 	if globalIsXL {
 		globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],
 			globalXLSetDriveCount)
@@ -447,12 +469,12 @@ func lookupConfigs(s config.Config) {
 		}
 	}
 
-	globalConfigTargetList, err = notify.GetNotificationTargets(s, GlobalServiceDoneCh, NewGatewayHTTPTransport())
+	globalConfigTargetList, err = notify.GetNotificationTargets(s, GlobalContext.Done(), NewGatewayHTTPTransport())
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
 
-	globalEnvTargetList, err = notify.GetNotificationTargets(newServerConfig(), GlobalServiceDoneCh, NewGatewayHTTPTransport())
+	globalEnvTargetList, err = notify.GetNotificationTargets(newServerConfig(), GlobalContext.Done(), NewGatewayHTTPTransport())
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
@@ -553,11 +575,11 @@ func newSrvConfig(objAPI ObjectLayer) error {
 	globalServerConfigMu.Unlock()
 
 	// Save config into file.
-	return saveServerConfig(context.Background(), objAPI, globalServerConfig)
+	return saveServerConfig(GlobalContext, objAPI, globalServerConfig)
 }
 
 func getValidConfig(objAPI ObjectLayer) (config.Config, error) {
-	return readServerConfig(context.Background(), objAPI)
+	return readServerConfig(GlobalContext, objAPI)
 }
 
 // loadConfig - loads a new config from disk, overrides params

cmd/config-encrypted.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 	"unicode/utf8"
 
 	etcd "github.com/coreos/etcd/clientv3"
@@ -50,7 +51,6 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 
 	rquorum := InsufficientReadQuorum{}
 	wquorum := InsufficientWriteQuorum{}
-	bucketNotFound := BucketNotFound{}
 
 	for !stop {
 		select {
@@ -58,7 +58,7 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 			if encrypted, err = checkBackendEncrypted(objAPI); err != nil {
 				if errors.Is(err, errDiskNotFound) ||
 					errors.As(err, &rquorum) ||
-					errors.As(err, &bucketNotFound) {
+					isErrBucketNotFound(err) {
 					logger.Info("Waiting for config backend to be encrypted..")
 					continue
 				}
@@ -108,7 +108,7 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 				if errors.Is(err, errDiskNotFound) ||
 					errors.As(err, &rquorum) ||
 					errors.As(err, &wquorum) ||
-					errors.As(err, &bucketNotFound) {
+					isErrBucketNotFound(err) {
 					logger.Info("Waiting for config backend to be encrypted..")
 					continue
 				}
@@ -139,7 +139,7 @@ func checkBackendEtcdEncrypted(ctx context.Context, client *etcd.Client) (bool,
 }
 
 func checkBackendEncrypted(objAPI ObjectLayer) (bool, error) {
-	data, err := readConfig(context.Background(), objAPI, backendEncryptedFile)
+	data, err := readConfig(GlobalContext, objAPI, backendEncryptedFile)
 	if err != nil && err != errConfigNotFound {
 		return false, err
 	}
@@ -163,10 +163,7 @@ func decryptData(edata []byte, creds ...auth.Credentials) ([]byte, error) {
 	return data, err
 }
 
-func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
-	defer cancel()
-
+func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client) error {
 	encrypted, err := checkBackendEtcdEncrypted(ctx, client)
 	if err != nil {
 		return err
@@ -207,7 +204,10 @@ func migrateIAMConfigsEtcdToEncrypted(client *etcd.Client) error {
 		logger.Info("Attempting encryption of all IAM users and policies on etcd")
 	}
 
-	r, err := client.Get(ctx, minioConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
+	listCtx, cancel := context.WithTimeout(ctx, 1*time.Minute)
+	defer cancel()
+
+	r, err := client.Get(listCtx, minioConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
 	if err != nil {
 		return err
 	}
@@ -288,14 +288,14 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 		logger.Info("Attempting encryption of all config, IAM users and policies on MinIO backend")
 	}
 
-	err := saveConfig(context.Background(), objAPI, backendEncryptedFile, backendEncryptedMigrationIncomplete)
+	err := saveConfig(GlobalContext, objAPI, backendEncryptedFile, backendEncryptedMigrationIncomplete)
 	if err != nil {
 		return err
 	}
 
 	var marker string
 	for {
-		res, err := objAPI.ListObjects(context.Background(), minioMetaBucket,
+		res, err := objAPI.ListObjects(GlobalContext, minioMetaBucket,
 			minioConfigPrefix, marker, "", maxObjectList)
 		if err != nil {
 			return err
@@ -306,7 +306,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 				cencdata []byte
 			)
 
-			cdata, err = readConfig(context.Background(), objAPI, obj.Name)
+			cdata, err = readConfig(GlobalContext, objAPI, obj.Name)
 			if err != nil {
 				return err
 			}
@@ -339,7 +339,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 				return err
 			}
 
-			if err = saveConfig(context.Background(), objAPI, obj.Name, cencdata); err != nil {
+			if err = saveConfig(GlobalContext, objAPI, obj.Name, cencdata); err != nil {
 				return err
 			}
 		}
@@ -355,5 +355,5 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 		logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
 	}
 
-	return saveConfig(context.Background(), objAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
+	return saveConfig(GlobalContext, objAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
 }

cmd/config-migrate.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -2430,7 +2429,7 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	defer func() {
 		if err == nil {
 			if globalEtcdClient != nil {
-				deleteKeyEtcd(context.Background(), globalEtcdClient, configFile)
+				deleteKeyEtcd(GlobalContext, globalEtcdClient, configFile)
 			} else {
 				// Rename config.json to config.json.deprecated only upon
 				// success of this function.
@@ -2440,7 +2439,7 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	}()
 
 	// Verify if backend already has the file (after holding lock)
-	if err = checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
+	if err = checkConfig(GlobalContext, objAPI, configFile); err != errConfigNotFound {
 		return err
 	} // if errConfigNotFound proceed to migrate..
 
@@ -2466,7 +2465,7 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 		// Initialize the server config, if no config exists.
 		return newSrvConfig(objAPI)
 	}
-	return saveServerConfig(context.Background(), objAPI, config)
+	return saveServerConfig(GlobalContext, objAPI, config)
 }
 
 // Migrates '.minio.sys/config.json' to v33.
@@ -2502,7 +2501,7 @@ func migrateMinioSysConfig(objAPI ObjectLayer) error {
 }
 
 func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (bool, []byte, error) {
-	data, err := readConfig(context.Background(), objAPI, configFile)
+	data, err := readConfig(GlobalContext, objAPI, configFile)
 	if err != nil {
 		return false, nil, err
 	}
@@ -2548,7 +2547,7 @@ func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
 	cfg.Version = "28"
 	cfg.KMS = crypto.KMSConfig{}
 
-	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %w", err)
 	}
 
@@ -2575,7 +2574,7 @@ func migrateV28ToV29MinioSys(objAPI ObjectLayer) error {
 	}
 
 	cfg.Version = "29"
-	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %w", err)
 	}
 
@@ -2607,7 +2606,7 @@ func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
 	cfg.Compression.Extensions = strings.Split(compress.DefaultExtensions, config.ValueSeparator)
 	cfg.Compression.MimeTypes = strings.Split(compress.DefaultMimeTypes, config.ValueSeparator)
 
-	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘29’ to ‘30’. %w", err)
 	}
 
@@ -2642,7 +2641,7 @@ func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
 		AuthToken: "",
 	}
 
-	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘30’ to ‘31’. %w", err)
 	}
 
@@ -2672,7 +2671,7 @@ func migrateV31ToV32MinioSys(objAPI ObjectLayer) error {
 	cfg.Notify.NSQ = make(map[string]target.NSQArgs)
 	cfg.Notify.NSQ["1"] = target.NSQArgs{}
 
-	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘31’ to ‘32’. %w", err)
 	}
 
@@ -2700,7 +2699,7 @@ func migrateV32ToV33MinioSys(objAPI ObjectLayer) error {
 
 	cfg.Version = "33"
 
-	if err = saveServerConfig(context.Background(), objAPI, cfg); err != nil {
+	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from '32' to '33' . %w", err)
 	}
 
@@ -2777,7 +2776,7 @@ func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
 		notify.SetNotifyWebhook(newCfg, k, args)
 	}
 
-	if err = saveServerConfig(context.Background(), objAPI, newCfg); err != nil {
+	if err = saveServerConfig(GlobalContext, objAPI, newCfg); err != nil {
 		return err
 	}
 

cmd/config.go

@@ -186,16 +186,14 @@ func (sys *ConfigSys) Load(objAPI ObjectLayer) error {
 }
 
 // WatchConfigNASDisk - watches nas disk on periodic basis.
-func (sys *ConfigSys) WatchConfigNASDisk(objAPI ObjectLayer) {
+func (sys *ConfigSys) WatchConfigNASDisk(ctx context.Context, objAPI ObjectLayer) {
 	configInterval := globalRefreshIAMInterval
 	watchDisk := func() {
-		ticker := time.NewTicker(configInterval)
-		defer ticker.Stop()
 		for {
 			select {
-			case <-GlobalServiceDoneCh:
+			case <-ctx.Done():
 				return
-			case <-ticker.C:
+			case <-time.After(configInterval):
 				loadConfig(objAPI)
 			}
 		}

cmd/config/api/api.go

@@ -0,0 +1,91 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package api
+
+import (
+	"encoding/json"
+	"errors"
+	"strconv"
+	"time"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+)
+
+const (
+	apiRequestsMax      = "requests_max"
+	apiRequestsDeadline = "requests_deadline"
+)
+
+// DefaultKVS - default storage class config
+var (
+	DefaultKVS = config.KVS{
+		config.KV{
+			Key:   apiRequestsMax,
+			Value: "0",
+		},
+		config.KV{
+			Key:   apiRequestsDeadline,
+			Value: "10s",
+		},
+	}
+)
+
+// Config storage class configuration
+type Config struct {
+	APIRequestsMax      int           `json:"requests_max"`
+	APIRequestsDeadline time.Duration `json:"requests_deadline"`
+}
+
+// UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
+func (sCfg *Config) UnmarshalJSON(data []byte) error {
+	type Alias Config
+	aux := &struct {
+		*Alias
+	}{
+		Alias: (*Alias)(sCfg),
+	}
+	return json.Unmarshal(data, &aux)
+}
+
+// LookupConfig - lookup api config and override with valid environment settings if any.
+func LookupConfig(kvs config.KVS) (cfg Config, err error) {
+	if err = config.CheckValidKeys(config.APISubSys, kvs, DefaultKVS); err != nil {
+		return cfg, err
+	}
+
+	// Check environment variables parameters
+	requestsMax, err := strconv.Atoi(env.Get(config.EnvAPIRequestsMax, kvs.Get(apiRequestsMax)))
+	if err != nil {
+		return cfg, err
+	}
+
+	if requestsMax < 0 {
+		return cfg, errors.New("invalid API max requests value")
+	}
+
+	requestsDeadline, err := time.ParseDuration(env.Get(config.EnvAPIRequestsDeadline, kvs.Get(apiRequestsDeadline)))
+	if err != nil {
+		return cfg, err
+	}
+
+	cfg = Config{
+		APIRequestsMax:      requestsMax,
+		APIRequestsDeadline: requestsDeadline,
+	}
+	return cfg, nil
+}

cmd/config/api/help.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,24 +14,24 @@
  * limitations under the License.
  */
 
-package cpu
+package api
 
-import (
-	"syscall"
-	"time"
-	"unsafe"
+import "github.com/minio/minio/cmd/config"
 
-	"golang.org/x/sys/unix"
+// Help template for storageclass feature.
+var (
+	Help = config.HelpKVS{
+		config.HelpKV{
+			Key:         apiRequestsMax,
+			Description: `set the maximum number of concurrent requests, e.g. "1600"`,
+			Optional:    true,
+			Type:        "number",
+		},
+		config.HelpKV{
+			Key:         apiRequestsDeadline,
+			Description: `set the deadline for API requests waiting to be processed e.g. "1m"`,
+			Optional:    true,
+			Type:        "duration",
+		},
+	}
 )
-
-func newCounter() (counter, error) {
-	return counter{}, nil
-}
-
-func (c counter) now() time.Time {
-	var ts syscall.Timespec
-	// Retrieve Per-process CPU-time clock
-	syscall.Syscall(syscall.SYS_CLOCK_GETTIME, unix.CLOCK_PROCESS_CPUTIME_ID, uintptr(unsafe.Pointer(&ts)), 0)
-	sec, nsec := ts.Unix()
-	return time.Unix(sec, nsec)
-}

cmd/config/config.go

@@ -70,6 +70,7 @@ const (
 	RegionSubSys         = "region"
 	EtcdSubSys           = "etcd"
 	StorageClassSubSys   = "storage_class"
+	APISubSys            = "api"
 	CompressionSubSys    = "compression"
 	KmsVaultSubSys       = "kms_vault"
 	KmsKesSubSys         = "kms_kes"
@@ -101,6 +102,7 @@ var SubSystems = set.CreateStringSet([]string{
 	RegionSubSys,
 	EtcdSubSys,
 	CacheSubSys,
+	APISubSys,
 	StorageClassSubSys,
 	CompressionSubSys,
 	KmsVaultSubSys,
@@ -128,6 +130,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
 	RegionSubSys,
 	EtcdSubSys,
 	CacheSubSys,
+	APISubSys,
 	StorageClassSubSys,
 	CompressionSubSys,
 	KmsVaultSubSys,
@@ -196,6 +199,23 @@ func (kvs KVS) Empty() bool {
 	return len(kvs) == 0
 }
 
+// Keys returns the list of keys for the current KVS
+func (kvs KVS) Keys() []string {
+	var keys = make([]string, len(kvs))
+	var foundComment bool
+	for i := range kvs {
+		if kvs[i].Key == madmin.CommentKey {
+			foundComment = true
+		}
+		keys[i] = kvs[i].Key
+	}
+	// Comment KV not found, add it explicitly.
+	if !foundComment {
+		keys = append(keys, madmin.CommentKey)
+	}
+	return keys
+}
+
 func (kvs KVS) String() string {
 	var s strings.Builder
 	for _, kv := range kvs {
@@ -581,9 +601,20 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 		return Errorf("sub-system '%s' only supports single target", subSystemValue[0])
 	}
 
+	tgt := Default
+	subSys := subSystemValue[0]
+	if len(subSystemValue) == 2 {
+		tgt = subSystemValue[1]
+	}
+
+	fields := madmin.KvFields(inputs[1], defaultKVS[subSys].Keys())
+	if len(fields) == 0 {
+		return Errorf("sub-system '%s' cannot have empty keys", subSys)
+	}
+
 	var kvs = KVS{}
 	var prevK string
-	for _, v := range strings.Fields(inputs[1]) {
+	for _, v := range fields {
 		kv := strings.SplitN(v, KvSeparator, 2)
 		if len(kv) == 0 {
 			continue
@@ -604,12 +635,6 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 		return Errorf("key '%s', cannot have empty value", kv[0])
 	}
 
-	tgt := Default
-	subSys := subSystemValue[0]
-	if len(subSystemValue) == 2 {
-		tgt = subSystemValue[1]
-	}
-
 	_, ok := kvs.Lookup(Enable)
 	// Check if state is required
 	_, enableRequired := defaultKVS[subSys].Lookup(Enable)

cmd/config/config_test.go

@@ -17,7 +17,96 @@
 
 package config
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func TestKVFields(t *testing.T) {
+	tests := []struct {
+		input          string
+		keys           []string
+		expectedFields map[string]struct{}
+	}{
+		// No keys present
+		{
+			input:          "",
+			keys:           []string{"comment"},
+			expectedFields: map[string]struct{}{},
+		},
+		// No keys requested for tokenizing
+		{
+			input:          `comment="Hi this is my comment ="`,
+			keys:           []string{},
+			expectedFields: map[string]struct{}{},
+		},
+		// Single key requested and present
+		{
+			input:          `comment="Hi this is my comment ="`,
+			keys:           []string{"comment"},
+			expectedFields: map[string]struct{}{`comment="Hi this is my comment ="`: {}},
+		},
+		// Keys and input order of k=v is same.
+		{
+			input: `connection_string="host=localhost port=2832" comment="really long comment"`,
+			keys:  []string{"connection_string", "comment"},
+			expectedFields: map[string]struct{}{
+				`connection_string="host=localhost port=2832"`: {},
+				`comment="really long comment"`:                {},
+			},
+		},
+		// Keys with spaces in between
+		{
+			input: `enable=on format=namespace connection_string=" host=localhost port=5432 dbname = cesnietor sslmode=disable" table=holicrayoli`,
+			keys:  []string{"enable", "connection_string", "comment", "format", "table"},
+			expectedFields: map[string]struct{}{
+				`enable=on`:        {},
+				`format=namespace`: {},
+				`connection_string=" host=localhost port=5432 dbname = cesnietor sslmode=disable"`: {},
+				`table=holicrayoli`: {},
+			},
+		},
+		// One of the keys is not present and order of input has changed.
+		{
+			input: `comment="really long comment" connection_string="host=localhost port=2832"`,
+			keys:  []string{"connection_string", "comment", "format"},
+			expectedFields: map[string]struct{}{
+				`connection_string="host=localhost port=2832"`: {},
+				`comment="really long comment"`:                {},
+			},
+		},
+		// Incorrect delimiter, expected fields should be empty.
+		{
+			input:          `comment:"really long comment" connection_string:"host=localhost port=2832"`,
+			keys:           []string{"connection_string", "comment"},
+			expectedFields: map[string]struct{}{},
+		},
+		// Incorrect type of input v/s required keys.
+		{
+			input:          `comme="really long comment" connection_str="host=localhost port=2832"`,
+			keys:           []string{"connection_string", "comment"},
+			expectedFields: map[string]struct{}{},
+		},
+	}
+	for _, test := range tests {
+		test := test
+		t.Run("", func(t *testing.T) {
+			gotFields := madmin.KvFields(test.input, test.keys)
+			if len(gotFields) != len(test.expectedFields) {
+				t.Errorf("Expected keys %d, found %d", len(test.expectedFields), len(gotFields))
+			}
+			found := true
+			for _, field := range gotFields {
+				_, ok := test.expectedFields[field]
+				found = found && ok
+			}
+			if !found {
+				t.Errorf("Expected %s, got %s", test.expectedFields, gotFields)
+			}
+		})
+	}
+}
 
 func TestValidRegion(t *testing.T) {
 	tests := []struct {

cmd/config/errors.go

@@ -33,7 +33,7 @@ var (
 	ErrInvalidErasureSetSize = newErrFn(
 		"Invalid erasure set size",
 		"Please check the passed value",
-		"Erasure set can only accept any of [4, 6, 8, 10, 12, 14, 16] values",
+		"Erasure set can only accept any of [4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] values",
 	)
 
 	ErrInvalidWormValue = newErrFn(

cmd/config/identity/openid/ecdsa-sha3.go

@@ -0,0 +1,51 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package openid
+
+import (
+	"crypto"
+	"github.com/dgrijalva/jwt-go"
+	// Needed for SHA3 to work - See: https://golang.org/src/crypto/crypto.go?s=1034:1288
+	_ "golang.org/x/crypto/sha3"
+)
+
+// Specific instances for EC256 and company
+var (
+	SigningMethodES3256 *jwt.SigningMethodECDSA
+	SigningMethodES3384 *jwt.SigningMethodECDSA
+	SigningMethodES3512 *jwt.SigningMethodECDSA
+)
+
+func init() {
+	// ES256
+	SigningMethodES3256 = &jwt.SigningMethodECDSA{Name: "ES3256", Hash: crypto.SHA3_256, KeySize: 32, CurveBits: 256}
+	jwt.RegisterSigningMethod(SigningMethodES3256.Alg(), func() jwt.SigningMethod {
+		return SigningMethodES3256
+	})
+
+	// ES384
+	SigningMethodES3384 = &jwt.SigningMethodECDSA{Name: "ES3384", Hash: crypto.SHA3_384, KeySize: 48, CurveBits: 384}
+	jwt.RegisterSigningMethod(SigningMethodES3384.Alg(), func() jwt.SigningMethod {
+		return SigningMethodES3384
+	})
+
+	// ES512
+	SigningMethodES3512 = &jwt.SigningMethodECDSA{Name: "ES3512", Hash: crypto.SHA3_512, KeySize: 66, CurveBits: 521}
+	jwt.RegisterSigningMethod(SigningMethodES3512.Alg(), func() jwt.SigningMethod {
+		return SigningMethodES3512
+	})
+}

cmd/config/identity/openid/jwt.go

@@ -30,7 +30,7 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/env"
-	iampolicy "github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/iam/policy"
 	xnet "github.com/minio/minio/pkg/net"
 )
 
@@ -168,7 +168,10 @@ func updateClaimsExpiry(dsecs string, claims map[string]interface{}) error {
 // Validate - validates the access token.
 func (p *JWT) Validate(token, dsecs string) (map[string]interface{}, error) {
 	jp := new(jwtgo.Parser)
-	jp.ValidMethods = []string{"RS256", "RS384", "RS512", "ES256", "ES384", "ES512"}
+	jp.ValidMethods = []string{
+		"RS256", "RS384", "RS512", "ES256", "ES384", "ES512",
+		"RS3256", "RS3384", "RS3512", "ES3256", "ES3384", "ES3512",
+	}
 
 	keyFuncCallback := func(jwtToken *jwtgo.Token) (interface{}, error) {
 		kid, ok := jwtToken.Header["kid"].(string)

cmd/config/identity/openid/rsa-sha3.go

@@ -0,0 +1,51 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package openid
+
+import (
+	"crypto"
+	"github.com/dgrijalva/jwt-go"
+	// Needed for SHA3 to work - See: https://golang.org/src/crypto/crypto.go?s=1034:1288
+	_ "golang.org/x/crypto/sha3"
+)
+
+// Specific instances for RS256 and company
+var (
+	SigningMethodRS3256 *jwt.SigningMethodRSA
+	SigningMethodRS3384 *jwt.SigningMethodRSA
+	SigningMethodRS3512 *jwt.SigningMethodRSA
+)
+
+func init() {
+	// RS3256
+	SigningMethodRS3256 = &jwt.SigningMethodRSA{Name: "RS3256", Hash: crypto.SHA3_256}
+	jwt.RegisterSigningMethod(SigningMethodRS3256.Alg(), func() jwt.SigningMethod {
+		return SigningMethodRS3256
+	})
+
+	// RS3384
+	SigningMethodRS3384 = &jwt.SigningMethodRSA{Name: "RS3384", Hash: crypto.SHA3_384}
+	jwt.RegisterSigningMethod(SigningMethodRS3384.Alg(), func() jwt.SigningMethod {
+		return SigningMethodRS3384
+	})
+
+	// RS3512
+	SigningMethodRS3512 = &jwt.SigningMethodRSA{Name: "RS3512", Hash: crypto.SHA3_512}
+	jwt.RegisterSigningMethod(SigningMethodRS3512.Alg(), func() jwt.SigningMethod {
+		return SigningMethodRS3512
+	})
+}

cmd/config/notify/help.go

@@ -24,7 +24,7 @@ import (
 const (
 	formatComment     = `'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'`
 	queueDirComment   = `staging dir for undelivered messages e.g. '/home/events'`
-	queueLimitComment = `maximum limit for undelivered messages, defaults to '10000'`
+	queueLimitComment = `maximum limit for undelivered messages, defaults to '100000'`
 )
 
 // Help template inputs for all notification targets
@@ -297,7 +297,7 @@ var (
 	HelpPostgres = config.HelpKVS{
 		config.HelpKV{
 			Key:         target.PostgresConnectionString,
-			Description: "Postgres server connection-string",
+			Description: `Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable"`,
 			Type:        "string",
 		},
 		config.HelpKV{
@@ -310,36 +310,6 @@ var (
 			Description: formatComment,
 			Type:        "namespace*|access",
 		},
-		config.HelpKV{
-			Key:         target.PostgresHost,
-			Description: "Postgres server hostname (used only if `connection_string` is empty)",
-			Optional:    true,
-			Type:        "hostname",
-		},
-		config.HelpKV{
-			Key:         target.PostgresPort,
-			Description: "Postgres server port, defaults to `5432` (used only if `connection_string` is empty)",
-			Optional:    true,
-			Type:        "port",
-		},
-		config.HelpKV{
-			Key:         target.PostgresUsername,
-			Description: "database username (used only if `connection_string` is empty)",
-			Optional:    true,
-			Type:        "string",
-		},
-		config.HelpKV{
-			Key:         target.PostgresPassword,
-			Description: "database password (used only if `connection_string` is empty)",
-			Optional:    true,
-			Type:        "string",
-		},
-		config.HelpKV{
-			Key:         target.PostgresDatabase,
-			Description: "database name (used only if `connection_string` is empty)",
-			Optional:    true,
-			Type:        "string",
-		},
 		config.HelpKV{
 			Key:         target.PostgresQueueDir,
 			Description: queueDirComment,
@@ -363,7 +333,7 @@ var (
 	HelpMySQL = config.HelpKVS{
 		config.HelpKV{
 			Key:         target.MySQLDSNString,
-			Description: "MySQL data-source-name connection string",
+			Description: `MySQL data-source-name connection string e.g. "<user>:<password>@tcp(<host>:<port>)/<database>"`,
 			Optional:    true,
 			Type:        "string",
 		},
@@ -377,36 +347,6 @@ var (
 			Description: formatComment,
 			Type:        "namespace*|access",
 		},
-		config.HelpKV{
-			Key:         target.MySQLHost,
-			Description: "MySQL server hostname (used only if `dsn_string` is empty)",
-			Optional:    true,
-			Type:        "hostname",
-		},
-		config.HelpKV{
-			Key:         target.MySQLPort,
-			Description: "MySQL server port (used only if `dsn_string` is empty)",
-			Optional:    true,
-			Type:        "port",
-		},
-		config.HelpKV{
-			Key:         target.MySQLUsername,
-			Description: "database username (used only if `dsn_string` is empty)",
-			Optional:    true,
-			Type:        "string",
-		},
-		config.HelpKV{
-			Key:         target.MySQLPassword,
-			Description: "database password (used only if `dsn_string` is empty)",
-			Optional:    true,
-			Type:        "string",
-		},
-		config.HelpKV{
-			Key:         target.MySQLDatabase,
-			Description: "database name (used only if `dsn_string` is empty)",
-			Optional:    true,
-			Type:        "string",
-		},
 		config.HelpKV{
 			Key:         target.MySQLQueueDir,
 			Description: queueDirComment,

cmd/config/notify/legacy.go

@@ -1,3 +1,19 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package notify
 
 import (

cmd/config/notify/parse.go

@@ -17,8 +17,10 @@
 package notify
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"net/http"
 	"strconv"
 	"strings"
@@ -37,12 +39,16 @@ const (
 	formatAccess    = "access"
 )
 
+// ErrTargetsOffline - Indicates single/multiple target failures.
+var ErrTargetsOffline = errors.New("one or more targets are offline. Please use `mc admin info --json` to check the offline targets")
+
 // TestNotificationTargets is similar to GetNotificationTargets()
 // avoids explicit registration.
 func TestNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport,
 	targetIDs []event.TargetID) error {
 	test := true
-	targets, err := RegisterNotificationTargets(cfg, doneCh, transport, targetIDs, test)
+	returnOnTargetError := true
+	targets, err := RegisterNotificationTargets(cfg, doneCh, transport, targetIDs, test, returnOnTargetError)
 	if err == nil {
 		// Close all targets since we are only testing connections.
 		for _, t := range targets.TargetMap() {
@@ -57,7 +63,8 @@ func TestNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transpor
 // targets, returns error if any.
 func GetNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport) (*event.TargetList, error) {
 	test := false
-	return RegisterNotificationTargets(cfg, doneCh, transport, nil, test)
+	returnOnTargetError := false
+	return RegisterNotificationTargets(cfg, doneCh, transport, nil, test, returnOnTargetError)
 }
 
 // RegisterNotificationTargets - returns TargetList which contains enabled targets in serverConfig.
@@ -65,12 +72,40 @@ func GetNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport
 // * Add a new target in pkg/event/target package.
 // * Add newly added target configuration to serverConfig.Notify.<TARGET_NAME>.
 // * Handle the configuration in this function to create/add into TargetList.
-func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, targetIDs []event.TargetID, test bool) (_ *event.TargetList, err error) {
+func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, targetIDs []event.TargetID, test bool, returnOnTargetError bool) (*event.TargetList, error) {
+
+	targetList, err := FetchRegisteredTargets(cfg, doneCh, transport, test, returnOnTargetError)
+	if err != nil {
+		return targetList, err
+	}
+
+	if test {
+		// Verify if user is trying to disable already configured
+		// notification targets, based on their target IDs
+		for _, targetID := range targetIDs {
+			if !targetList.Exists(targetID) {
+				return nil, config.Errorf(
+					"Unable to disable configured targets '%v'",
+					targetID)
+			}
+		}
+	}
+
+	return targetList, nil
+}
+
+// FetchRegisteredTargets - Returns a set of configured TargetList
+// If `returnOnTargetError` is set to true, The function returns when a target initialization fails
+// Else, the function will return a complete TargetList irrespective of errors
+func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, test bool, returnOnTargetError bool) (_ *event.TargetList, err error) {
 	targetList := event.NewTargetList()
+	var targetsOffline bool
 
 	defer func() {
-		// Automatically close all connections to targets when an error occur
-		if err != nil {
+		// Automatically close all connections to targets when an error occur.
+		// Close all the targets if returnOnTargetError is set
+		// Else, close only the failed targets
+		if err != nil && returnOnTargetError {
 			for _, t := range targetList.TargetMap() {
 				_ = t.Close()
 			}
@@ -137,10 +172,18 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		}
 		newTarget, err := target.NewAMQPTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
-			return nil, err
+			targetsOffline = true
+			if returnOnTargetError {
+				return nil, err
+			}
+			_ = newTarget.Close()
 		}
+
 		if err = targetList.Add(newTarget); err != nil {
-			return nil, err
+			logger.LogIf(context.Background(), err)
+			if returnOnTargetError {
+				return nil, err
+			}
 		}
 	}
 
@@ -150,11 +193,17 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		}
 		newTarget, err := target.NewElasticsearchTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
-			return nil, err
-
+			targetsOffline = true
+			if returnOnTargetError {
+				return nil, err
+			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
-			return nil, err
+			logger.LogIf(context.Background(), err)
+			if returnOnTargetError {
+				return nil, err
+			}
 		}
 	}
 
@@ -165,10 +214,17 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		args.TLS.RootCAs = transport.TLSClientConfig.RootCAs
 		newTarget, err := target.NewKafkaTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
-			return nil, err
+			targetsOffline = true
+			if returnOnTargetError {
+				return nil, err
+			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
-			return nil, err
+			logger.LogIf(context.Background(), err)
+			if returnOnTargetError {
+				return nil, err
+			}
 		}
 	}
 
@@ -179,10 +235,17 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		args.RootCAs = transport.TLSClientConfig.RootCAs
 		newTarget, err := target.NewMQTTTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
-			return nil, err
+			targetsOffline = true
+			if returnOnTargetError {
+				return nil, err
+			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
-			return nil, err
+			logger.LogIf(context.Background(), err)
+			if returnOnTargetError {
+				return nil, err
+			}
 		}
 	}
 
@@ -192,10 +255,17 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		}
 		newTarget, err := target.NewMySQLTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
-			return nil, err
+			targetsOffline = true
+			if returnOnTargetError {
+				return nil, err
+			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
-			return nil, err
+			logger.LogIf(context.Background(), err)
+			if returnOnTargetError {
+				return nil, err
+			}
 		}
 	}
 
@@ -205,10 +275,17 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		}
 		newTarget, err := target.NewNATSTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
-			return nil, err
+			targetsOffline = true
+			if returnOnTargetError {
+				return nil, err
+			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
-			return nil, err
+			logger.LogIf(context.Background(), err)
+			if returnOnTargetError {
+				return nil, err
+			}
 		}
 	}
 
@@ -218,10 +295,17 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		}
 		newTarget, err := target.NewNSQTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
-			return nil, err
+			targetsOffline = true
+			if returnOnTargetError {
+				return nil, err
+			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
-			return nil, err
+			logger.LogIf(context.Background(), err)
+			if returnOnTargetError {
+				return nil, err
+			}
 		}
 	}
 
@@ -231,10 +315,17 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		}
 		newTarget, err := target.NewPostgreSQLTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
-			return nil, err
+			targetsOffline = true
+			if returnOnTargetError {
+				return nil, err
+			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
-			return nil, err
+			logger.LogIf(context.Background(), err)
+			if returnOnTargetError {
+				return nil, err
+			}
 		}
 	}
 
@@ -244,10 +335,17 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		}
 		newTarget, err := target.NewRedisTarget(id, args, doneCh, logger.LogOnceIf, test)
 		if err != nil {
-			return nil, err
+			targetsOffline = true
+			if returnOnTargetError {
+				return nil, err
+			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
-			return nil, err
+			logger.LogIf(context.Background(), err)
+			if returnOnTargetError {
+				return nil, err
+			}
 		}
 	}
 
@@ -257,23 +355,22 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 		}
 		newTarget, err := target.NewWebhookTarget(id, args, doneCh, logger.LogOnceIf, transport, test)
 		if err != nil {
-			return nil, err
+			targetsOffline = true
+			if returnOnTargetError {
+				return nil, err
+			}
+			_ = newTarget.Close()
 		}
-		if err := targetList.Add(newTarget); err != nil {
-			return nil, err
+		if err = targetList.Add(newTarget); err != nil {
+			logger.LogIf(context.Background(), err)
+			if returnOnTargetError {
+				return nil, err
+			}
 		}
 	}
 
-	if test {
-		// Verify if user is trying to disable already configured
-		// notification targets, based on their target IDs
-		for _, targetID := range targetIDs {
-			if !targetList.Exists(targetID) {
-				return nil, config.Errorf(
-					"Unable to disable configured targets '%v'",
-					targetID)
-			}
-		}
+	if targetsOffline {
+		return targetList, ErrTargetsOffline
 	}
 
 	return targetList, nil
@@ -306,8 +403,10 @@ func checkValidNotificationKeys(cfg config.Config) error {
 			if tname != config.Default {
 				subSysTarget = subSys + config.SubSystemSeparator + tname
 			}
-			if err := config.CheckValidKeys(subSysTarget, kv, validKVS); err != nil {
-				return err
+			if v, ok := kv.Lookup(config.Enable); ok && v == config.EnableOn {
+				if err := config.CheckValidKeys(subSysTarget, kv, validKVS); err != nil {
+					return err
+				}
 			}
 		}
 	}
@@ -696,26 +795,6 @@ var (
 			Key:   target.MySQLFormat,
 			Value: formatNamespace,
 		},
-		config.KV{
-			Key:   target.MySQLHost,
-			Value: "",
-		},
-		config.KV{
-			Key:   target.MySQLPort,
-			Value: "",
-		},
-		config.KV{
-			Key:   target.MySQLUsername,
-			Value: "",
-		},
-		config.KV{
-			Key:   target.MySQLPassword,
-			Value: "",
-		},
-		config.KV{
-			Key:   target.MySQLDatabase,
-			Value: "",
-		},
 		config.KV{
 			Key:   target.MySQLDSNString,
 			Value: "",
@@ -752,16 +831,6 @@ func GetNotifyMySQL(mysqlKVS map[string]config.KVS) (map[string]target.MySQLArgs
 			continue
 		}
 
-		hostEnv := target.EnvMySQLHost
-		if k != config.Default {
-			hostEnv = hostEnv + config.Default + k
-		}
-
-		host, err := xnet.ParseURL(env.Get(hostEnv, kv.Get(target.MySQLHost)))
-		if err != nil {
-			return nil, err
-		}
-
 		queueLimitEnv := target.EnvMySQLQueueLimit
 		if k != config.Default {
 			queueLimitEnv = queueLimitEnv + config.Default + k
@@ -775,30 +844,17 @@ func GetNotifyMySQL(mysqlKVS map[string]config.KVS) (map[string]target.MySQLArgs
 		if k != config.Default {
 			formatEnv = formatEnv + config.Default + k
 		}
+
 		dsnStringEnv := target.EnvMySQLDSNString
 		if k != config.Default {
 			dsnStringEnv = dsnStringEnv + config.Default + k
 		}
+
 		tableEnv := target.EnvMySQLTable
 		if k != config.Default {
 			tableEnv = tableEnv + config.Default + k
 		}
-		portEnv := target.EnvMySQLPort
-		if k != config.Default {
-			portEnv = portEnv + config.Default + k
-		}
-		usernameEnv := target.EnvMySQLUsername
-		if k != config.Default {
-			usernameEnv = usernameEnv + config.Default + k
-		}
-		passwordEnv := target.EnvMySQLPassword
-		if k != config.Default {
-			passwordEnv = passwordEnv + config.Default + k
-		}
-		databaseEnv := target.EnvMySQLDatabase
-		if k != config.Default {
-			databaseEnv = databaseEnv + config.Default + k
-		}
+
 		queueDirEnv := target.EnvMySQLQueueDir
 		if k != config.Default {
 			queueDirEnv = queueDirEnv + config.Default + k
@@ -808,11 +864,6 @@ func GetNotifyMySQL(mysqlKVS map[string]config.KVS) (map[string]target.MySQLArgs
 			Format:     env.Get(formatEnv, kv.Get(target.MySQLFormat)),
 			DSN:        env.Get(dsnStringEnv, kv.Get(target.MySQLDSNString)),
 			Table:      env.Get(tableEnv, kv.Get(target.MySQLTable)),
-			Host:       *host,
-			Port:       env.Get(portEnv, kv.Get(target.MySQLPort)),
-			User:       env.Get(usernameEnv, kv.Get(target.MySQLUsername)),
-			Password:   env.Get(passwordEnv, kv.Get(target.MySQLPassword)),
-			Database:   env.Get(databaseEnv, kv.Get(target.MySQLDatabase)),
 			QueueDir:   env.Get(queueDirEnv, kv.Get(target.MySQLQueueDir)),
 			QueueLimit: queueLimit,
 		}
@@ -1180,26 +1231,6 @@ var (
 			Key:   target.PostgresTable,
 			Value: "",
 		},
-		config.KV{
-			Key:   target.PostgresHost,
-			Value: "",
-		},
-		config.KV{
-			Key:   target.PostgresPort,
-			Value: "",
-		},
-		config.KV{
-			Key:   target.PostgresUsername,
-			Value: "",
-		},
-		config.KV{
-			Key:   target.PostgresPassword,
-			Value: "",
-		},
-		config.KV{
-			Key:   target.PostgresDatabase,
-			Value: "",
-		},
 		config.KV{
 			Key:   target.PostgresQueueDir,
 			Value: "",
@@ -1228,16 +1259,6 @@ func GetNotifyPostgres(postgresKVS map[string]config.KVS) (map[string]target.Pos
 			continue
 		}
 
-		hostEnv := target.EnvPostgresHost
-		if k != config.Default {
-			hostEnv = hostEnv + config.Default + k
-		}
-
-		host, err := xnet.ParseHost(env.Get(hostEnv, kv.Get(target.PostgresHost)))
-		if err != nil {
-			return nil, err
-		}
-
 		queueLimitEnv := target.EnvPostgresQueueLimit
 		if k != config.Default {
 			queueLimitEnv = queueLimitEnv + config.Default + k
@@ -1263,26 +1284,6 @@ func GetNotifyPostgres(postgresKVS map[string]config.KVS) (map[string]target.Pos
 			tableEnv = tableEnv + config.Default + k
 		}
 
-		portEnv := target.EnvPostgresPort
-		if k != config.Default {
-			portEnv = portEnv + config.Default + k
-		}
-
-		usernameEnv := target.EnvPostgresUsername
-		if k != config.Default {
-			usernameEnv = usernameEnv + config.Default + k
-		}
-
-		passwordEnv := target.EnvPostgresPassword
-		if k != config.Default {
-			passwordEnv = passwordEnv + config.Default + k
-		}
-
-		databaseEnv := target.EnvPostgresDatabase
-		if k != config.Default {
-			databaseEnv = databaseEnv + config.Default + k
-		}
-
 		queueDirEnv := target.EnvPostgresQueueDir
 		if k != config.Default {
 			queueDirEnv = queueDirEnv + config.Default + k
@@ -1293,11 +1294,6 @@ func GetNotifyPostgres(postgresKVS map[string]config.KVS) (map[string]target.Pos
 			Format:           env.Get(formatEnv, kv.Get(target.PostgresFormat)),
 			ConnectionString: env.Get(connectionStringEnv, kv.Get(target.PostgresConnectionString)),
 			Table:            env.Get(tableEnv, kv.Get(target.PostgresTable)),
-			Host:             *host,
-			Port:             env.Get(portEnv, kv.Get(target.PostgresPort)),
-			User:             env.Get(usernameEnv, kv.Get(target.PostgresUsername)),
-			Password:         env.Get(passwordEnv, kv.Get(target.PostgresPassword)),
-			Database:         env.Get(databaseEnv, kv.Get(target.PostgresDatabase)),
 			QueueDir:         env.Get(queueDirEnv, kv.Get(target.PostgresQueueDir)),
 			QueueLimit:       uint64(queueLimit),
 		}

cmd/consolelogger.go

@@ -74,7 +74,7 @@ func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
 }
 
 // Subscribe starts console logging for this node.
-func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh chan struct{}, node string, last int, logKind string, filter func(entry interface{}) bool) {
+func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan struct{}, node string, last int, logKind string, filter func(entry interface{}) bool) {
 	// Enable console logging for remote client.
 	if !sys.HasLogListeners() {
 		logger.AddTarget(sys)

cmd/daily-lifecycle-ops.go

@@ -63,6 +63,8 @@ func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 			continue
 		}
 
+		_, bucketHasLockConfig := globalBucketObjectLockConfig.Get(bucket.Name)
+
 		// Calculate the common prefix of all lifecycle rules
 		var prefixes []string
 		for _, rule := range l.Rules {
@@ -85,9 +87,11 @@ func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 					// Reached maximum delete requests, attempt a delete for now.
 					break
 				}
-
 				// Find the action that need to be executed
 				if l.ComputeAction(obj.Name, obj.UserTags, obj.ModTime) == lifecycle.DeleteAction {
+					if bucketHasLockConfig && enforceRetentionForLifecycle(ctx, obj) {
+						continue
+					}
 					objects = append(objects, obj.Name)
 				}
 			}

cmd/data-update-tracker.go

@@ -0,0 +1,611 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/binary"
+	"errors"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/env"
+	"github.com/willf/bloom"
+)
+
+const (
+	// Estimate bloom filter size. With this many items
+	dataUpdateTrackerEstItems = 1000000
+	// ... we want this false positive rate:
+	dataUpdateTrackerFP        = 0.99
+	dataUpdateTrackerQueueSize = 10000
+
+	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
+	dataUpdateTrackerVersion      = 1
+	dataUpdateTrackerSaveInterval = 5 * time.Minute
+
+	// Reset bloom filters every n cycle
+	dataUpdateTrackerResetEvery = 1000
+)
+
+var (
+	objectUpdatedCh      chan<- string
+	intDataUpdateTracker *dataUpdateTracker
+)
+
+func init() {
+	intDataUpdateTracker = newDataUpdateTracker()
+	objectUpdatedCh = intDataUpdateTracker.input
+}
+
+type dataUpdateTracker struct {
+	mu         sync.Mutex
+	input      chan string
+	save       chan struct{}
+	debug      bool
+	saveExited chan struct{}
+
+	Current dataUpdateFilter
+	History dataUpdateTrackerHistory
+	Saved   time.Time
+}
+
+// newDataUpdateTracker returns a dataUpdateTracker with default settings.
+func newDataUpdateTracker() *dataUpdateTracker {
+	d := &dataUpdateTracker{
+		Current: dataUpdateFilter{
+			idx: 1,
+		},
+		debug:      env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn,
+		input:      make(chan string, dataUpdateTrackerQueueSize),
+		save:       make(chan struct{}, 1),
+		saveExited: make(chan struct{}),
+	}
+	d.Current.bf = d.newBloomFilter()
+	return d
+}
+
+type dataUpdateTrackerHistory []dataUpdateFilter
+
+type dataUpdateFilter struct {
+	idx uint64
+	bf  bloomFilter
+}
+
+type bloomFilter struct {
+	*bloom.BloomFilter
+}
+
+// emptyBloomFilter returns an empty bloom filter.
+func emptyBloomFilter() bloomFilter {
+	return bloomFilter{BloomFilter: &bloom.BloomFilter{}}
+}
+
+// containsDir returns whether the bloom filter contains a directory.
+// Note that objects in XL mode are also considered directories.
+func (b bloomFilter) containsDir(in string) bool {
+	split := splitPathDeterministic(path.Clean(in))
+
+	if len(split) == 0 {
+		return false
+	}
+	var tmp [dataUsageHashLen]byte
+	hashPath(path.Join(split...)).bytes(tmp[:])
+	return b.Test(tmp[:])
+}
+
+// bytes returns the bloom filter serialized as a byte slice.
+func (b bloomFilter) bytes() []byte {
+	if b.BloomFilter == nil {
+		return nil
+	}
+	var buf bytes.Buffer
+	_, err := b.WriteTo(&buf)
+	if err != nil {
+		logger.LogIf(GlobalContext, err)
+		return nil
+	}
+	return buf.Bytes()
+}
+
+// sort the dataUpdateTrackerHistory, newest first.
+// Returns whether the history is complete.
+func (d dataUpdateTrackerHistory) sort() bool {
+	if len(d) == 0 {
+		return true
+	}
+	sort.Slice(d, func(i, j int) bool {
+		return d[i].idx > d[j].idx
+	})
+	return d[0].idx-d[len(d)-1].idx == uint64(len(d))
+}
+
+// removeOlderThan will remove entries older than index 'n'.
+func (d *dataUpdateTrackerHistory) removeOlderThan(n uint64) {
+	d.sort()
+	dd := *d
+	end := len(dd)
+	for i := end - 1; i >= 0; i-- {
+		if dd[i].idx < n {
+			end = i
+		}
+	}
+	dd = dd[:end]
+	*d = dd
+}
+
+// newBloomFilter returns a new bloom filter with default settings.
+func (d *dataUpdateTracker) newBloomFilter() bloomFilter {
+	return bloomFilter{bloom.NewWithEstimates(dataUpdateTrackerEstItems, dataUpdateTrackerFP)}
+}
+
+// current returns the current index.
+func (d *dataUpdateTracker) current() uint64 {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	return d.Current.idx
+}
+
+// start will load the current data from the drives start collecting information and
+// start a saver goroutine.
+// All of these will exit when the context is canceled.
+func (d *dataUpdateTracker) start(ctx context.Context, drives ...string) {
+	if len(drives) <= 0 {
+		logger.LogIf(ctx, errors.New("dataUpdateTracker.start: No drives specified"))
+		return
+	}
+	d.load(ctx, drives...)
+	go d.startCollector(ctx)
+	go d.startSaver(ctx, dataUpdateTrackerSaveInterval, drives)
+}
+
+// load will attempt to load data tracking information from the supplied drives.
+// The data will only be loaded if d.Saved is older than the one found on disk.
+// The newest working cache will be kept in d.
+// If no valid data usage tracker can be found d will remain unchanged.
+// If object is shared the caller should lock it.
+func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
+	if len(drives) <= 0 {
+		logger.LogIf(ctx, errors.New("dataUpdateTracker.load: No drives specified"))
+		return
+	}
+	for _, drive := range drives {
+
+		cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
+		f, err := os.Open(cacheFormatPath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+			logger.LogIf(ctx, err)
+			continue
+		}
+		err = d.deserialize(f, d.Saved)
+		if err != nil {
+			logger.LogIf(ctx, err)
+		}
+		f.Close()
+	}
+}
+
+// startSaver will start a saver that will write d to all supplied drives at specific intervals.
+// The saver will save and exit when supplied context is closed.
+func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Duration, drives []string) {
+	t := time.NewTicker(interval)
+	defer t.Stop()
+	var buf bytes.Buffer
+	d.mu.Lock()
+	saveNow := d.save
+	exited := make(chan struct{})
+	d.saveExited = exited
+	d.mu.Unlock()
+	defer close(exited)
+	for {
+		var exit bool
+		select {
+		case <-ctx.Done():
+			exit = true
+		case <-t.C:
+		case <-saveNow:
+		}
+		buf.Reset()
+		d.mu.Lock()
+		d.Saved = UTCNow()
+		err := d.serialize(&buf)
+		if d.debug {
+			logger.Info(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v", buf.Len(), d.Current.idx)
+		}
+		d.mu.Unlock()
+		if err != nil {
+			logger.LogIf(ctx, err, "Error serializing usage tracker data")
+			if exit {
+				return
+			}
+			continue
+		}
+		if buf.Len() == 0 {
+			logger.LogIf(ctx, errors.New("zero sized output, skipping save"))
+			continue
+		}
+		for _, drive := range drives {
+			cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
+			err := ioutil.WriteFile(cacheFormatPath, buf.Bytes(), os.ModePerm)
+			if err != nil {
+				if os.IsNotExist(err) {
+					continue
+				}
+				logger.LogIf(ctx, err)
+				continue
+			}
+		}
+		if exit {
+			return
+		}
+	}
+}
+
+// serialize all data in d to dst.
+// Caller should hold lock if d is expected to be shared.
+// If an error is returned, there will likely be partial data written to dst.
+func (d *dataUpdateTracker) serialize(dst io.Writer) (err error) {
+	ctx := GlobalContext
+	var tmp [8]byte
+	o := bufio.NewWriter(dst)
+	defer func() {
+		if err == nil {
+			err = o.Flush()
+		}
+	}()
+
+	// Version
+	if err := o.WriteByte(dataUpdateTrackerVersion); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+	// Timestamp.
+	binary.LittleEndian.PutUint64(tmp[:], uint64(d.Saved.Unix()))
+	if _, err := o.Write(tmp[:]); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+
+	// Current
+	binary.LittleEndian.PutUint64(tmp[:], d.Current.idx)
+	if _, err := o.Write(tmp[:]); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+
+	if _, err := d.Current.bf.WriteTo(o); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+
+	// History
+	binary.LittleEndian.PutUint64(tmp[:], uint64(len(d.History)))
+	if _, err := o.Write(tmp[:]); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+
+	for _, bf := range d.History {
+		// Current
+		binary.LittleEndian.PutUint64(tmp[:], bf.idx)
+		if _, err := o.Write(tmp[:]); err != nil {
+			if d.debug {
+				logger.LogIf(ctx, err)
+			}
+			return err
+		}
+
+		if _, err := bf.bf.WriteTo(o); err != nil {
+			if d.debug {
+				logger.LogIf(ctx, err)
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+// deserialize will deserialize the supplied input if the input is newer than the supplied time.
+func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) error {
+	ctx := GlobalContext
+	var dst dataUpdateTracker
+	var tmp [8]byte
+
+	// Version
+	if _, err := io.ReadFull(src, tmp[:1]); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+	switch tmp[0] {
+	case dataUpdateTrackerVersion:
+	default:
+		return errors.New("dataUpdateTracker: Unknown data version")
+	}
+	// Timestamp.
+	if _, err := io.ReadFull(src, tmp[:8]); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+	t := time.Unix(int64(binary.LittleEndian.Uint64(tmp[:])), 0)
+	if !t.After(newerThan) {
+		return nil
+	}
+
+	// Current
+	if _, err := io.ReadFull(src, tmp[:8]); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+	dst.Current.idx = binary.LittleEndian.Uint64(tmp[:])
+	dst.Current.bf = emptyBloomFilter()
+	if _, err := dst.Current.bf.ReadFrom(src); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+
+	// History
+	if _, err := io.ReadFull(src, tmp[:8]); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+	n := binary.LittleEndian.Uint64(tmp[:])
+	dst.History = make(dataUpdateTrackerHistory, int(n))
+	for i, e := range dst.History {
+		if _, err := io.ReadFull(src, tmp[:8]); err != nil {
+			if d.debug {
+				logger.LogIf(ctx, err)
+			}
+			return err
+		}
+		e.idx = binary.LittleEndian.Uint64(tmp[:])
+		e.bf = emptyBloomFilter()
+		if _, err := e.bf.ReadFrom(src); err != nil {
+			if d.debug {
+				logger.LogIf(ctx, err)
+			}
+			return err
+		}
+		dst.History[i] = e
+	}
+	// Ignore what remains on the stream.
+	// Update d:
+	d.Current = dst.Current
+	d.History = dst.History
+	d.Saved = dst.Saved
+	return nil
+}
+
+// start a collector that picks up entries from objectUpdatedCh
+// and adds them  to the current bloom filter.
+func (d *dataUpdateTracker) startCollector(ctx context.Context) {
+	var tmp [dataUsageHashLen]byte
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case in := <-d.input:
+			bucket, _ := path2BucketObjectWithBasePath("", in)
+			if bucket == "" {
+				if d.debug && len(in) > 0 {
+					logger.Info(color.Green("data-usage:")+" no bucket (%s)", in)
+				}
+				continue
+			}
+
+			if isReservedOrInvalidBucket(bucket, false) {
+				if false && d.debug {
+					logger.Info(color.Green("data-usage:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
+				}
+				continue
+			}
+			split := splitPathDeterministic(in)
+
+			// Add all paths until level 3.
+			d.mu.Lock()
+			for i := range split {
+				if d.debug && false {
+					logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
+				}
+				hashPath(path.Join(split[:i+1]...)).bytes(tmp[:])
+				d.Current.bf.Add(tmp[:])
+			}
+			d.mu.Unlock()
+		}
+	}
+}
+
+// find entry with specified index.
+// Returns nil if not found.
+func (d dataUpdateTrackerHistory) find(idx uint64) *dataUpdateFilter {
+	for _, f := range d {
+		if f.idx == idx {
+			return &f
+		}
+	}
+	return nil
+}
+
+// filterFrom will return a combined bloom filter.
+func (d *dataUpdateTracker) filterFrom(ctx context.Context, oldest, newest uint64) *bloomFilterResponse {
+	bf := d.newBloomFilter()
+	bfr := bloomFilterResponse{
+		OldestIdx:  oldest,
+		CurrentIdx: d.Current.idx,
+		Complete:   true,
+	}
+	// Loop through each index requested.
+	for idx := oldest; idx <= newest; idx++ {
+		v := d.History.find(idx)
+		if v == nil {
+			if d.Current.idx == idx {
+				// Merge current.
+				err := bf.Merge(d.Current.bf.BloomFilter)
+				logger.LogIf(ctx, err)
+				if err != nil {
+					bfr.Complete = false
+				}
+				continue
+			}
+			bfr.Complete = false
+			bfr.OldestIdx = idx + 1
+			continue
+		}
+
+		err := bf.Merge(v.bf.BloomFilter)
+		if err != nil {
+			bfr.Complete = false
+			logger.LogIf(ctx, err)
+			continue
+		}
+		bfr.NewestIdx = idx
+	}
+	var dst bytes.Buffer
+	_, err := bf.WriteTo(&dst)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return nil
+	}
+	bfr.Filter = dst.Bytes()
+
+	return &bfr
+}
+
+// cycleFilter will cycle the bloom filter to start recording to index y if not already.
+// The response will contain a bloom filter starting at index x up to, but not including index y.
+// If y is 0, the response will not update y, but return the currently recorded information
+// from the up until and including current y.
+func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uint64) (*bloomFilterResponse, error) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	if current == 0 {
+		if len(d.History) == 0 {
+			return d.filterFrom(ctx, d.Current.idx, d.Current.idx), nil
+		}
+		d.History.sort()
+		return d.filterFrom(ctx, d.History[len(d.History)-1].idx, d.Current.idx), nil
+	}
+
+	// Move current to history if new one requested
+	if d.Current.idx != current {
+		if d.debug {
+			logger.Info(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v", d.Current.idx, current)
+		}
+
+		d.History = append(d.History, d.Current)
+		d.Current.idx = current
+		d.Current.bf = d.newBloomFilter()
+		select {
+		case d.save <- struct{}{}:
+		default:
+		}
+	}
+	d.History.removeOlderThan(oldest)
+	return d.filterFrom(ctx, oldest, current), nil
+}
+
+// splitPathDeterministic will split the provided relative path
+// deterministically and return up to the first 3 elements of the path.
+// Slash and dot prefixes are removed.
+// Trailing slashes are removed.
+// Returns 0 length if no parts are found after trimming.
+func splitPathDeterministic(in string) []string {
+	split := strings.Split(in, SlashSeparator)
+
+	// Trim empty start/end
+	for len(split) > 0 {
+		if len(split[0]) > 0 && split[0] != "." {
+			break
+		}
+		split = split[1:]
+	}
+	for len(split) > 0 {
+		if len(split[len(split)-1]) > 0 {
+			break
+		}
+		split = split[:len(split)-1]
+	}
+
+	// Return up to 3 parts.
+	if len(split) > 3 {
+		split = split[:3]
+	}
+	return split
+}
+
+// bloomFilterRequest request bloom filters.
+// Current index will be updated to current and entries back to Oldest is returned.
+type bloomFilterRequest struct {
+	Oldest  uint64
+	Current uint64
+}
+
+type bloomFilterResponse struct {
+	// Current index being written to.
+	CurrentIdx uint64
+	// Oldest index in the returned bloom filter.
+	OldestIdx uint64
+	// Newest Index in the returned bloom filter.
+	NewestIdx uint64
+	// Are all indexes between oldest and newest filled?
+	Complete bool
+	// Binary data of the bloom filter.
+	Filter []byte
+}
+
+// ObjectPathUpdated indicates a path has been updated.
+// The function will never block.
+func ObjectPathUpdated(s string) {
+	select {
+	case objectUpdatedCh <- s:
+	default:
+	}
+}

cmd/data-update-tracker_test.go

@@ -0,0 +1,262 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/logger/message/log"
+)
+
+type testLoggerI interface {
+	Helper()
+	Log(args ...interface{})
+}
+
+type testingLogger struct {
+	mu sync.Mutex
+	t  testLoggerI
+}
+
+func (t *testingLogger) Send(entry interface{}, errKind string) error {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if t.t == nil {
+		return nil
+	}
+	e, ok := entry.(log.Entry)
+	if !ok {
+		return fmt.Errorf("unexpected log entry structure %#v", entry)
+	}
+
+	t.t.Helper()
+	t.t.Log(e.Level, ":", errKind, e.Message)
+	return nil
+}
+
+func addTestingLogging(t testLoggerI) func() {
+	tl := &testingLogger{t: t}
+	logger.AddTarget(tl)
+	return func() {
+		tl.mu.Lock()
+		defer tl.mu.Unlock()
+		tl.t = nil
+	}
+}
+
+func TestDataUpdateTracker(t *testing.T) {
+	dut := newDataUpdateTracker()
+	// Change some defaults.
+	dut.debug = testing.Verbose()
+	dut.input = make(chan string)
+	dut.save = make(chan struct{})
+
+	defer addTestingLogging(t)()
+
+	dut.Current.bf = dut.newBloomFilter()
+
+	tmpDir, err := ioutil.TempDir("", "TestDataUpdateTracker")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	dut.start(ctx, tmpDir)
+
+	var tests = []struct {
+		in    string
+		check []string // if not empty, check against these instead.
+		exist bool
+	}{
+		{
+			in:    "bucket/directory/file.txt",
+			check: []string{"bucket", "bucket/", "/bucket", "bucket/directory", "bucket/directory/", "bucket/directory/file.txt", "/bucket/directory/file.txt"},
+			exist: true,
+		},
+		{
+			// System bucket
+			in:    ".minio.sys/ignoreme/pls",
+			exist: false,
+		},
+		{
+			// Not a valid bucket
+			in:    "./bucket/okfile.txt",
+			check: []string{"./bucket/okfile.txt", "/bucket/okfile.txt", "bucket/okfile.txt"},
+			exist: false,
+		},
+		{
+			// Not a valid bucket
+			in:    "æ/okfile.txt",
+			check: []string{"æ/okfile.txt", "æ/okfile.txt", "æ"},
+			exist: false,
+		},
+		{
+			in:    "/bucket2/okfile2.txt",
+			check: []string{"./bucket2/okfile2.txt", "/bucket2/okfile2.txt", "bucket2/okfile2.txt", "bucket2"},
+			exist: true,
+		},
+		{
+			in:    "/bucket3/prefix/okfile2.txt",
+			check: []string{"./bucket3/prefix/okfile2.txt", "/bucket3/prefix/okfile2.txt", "bucket3/prefix/okfile2.txt", "bucket3/prefix", "bucket3"},
+			exist: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.in, func(t *testing.T) {
+			dut.input <- tt.in
+			dut.input <- "" // Sending empty string ensures the previous is added to filter.
+			dut.mu.Lock()
+			defer dut.mu.Unlock()
+			if len(tt.check) == 0 {
+				got := dut.Current.bf.containsDir(tt.in)
+				if got != tt.exist {
+					// For unlimited tests this could lead to false positives,
+					// but it should be deterministic.
+					t.Errorf("entry %q, got: %v, want %v", tt.in, got, tt.exist)
+				}
+				return
+			}
+			for _, check := range tt.check {
+				got := dut.Current.bf.containsDir(check)
+				if got != tt.exist {
+					// For unlimited tests this could lead to false positives,
+					// but it should be deterministic.
+					t.Errorf("entry %q, check: %q, got: %v, want %v", tt.in, check, got, tt.exist)
+				}
+				continue
+			}
+		})
+	}
+	// Cycle to history
+	_, err = dut.cycleFilter(ctx, 1, 2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dut.input <- "cycle2/file.txt"
+	dut.input <- "" // Sending empty string ensures the previous is added to filter.
+
+	tests = append(tests, struct {
+		in    string
+		check []string
+		exist bool
+	}{in: "cycle2/file.txt", exist: true})
+
+	// Shut down
+	cancel()
+	<-dut.saveExited
+
+	if dut.current() != 2 {
+		t.Fatal("wrong current idx after save. want 2, got:", dut.current())
+	}
+
+	ctx, cancel = context.WithCancel(context.Background())
+	defer cancel()
+
+	// Reload...
+	dut = newDataUpdateTracker()
+	dut.start(ctx, tmpDir)
+
+	if dut.current() != 2 {
+		t.Fatal("current idx after load not preserved. want 2, got:", dut.current())
+	}
+	bfr2, err := dut.cycleFilter(ctx, 1, 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bfr2.Complete {
+		t.Fatal("Wanted complete, didn't get it")
+	}
+	if bfr2.CurrentIdx != 3 {
+		t.Fatal("wanted index 3, got", bfr2.CurrentIdx)
+	}
+	if bfr2.OldestIdx != 1 {
+		t.Fatal("wanted oldest index 3, got", bfr2.OldestIdx)
+	}
+
+	// Rerun test with returned bfr2
+	bf := dut.newBloomFilter()
+	_, err = bf.ReadFrom(bytes.NewBuffer(bfr2.Filter))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, tt := range tests {
+		t.Run(tt.in+"-reloaded", func(t *testing.T) {
+			if len(tt.check) == 0 {
+				got := bf.containsDir(tt.in)
+				if got != tt.exist {
+					// For unlimited tests this could lead to false positives,
+					// but it should be deterministic.
+					t.Errorf("entry %q, got: %v, want %v", tt.in, got, tt.exist)
+				}
+				return
+			}
+			for _, check := range tt.check {
+				got := bf.containsDir(check)
+				if got != tt.exist {
+					// For unlimited tests this could lead to false positives,
+					// but it should be deterministic.
+					t.Errorf("entry %q, check: %q, got: %v, want %v", tt.in, check, got, tt.exist)
+				}
+				continue
+			}
+		})
+	}
+}
+
+func BenchmarkDataUpdateTracker(b *testing.B) {
+	dut := newDataUpdateTracker()
+	// Change some defaults.
+	dut.debug = false
+	dut.input = make(chan string)
+	dut.save = make(chan struct{})
+
+	defer addTestingLogging(b)()
+
+	dut.Current.bf = dut.newBloomFilter()
+	// We do this unbuffered. This will very significantly reduce throughput, so this is a worst case.
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go dut.startCollector(ctx)
+	input := make([]string, 1000)
+	rng := rand.New(rand.NewSource(0xabad1dea))
+	tmp := []string{"bucket", "aprefix", "nextprefixlevel", "maybeobjname", "evendeeper", "ok-one-morelevel", "final.object"}
+	for i := range input {
+		tmp := tmp[:1+rng.Intn(cap(tmp)-1)]
+		input[i] = path.Join(tmp...)
+	}
+	b.SetBytes(1)
+	b.ResetTimer()
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		dut.input <- input[rng.Intn(len(input))]
+	}
+}

cmd/data-usage-cache.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"io"
 	"path"
@@ -62,9 +63,10 @@ type dataUsageEntryInfo struct {
 
 type dataUsageCacheInfo struct {
 	// Name of the bucket. Also root element.
-	Name       string
-	LastUpdate time.Time
-	NextCycle  uint8
+	Name        string
+	LastUpdate  time.Time
+	NextCycle   uint32
+	BloomFilter []byte `msg:"BloomFilter,omitempty"`
 }
 
 // merge other data usage entry into this, excluding children.
@@ -77,8 +79,8 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 }
 
 // mod returns true if the hash mod cycles == cycle.
-func (h dataUsageHash) mod(cycle uint8, cycles uint8) bool {
-	return uint8(h)%cycles == cycle%cycles
+func (h dataUsageHash) mod(cycle uint32, cycles uint32) bool {
+	return uint32(h)%cycles == cycle%cycles
 }
 
 // addChildString will add a child based on its name.
@@ -110,6 +112,7 @@ func (d *dataUsageCache) find(path string) *dataUsageEntry {
 }
 
 // dui converts the flattened version of the path to DataUsageInfo.
+// As a side effect d will be flattened, use a clone if this is not ok.
 func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
 	e := d.find(path)
 	if e == nil {
@@ -158,6 +161,32 @@ func (d *dataUsageCache) replaceHashed(hash dataUsageHash, parent *dataUsageHash
 	}
 }
 
+// copyWithChildren will copy entry with hash from src if it exists along with any children.
+// If a parent is specified it will be added to that if not already there.
+// If the parent does not exist, it will be added.
+func (d *dataUsageCache) copyWithChildren(src *dataUsageCache, hash dataUsageHash, parent *dataUsageHash) {
+	if d.Cache == nil {
+		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+	}
+	e, ok := src.Cache[hash]
+	if !ok {
+		return
+	}
+	d.Cache[hash] = e
+	for ch := range e.Children {
+		if ch == hash {
+			logger.LogIf(GlobalContext, errors.New("dataUsageCache.copyWithChildren: Circular reference"))
+			return
+		}
+		d.copyWithChildren(src, ch, &hash)
+	}
+	if parent != nil {
+		p := d.Cache[*parent]
+		p.addChild(hash)
+		d.Cache[*parent] = p
+	}
+}
+
 // StringAll returns a detailed string representation of all entries in the cache.
 func (d *dataUsageCache) StringAll() string {
 	s := fmt.Sprintf("info:%+v\n", d.Info)
@@ -167,6 +196,12 @@ func (d *dataUsageCache) StringAll() string {
 	return strings.TrimSpace(s)
 }
 
+// insert the hash into dst.
+// dst must be at least dataUsageHashLen bytes long.
+func (h dataUsageHash) bytes(dst []byte) {
+	binary.LittleEndian.PutUint64(dst, uint64(h))
+}
+
 // String returns a human readable representation of the string.
 func (h dataUsageHash) String() string {
 	return fmt.Sprintf("%x", uint64(h))
@@ -297,7 +332,7 @@ func (d *dataUsageCache) load(ctx context.Context, store ObjectLayer, name strin
 	var buf bytes.Buffer
 	err := store.GetObject(ctx, dataUsageBucket, name, 0, -1, &buf, "", ObjectOptions{})
 	if err != nil {
-		if !isErrObjectNotFound(err) {
+		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) {
 			return toObjectErr(err, dataUsageBucket, name)
 		}
 		*d = dataUsageCache{}
@@ -325,6 +360,9 @@ func (d *dataUsageCache) save(ctx context.Context, store ObjectLayer, name strin
 		name,
 		NewPutObjReader(r, nil, nil),
 		ObjectOptions{})
+	if isErrBucketNotFound(err) {
+		return nil
+	}
 	return err
 }
 

cmd/data-usage-cache_gen.go

@@ -37,11 +37,17 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "NextCycle":
-			z.NextCycle, err = dc.ReadUint8()
+			z.NextCycle, err = dc.ReadUint32()
 			if err != nil {
 				err = msgp.WrapError(err, "NextCycle")
 				return
 			}
+		case "BloomFilter":
+			z.BloomFilter, err = dc.ReadBytes(z.BloomFilter)
+			if err != nil {
+				err = msgp.WrapError(err, "BloomFilter")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -54,10 +60,24 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 }
 
 // EncodeMsg implements msgp.Encodable
-func (z dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 3
+func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
+	// omitempty: check for empty values
+	zb0001Len := uint32(4)
+	var zb0001Mask uint8 /* 4 bits */
+	if z.BloomFilter == nil {
+		zb0001Len--
+		zb0001Mask |= 0x8
+	}
+	// variable map header, size zb0001Len
+	err = en.Append(0x80 | uint8(zb0001Len))
+	if err != nil {
+		return
+	}
+	if zb0001Len == 0 {
+		return
+	}
 	// write "Name"
-	err = en.Append(0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -81,27 +101,55 @@ func (z dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteUint8(z.NextCycle)
+	err = en.WriteUint32(z.NextCycle)
 	if err != nil {
 		err = msgp.WrapError(err, "NextCycle")
 		return
 	}
+	if (zb0001Mask & 0x8) == 0 { // if not empty
+		// write "BloomFilter"
+		err = en.Append(0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
+		if err != nil {
+			return
+		}
+		err = en.WriteBytes(z.BloomFilter)
+		if err != nil {
+			err = msgp.WrapError(err, "BloomFilter")
+			return
+		}
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
-func (z dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
+func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 3
+	// omitempty: check for empty values
+	zb0001Len := uint32(4)
+	var zb0001Mask uint8 /* 4 bits */
+	if z.BloomFilter == nil {
+		zb0001Len--
+		zb0001Mask |= 0x8
+	}
+	// variable map header, size zb0001Len
+	o = append(o, 0x80|uint8(zb0001Len))
+	if zb0001Len == 0 {
+		return
+	}
 	// string "Name"
-	o = append(o, 0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "LastUpdate"
 	o = append(o, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65)
 	o = msgp.AppendTime(o, z.LastUpdate)
 	// string "NextCycle"
 	o = append(o, 0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
-	o = msgp.AppendUint8(o, z.NextCycle)
+	o = msgp.AppendUint32(o, z.NextCycle)
+	if (zb0001Mask & 0x8) == 0 { // if not empty
+		// string "BloomFilter"
+		o = append(o, 0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
+		o = msgp.AppendBytes(o, z.BloomFilter)
+	}
 	return
 }
 
@@ -136,11 +184,17 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				return
 			}
 		case "NextCycle":
-			z.NextCycle, bts, err = msgp.ReadUint8Bytes(bts)
+			z.NextCycle, bts, err = msgp.ReadUint32Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "NextCycle")
 				return
 			}
+		case "BloomFilter":
+			z.BloomFilter, bts, err = msgp.ReadBytesBytes(bts, z.BloomFilter)
+			if err != nil {
+				err = msgp.WrapError(err, "BloomFilter")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -154,8 +208,8 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 }
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z dataUsageCacheInfo) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.TimeSize + 10 + msgp.Uint8Size
+func (z *dataUsageCacheInfo) Msgsize() (s int) {
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.TimeSize + 10 + msgp.Uint32Size + 12 + msgp.BytesPrefixSize + len(z.BloomFilter)
 	return
 }
 

cmd/data-usage.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"encoding/binary"
 	"encoding/json"
 	"errors"
 	"os"
@@ -32,19 +33,23 @@ import (
 	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/willf/bloom"
 )
 
 const (
+	envDataUsageCrawlConf  = "MINIO_DISK_USAGE_CRAWL_ENABLE"
+	envDataUsageCrawlDelay = "MINIO_DISK_USAGE_CRAWL_DELAY"
+	envDataUsageCrawlDebug = "MINIO_DISK_USAGE_CRAWL_DEBUG"
+
+	dataUsageRoot   = SlashSeparator
+	dataUsageBucket = minioMetaBucket + SlashSeparator + bucketMetaPrefix
+
 	dataUsageObjName         = ".usage.json"
 	dataUsageCacheName       = ".usage-cache.bin"
-	envDataUsageCrawlConf    = "MINIO_DISK_USAGE_CRAWL_ENABLE"
-	envDataUsageCrawlDelay   = "MINIO_DISK_USAGE_CRAWL_DELAY"
-	envDataUsageCrawlDebug   = "MINIO_DISK_USAGE_CRAWL_DEBUG"
+	dataUsageBloomName       = ".bloomcycle.bin"
 	dataUsageSleepPerFolder  = 1 * time.Millisecond
 	dataUsageSleepDefMult    = 10.0
 	dataUsageUpdateDirCycles = 16
-	dataUsageRoot            = SlashSeparator
-	dataUsageBucket          = minioMetaBucket + SlashSeparator + bucketMetaPrefix
 	dataUsageStartDelay      = 5 * time.Minute // Time to wait on startup and between cycles.
 )
 
@@ -56,6 +61,20 @@ func initDataUsageStats(ctx context.Context, objAPI ObjectLayer) {
 }
 
 func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer) {
+	// Load current bloom cycle
+	nextBloomCycle := intDataUpdateTracker.current() + 1
+	var buf bytes.Buffer
+	err := objAPI.GetObject(ctx, dataUsageBucket, dataUsageBloomName, 0, -1, &buf, "", ObjectOptions{})
+	if err != nil {
+		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) {
+			logger.LogIf(ctx, err)
+		}
+	} else {
+		if buf.Len() == 8 {
+			nextBloomCycle = binary.LittleEndian.Uint64(buf.Bytes())
+		}
+	}
+
 	for {
 		select {
 		case <-ctx.Done():
@@ -64,9 +83,33 @@ func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer) {
 			// Wait before starting next cycle and wait on startup.
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
-			err := objAPI.CrawlAndGetDataUsage(ctx, results)
+			bf, err := globalNotificationSys.updateBloomFilter(ctx, nextBloomCycle)
+			logger.LogIf(ctx, err)
+			err = objAPI.CrawlAndGetDataUsage(ctx, bf, results)
 			close(results)
 			logger.LogIf(ctx, err)
+			if err == nil {
+				// Store new cycle...
+				nextBloomCycle++
+				if nextBloomCycle%dataUpdateTrackerResetEvery == 0 {
+					if intDataUpdateTracker.debug {
+						logger.Info(color.Green("runDataUsageInfo:") + " Resetting bloom filter for next runs.")
+					}
+					nextBloomCycle++
+				}
+				var tmp [8]byte
+				binary.LittleEndian.PutUint64(tmp[:], nextBloomCycle)
+				r, err := hash.NewReader(bytes.NewReader(tmp[:]), int64(len(tmp)), "", "", int64(len(tmp)), false)
+				if err != nil {
+					logger.LogIf(ctx, err)
+					continue
+				}
+
+				_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageBloomName, NewPutObjReader(r, nil, nil), ObjectOptions{})
+				if !isErrBucketNotFound(err) {
+					logger.LogIf(ctx, err)
+				}
+			}
 		}
 	}
 }
@@ -87,7 +130,9 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, gui <-chan
 		}
 
 		_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
-		logger.LogIf(ctx, err)
+		if !isErrBucketNotFound(err) {
+			logger.LogIf(ctx, err)
+		}
 	}
 }
 
@@ -96,7 +141,7 @@ func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsag
 
 	err := objAPI.GetObject(ctx, dataUsageBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
 	if err != nil {
-		if isErrObjectNotFound(err) {
+		if isErrObjectNotFound(err) || isErrBucketNotFound(err) {
 			return DataUsageInfo{}, nil
 		}
 		return DataUsageInfo{}, toObjectErr(err, dataUsageBucket, dataUsageObjName)
@@ -130,6 +175,7 @@ type folderScanner struct {
 	getSize            getSizeFn
 	oldCache           dataUsageCache
 	newCache           dataUsageCache
+	withFilter         *bloomFilter
 	waitForLowActiveIO func()
 
 	dataUsageCrawlMult  float64
@@ -164,12 +210,22 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			return nil, ctx.Err()
 		default:
 		}
+		thisHash := hashPath(folder.name)
 
+		if _, ok := f.oldCache.Cache[thisHash]; f.withFilter != nil && ok {
+			// If folder isn't in filter and we have data, skip it completely.
+			if folder.name != dataUsageRoot && !f.withFilter.containsDir(folder.name) {
+				f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" Skipping non-updated folder: %v", folder.name)
+				}
+				continue
+			}
+		}
 		f.waitForLowActiveIO()
 		sleepDuration(dataUsageSleepPerFolder, f.dataUsageCrawlMult)
 
 		cache := dataUsageEntry{}
-		thisHash := hashPath(folder.name)
 
 		err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error {
 			// Parse
@@ -301,11 +357,14 @@ func updateUsage(ctx context.Context, basePath string, cache dataUsageCache, wai
 	t := UTCNow()
 
 	dataUsageDebug := env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn
-	defer func() {
-		if dataUsageDebug {
-			logger.Info(color.Green("updateUsage")+" Crawl time at %s: %v", basePath, time.Since(t))
-		}
-	}()
+	logPrefix := color.Green("data-usage: ")
+	logSuffix := color.Blue(" - %v + %v", basePath, cache.Info.Name)
+	if dataUsageDebug {
+		defer func() {
+			logger.Info(logPrefix+" Crawl time: %v"+logSuffix, time.Since(t))
+		}()
+
+	}
 
 	if cache.Info.Name == "" {
 		cache.Info.Name = dataUsageRoot
@@ -329,8 +388,16 @@ func updateUsage(ctx context.Context, basePath string, cache dataUsageCache, wai
 		dataUsageCrawlDebug: dataUsageDebug,
 	}
 
+	if len(cache.Info.BloomFilter) > 0 {
+		s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
+		_, err := s.withFilter.ReadFrom(bytes.NewBuffer(cache.Info.BloomFilter))
+		if err != nil {
+			logger.LogIf(ctx, err, logPrefix+"Error reading bloom filter")
+			s.withFilter = nil
+		}
+	}
 	if s.dataUsageCrawlDebug {
-		logger.Info(color.Green("runDataUsageInfo:") + " Starting crawler master")
+		logger.Info(logPrefix+"Start crawling. Bloom filter: %v"+logSuffix, s.withFilter != nil)
 	}
 
 	done := ctx.Done()
@@ -341,14 +408,8 @@ func updateUsage(ctx context.Context, basePath string, cache dataUsageCache, wai
 		flattenLevels--
 	}
 
-	var logPrefix, logSuffix string
 	if s.dataUsageCrawlDebug {
-		logPrefix = color.Green("data-usage: ")
-		logSuffix = color.Blue(" - %v + %v", basePath, cache.Info.Name)
-	}
-
-	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Cycle: %v"+logSuffix, cache.Info.NextCycle)
+		logger.Info(logPrefix+"Cycle: %v, Entries: %v"+logSuffix, cache.Info.NextCycle, len(cache.Cache))
 	}
 
 	// Always scan flattenLevels deep. Cache root is level 0.
@@ -387,9 +448,10 @@ func updateUsage(ctx context.Context, basePath string, cache dataUsageCache, wai
 			continue
 		}
 		if du == nil {
-			logger.LogIf(ctx, errors.New("data-usage: no disk usage provided"))
+			logger.Info(logPrefix + "no disk usage provided" + logSuffix)
 			continue
 		}
+
 		s.newCache.replace(folder.name, "", *du)
 		// Add to parent manually
 		if folder.parent != nil {
@@ -415,6 +477,17 @@ func updateUsage(ctx context.Context, basePath string, cache dataUsageCache, wai
 			continue
 		}
 
+		if s.withFilter != nil {
+			// If folder isn't in filter, skip it completely.
+			if !s.withFilter.containsDir(folder.name) {
+				if s.dataUsageCrawlDebug {
+					logger.Info(logPrefix+"Skipping non-updated folder: %v"+logSuffix, folder)
+				}
+				s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h])
+				continue
+			}
+		}
+
 		// Update on this cycle...
 		du, err := s.deepScanFolder(ctx, folder.name)
 		if err != nil {
@@ -427,7 +500,9 @@ func updateUsage(ctx context.Context, basePath string, cache dataUsageCache, wai
 		}
 		s.newCache.replaceHashed(h, folder.parent, *du)
 	}
-
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Finished crawl, %v entries"+logSuffix, len(s.newCache.Cache))
+	}
 	s.newCache.Info.LastUpdate = UTCNow()
 	s.newCache.Info.NextCycle++
 	return s.newCache, nil

cmd/disk-cache-backend.go

@@ -21,10 +21,10 @@ import (
 	"context"
 	"crypto/rand"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"os"
 	"strings"
@@ -178,7 +178,7 @@ func (c *diskCache) diskUsageLow() bool {
 	di, err := disk.GetInfo(c.dir)
 	if err != nil {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 		logger.LogIf(ctx, err)
 		return false
 	}
@@ -193,7 +193,7 @@ func (c *diskCache) diskUsageHigh() bool {
 	di, err := disk.GetInfo(c.dir)
 	if err != nil {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 		logger.LogIf(ctx, err)
 		return false
 	}
@@ -207,7 +207,7 @@ func (c *diskCache) diskAvailable(size int64) bool {
 	di, err := disk.GetInfo(c.dir)
 	if err != nil {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 		logger.LogIf(ctx, err)
 		return false
 	}
@@ -221,13 +221,17 @@ func (c *diskCache) toClear() uint64 {
 	di, err := disk.GetInfo(c.dir)
 	if err != nil {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 		logger.LogIf(ctx, err)
 		return 0
 	}
 	return bytesToClear(int64(di.Total), int64(di.Free), uint64(c.quotaPct), uint64(c.lowWatermark))
 }
 
+var (
+	errDoneForNow = errors.New("done for now")
+)
+
 // Purge cache entries that were not accessed.
 func (c *diskCache) purge(ctx context.Context) {
 	if c.diskUsageLow() {
@@ -263,25 +267,24 @@ func (c *diskCache) purge(ctx context.Context) {
 		}
 		return fm
 	}
-	objDirs, err := ioutil.ReadDir(c.dir)
-	if err != nil {
-		log.Fatal(err)
-	}
 
-	for _, obj := range objDirs {
-		if obj.Name() == minioMetaBucket {
-			continue
+	filterFn := func(name string, typ os.FileMode) error {
+		if name == minioMetaBucket {
+			// Proceed to next file.
+			return nil
 		}
 
-		cacheDir := pathJoin(c.dir, obj.Name())
+		cacheDir := pathJoin(c.dir, name)
 		meta, _, numHits, err := c.statCachedMeta(ctx, cacheDir)
 		if err != nil {
 			// delete any partially filled cache entry left behind.
 			removeAll(cacheDir)
-			continue
+			// Proceed to next file.
+			return nil
 		}
+
 		// stat all cached file ranges and cacheDataFile.
-		cachedFiles := fiStatFn(meta.Ranges, cacheDataFile, pathJoin(c.dir, obj.Name()))
+		cachedFiles := fiStatFn(meta.Ranges, cacheDataFile, pathJoin(c.dir, name))
 		objInfo := meta.ToObjectInfo("", "")
 		cc := cacheControlOpts(objInfo)
 		for fname, fi := range cachedFiles {
@@ -291,9 +294,11 @@ func (c *diskCache) purge(ctx context.Context) {
 						logger.LogIf(ctx, err)
 					}
 					scorer.adjustSaveBytes(-fi.Size())
+
 					// break early if sufficient disk space reclaimed.
 					if c.diskUsageLow() {
-						return
+						// if we found disk usage is already low, we return nil filtering is complete.
+						return errDoneForNow
 					}
 				}
 				continue
@@ -305,12 +310,24 @@ func (c *diskCache) purge(ctx context.Context) {
 		if err != nil || (fi.ModTime().Before(expiry) && len(cachedFiles) == 0) {
 			removeAll(cacheDir)
 			scorer.adjustSaveBytes(-fi.Size())
-			continue
+			// Proceed to next file.
+			return nil
 		}
+
+		// if we found disk usage is already low, we return nil filtering is complete.
 		if c.diskUsageLow() {
-			return
+			return errDoneForNow
 		}
+
+		// Proceed to next file.
+		return nil
 	}
+
+	if err := readDirFilterFn(c.dir, filterFn); err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
+
 	for _, path := range scorer.fileNames() {
 		removeAll(path)
 		slashIdx := strings.LastIndex(path, SlashSeparator)
@@ -800,7 +817,7 @@ func (c *diskCache) bitrotReadFromCache(ctx context.Context, filePath string, of
 		if !bytes.Equal(hashBytes, checksumHash) {
 			err = fmt.Errorf("hashes do not match expected %s, got %s",
 				hex.EncodeToString(checksumHash), hex.EncodeToString(hashBytes))
-			logger.LogIf(context.Background(), err)
+			logger.LogIf(GlobalContext, err)
 			return err
 		}
 
@@ -847,11 +864,11 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 	if HasSuffix(object, SlashSeparator) {
 		// The lock taken above is released when
 		// objReader.Close() is called by the caller.
-		gr, gerr := NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
+		gr, gerr := NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts, nsUnlocker)
 		return gr, numHits, gerr
 	}
 
-	fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
+	fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts, nsUnlocker)
 	if nErr != nil {
 		return nil, numHits, nErr
 	}

cmd/disk-cache.go

@@ -207,6 +207,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			return cacheReader, nil
 		}
 		if cc != nil && cc.noStore {
+			cacheReader.Close()
 			c.cacheStats.incMiss()
 			bReader, err := c.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 			bReader.ObjInfo.CacheLookupStatus = CacheHit
@@ -220,9 +221,11 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		c.incCacheStats(cacheObjSize)
 		return cacheReader, nil
 	} else if err != nil {
+		if cacheErr == nil {
+			cacheReader.Close()
+		}
 		if _, ok := err.(ObjectNotFound); ok {
 			if cacheErr == nil {
-				cacheReader.Close()
 				// Delete cached entry if backend object
 				// was deleted.
 				dcache.Delete(ctx, bucket, object)
@@ -233,13 +236,19 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	}
 
 	if !objInfo.IsCacheable() {
+		if cacheErr == nil {
+			cacheReader.Close()
+		}
 		c.cacheStats.incMiss()
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
 	// skip cache for objects with locks
 	objRetention := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
 	legalHold := objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
-	if objRetention.Mode != objectlock.Invalid || legalHold.Status != "" {
+	if objRetention.Mode.Valid() || legalHold.Status.Valid() {
+		if cacheErr == nil {
+			cacheReader.Close()
+		}
 		c.cacheStats.incMiss()
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
@@ -312,7 +321,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	}()
 	cleanupBackend := func() { bkReader.Close() }
 	cleanupPipe := func() { pipeWriter.Close() }
-	return NewGetObjectReaderFromReader(teeReader, bkReader.ObjInfo, opts.CheckCopyPrecondFn, cleanupBackend, cleanupPipe)
+	return NewGetObjectReaderFromReader(teeReader, bkReader.ObjInfo, opts, cleanupBackend, cleanupPipe)
 }
 
 // Returns ObjectInfo from cache if available.
@@ -501,7 +510,7 @@ func (c *cacheObjects) hashIndex(bucket, object string) int {
 // or the global env overrides.
 func newCache(config cache.Config) ([]*diskCache, bool, error) {
 	var caches []*diskCache
-	ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{})
+	ctx := logger.SetReqInfo(GlobalContext, &logger.ReqInfo{})
 	formats, migrating, err := loadAndValidateCacheFormat(ctx, config.Drives)
 	if err != nil {
 		return nil, false, err
@@ -614,7 +623,7 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 	// skip cache for objects with locks
 	objRetention := objectlock.GetObjectRetentionMeta(opts.UserDefined)
 	legalHold := objectlock.GetObjectLegalHoldMeta(opts.UserDefined)
-	if objRetention.Mode != objectlock.Invalid || legalHold.Status != "" {
+	if objRetention.Mode.Valid() || legalHold.Status.Valid() {
 		dcache.Delete(ctx, bucket, object)
 		return putObjectFn(ctx, bucket, object, r, opts)
 	}

cmd/disk-cache_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"io"
 	"testing"
 
@@ -184,7 +183,7 @@ func TestDiskCacheMaxUse(t *testing.T) {
 		t.Fatal(err)
 	}
 	cache := d[0]
-	ctx := context.Background()
+	ctx := GlobalContext
 	bucketName := "testbucket"
 	objectName := "testobject"
 	content := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"

cmd/encryption-v1.go

@@ -29,6 +29,7 @@ import (
 	"net/http"
 	"path"
 	"strconv"
+	"strings"
 
 	"github.com/minio/minio-go/v6/pkg/encrypt"
 	"github.com/minio/minio/cmd/crypto"
@@ -168,39 +169,39 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
 	}
 }
 
-func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]string, sseS3 bool) ([]byte, error) {
+func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]string, sseS3 bool) (crypto.ObjectKey, error) {
 	var sealedKey crypto.SealedKey
 	if sseS3 {
 		if GlobalKMS == nil {
-			return nil, errKMSNotConfigured
+			return crypto.ObjectKey{}, errKMSNotConfigured
 		}
 		key, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
 		if err != nil {
-			return nil, err
+			return crypto.ObjectKey{}, err
 		}
 
 		objectKey := crypto.GenerateKey(key, rand.Reader)
 		sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
 		crypto.S3.CreateMetadata(metadata, GlobalKMS.KeyID(), encKey, sealedKey)
-		return objectKey[:], nil
+		return objectKey, nil
 	}
 	var extKey [32]byte
 	copy(extKey[:], key)
 	objectKey := crypto.GenerateKey(extKey, rand.Reader)
 	sealedKey = objectKey.Seal(extKey, crypto.GenerateIV(rand.Reader), crypto.SSEC.String(), bucket, object)
 	crypto.SSEC.CreateMetadata(metadata, sealedKey)
-	return objectKey[:], nil
+	return objectKey, nil
 }
 
-func newEncryptReader(content io.Reader, key []byte, bucket, object string, metadata map[string]string, sseS3 bool) (r io.Reader, encKey []byte, err error) {
+func newEncryptReader(content io.Reader, key []byte, bucket, object string, metadata map[string]string, sseS3 bool) (io.Reader, crypto.ObjectKey, error) {
 	objectEncryptionKey, err := newEncryptMetadata(key, bucket, object, metadata, sseS3)
 	if err != nil {
-		return nil, encKey, err
+		return nil, crypto.ObjectKey{}, err
 	}
 
 	reader, err := sio.EncryptReader(content, sio.Config{Key: objectEncryptionKey[:], MinVersion: sio.Version20})
 	if err != nil {
-		return nil, encKey, crypto.ErrInvalidCustomerKey
+		return nil, crypto.ObjectKey{}, crypto.ErrInvalidCustomerKey
 	}
 
 	return reader, objectEncryptionKey, nil
@@ -225,23 +226,24 @@ func setEncryptionMetadata(r *http.Request, bucket, object string, metadata map[
 // EncryptRequest takes the client provided content and encrypts the data
 // with the client provided key. It also marks the object as client-side-encrypted
 // and sets the correct headers.
-func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, metadata map[string]string) (reader io.Reader, objEncKey []byte, err error) {
-	var key []byte
-
+func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, metadata map[string]string) (io.Reader, crypto.ObjectKey, error) {
 	if crypto.S3.IsRequested(r.Header) && crypto.SSEC.IsRequested(r.Header) {
-		return nil, objEncKey, crypto.ErrIncompatibleEncryptionMethod
-	}
-	if crypto.SSEC.IsRequested(r.Header) {
-		key, err = ParseSSECustomerRequest(r)
-		if err != nil {
-			return nil, objEncKey, err
-		}
+		return nil, crypto.ObjectKey{}, crypto.ErrIncompatibleEncryptionMethod
 	}
 	if r.ContentLength > encryptBufferThreshold {
 		// The encryption reads in blocks of 64KB.
 		// We add a buffer on bigger files to reduce the number of syscalls upstream.
 		content = bufio.NewReaderSize(content, encryptBufferSize)
 	}
+
+	var key []byte
+	if crypto.SSEC.IsRequested(r.Header) {
+		var err error
+		key, err = ParseSSECustomerRequest(r)
+		if err != nil {
+			return nil, crypto.ObjectKey{}, err
+		}
+	}
 	return newEncryptReader(content, key, bucket, object, metadata, crypto.S3.IsRequested(r.Header))
 }
 
@@ -634,6 +636,47 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	return size, nil
 }
 
+// DecryptETag decrypts the ETag that is part of given object
+// with the given object encryption key.
+//
+// However, DecryptETag does not try to decrypt the ETag if
+// it consists of a 128 bit hex value (32 hex chars) and exactly
+// one '-' followed by a 32-bit number.
+// This special case adresses randomly-generated ETags generated
+// by the MinIO server when running in non-compat mode. These
+// random ETags are not encrypt.
+//
+// Calling DecryptETag with a non-randomly generated ETag will
+// fail.
+func DecryptETag(key crypto.ObjectKey, object ObjectInfo) (string, error) {
+	if n := strings.Count(object.ETag, "-"); n > 0 {
+		if n != 1 {
+			return "", errObjectTampered
+		}
+		i := strings.IndexByte(object.ETag, '-')
+		if len(object.ETag[:i]) != 32 {
+			return "", errObjectTampered
+		}
+		if _, err := hex.DecodeString(object.ETag[:32]); err != nil {
+			return "", errObjectTampered
+		}
+		if _, err := strconv.ParseInt(object.ETag[i+1:], 10, 32); err != nil {
+			return "", errObjectTampered
+		}
+		return object.ETag, nil
+	}
+
+	etag, err := hex.DecodeString(object.ETag)
+	if err != nil {
+		return "", err
+	}
+	etag, err = key.UnsealETag(etag)
+	if err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(etag), nil
+}
+
 // For encrypted objects, the ETag sent by client if available
 // is stored in encrypted form in the backend. Decrypt the ETag
 // if ETag was previously encrypted.
@@ -817,7 +860,7 @@ func (o *ObjectInfo) EncryptedSize() int64 {
 		// This cannot happen since AWS S3 allows parts to be 5GB at most
 		// sio max. size is 256 TB
 		reqInfo := (&logger.ReqInfo{}).AppendTags("size", strconv.FormatUint(size, 10))
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 		logger.CriticalIf(ctx, err)
 	}
 	return int64(size)
@@ -845,7 +888,7 @@ func DecryptCopyObjectInfo(info *ObjectInfo, headers http.Header) (errCode APIEr
 		}
 		var err error
 		if info.Size, err = info.DecryptedSize(); err != nil {
-			errCode = toAPIErrorCode(context.Background(), err)
+			errCode = toAPIErrorCode(GlobalContext, err)
 		}
 	}
 	return
@@ -939,6 +982,19 @@ func getOpts(ctx context.Context, r *http.Request, bucket, object string) (Objec
 		encryption encrypt.ServerSide
 		opts       ObjectOptions
 	)
+
+	var partNumber int
+	var err error
+	if pn := r.URL.Query().Get("partNumber"); pn != "" {
+		partNumber, err = strconv.Atoi(pn)
+		if err != nil {
+			return opts, err
+		}
+		if partNumber < 0 {
+			return opts, errInvalidArgument
+		}
+	}
+
 	if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) {
 		key, err := crypto.SSEC.ParseHTTP(r.Header)
 		if err != nil {
@@ -947,10 +1003,16 @@ func getOpts(ctx context.Context, r *http.Request, bucket, object string) (Objec
 		derivedKey := deriveClientKey(key, bucket, object)
 		encryption, err = encrypt.NewSSEC(derivedKey[:])
 		logger.CriticalIf(ctx, err)
-		return ObjectOptions{ServerSideEncryption: encryption}, nil
+		return ObjectOptions{ServerSideEncryption: encryption, PartNumber: partNumber}, nil
 	}
+
 	// default case of passing encryption headers to backend
-	return getDefaultOpts(r.Header, false, nil)
+	opts, err = getDefaultOpts(r.Header, false, nil)
+	if err != nil {
+		return opts, err
+	}
+	opts.PartNumber = partNumber
+	return opts, nil
 }
 
 // get ObjectOptions for PUT calls from encryption headers and metadata

cmd/encryption-v1_test.go

@@ -256,6 +256,78 @@ func TestDecryptObjectInfo(t *testing.T) {
 	}
 }
 
+var decryptETagTests = []struct {
+	ObjectKey  crypto.ObjectKey
+	ObjectInfo ObjectInfo
+	ShouldFail bool
+	ETag       string
+}{
+	{
+		ObjectKey:  [32]byte{},
+		ObjectInfo: ObjectInfo{ETag: "20000f00f27834c9a2654927546df57f9e998187496394d4ee80f3d9978f85f3c7d81f72600cdbe03d80dc5a13d69354"},
+		ETag:       "8ad3fe6b84bf38489e95c701c84355b6",
+	},
+	{
+		ObjectKey:  [32]byte{},
+		ObjectInfo: ObjectInfo{ETag: "20000f00f27834c9a2654927546df57f9e998187496394d4ee80f3d9978f85f3c7d81f72600cdbe03d80dc5a13d6935"},
+		ETag:       "",
+		ShouldFail: true, // ETag is not a valid hex value
+	},
+	{
+		ObjectKey:  [32]byte{},
+		ObjectInfo: ObjectInfo{ETag: "00000f00f27834c9a2654927546df57f9e998187496394d4ee80f3d9978f85f3c7d81f72600cdbe03d80dc5a13d69354"},
+		ETag:       "",
+		ShouldFail: true, // modified ETag
+	},
+
+	// Special tests for ETags that end with a '-x'
+	{
+		ObjectKey:  [32]byte{},
+		ObjectInfo: ObjectInfo{ETag: "916516b396f0f4d4f2a0e7177557bec4-1"},
+		ETag:       "916516b396f0f4d4f2a0e7177557bec4-1",
+	},
+	{
+		ObjectKey:  [32]byte{},
+		ObjectInfo: ObjectInfo{ETag: "916516b396f0f4d4f2a0e7177557bec4-738"},
+		ETag:       "916516b396f0f4d4f2a0e7177557bec4-738",
+	},
+	{
+		ObjectKey:  [32]byte{},
+		ObjectInfo: ObjectInfo{ETag: "916516b396f0f4d4f2a0e7177557bec4-Q"},
+		ETag:       "",
+		ShouldFail: true, // Q is not a number
+	},
+	{
+		ObjectKey:  [32]byte{},
+		ObjectInfo: ObjectInfo{ETag: "16516b396f0f4d4f2a0e7177557bec4-1"},
+		ETag:       "",
+		ShouldFail: true, // ETag prefix is not a valid hex value
+	},
+	{
+		ObjectKey:  [32]byte{},
+		ObjectInfo: ObjectInfo{ETag: "16516b396f0f4d4f2a0e7177557bec4-1-2"},
+		ETag:       "",
+		ShouldFail: true, // ETag contains multiple: -
+	},
+}
+
+func TestDecryptETag(t *testing.T) {
+	for i, test := range decryptETagTests {
+		etag, err := DecryptETag(test.ObjectKey, test.ObjectInfo)
+		if err != nil && !test.ShouldFail {
+			t.Fatalf("Test %d: should succeed but failed: %v", i, err)
+		}
+		if err == nil && test.ShouldFail {
+			t.Fatalf("Test %d: should fail but succeeded", i)
+		}
+		if err == nil {
+			if etag != test.ETag {
+				t.Fatalf("Test %d: ETag mismatch: got %s - want %s", i, etag, test.ETag)
+			}
+		}
+	}
+}
+
 // Tests for issue reproduced when getting the right encrypted
 // offset of the object.
 func TestGetDecryptedRange_Issue50(t *testing.T) {

cmd/endpoint-ellipses.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -63,11 +64,72 @@ var isValidSetSize = func(count uint64) bool {
 	return (count >= setSizes[0] && count <= setSizes[len(setSizes)-1])
 }
 
+func commonSetDriveCount(divisibleSize uint64, setCounts []uint64) (setSize uint64) {
+	// prefers setCounts to be sorted for optimal behavior.
+	if divisibleSize < setCounts[len(setCounts)-1] {
+		return divisibleSize
+	}
+
+	// Figure out largest value of total_drives_in_erasure_set which results
+	// in least number of total_drives/total_drives_erasure_set ratio.
+	prevD := divisibleSize / setCounts[0]
+	for _, cnt := range setCounts {
+		if divisibleSize%cnt == 0 {
+			d := divisibleSize / cnt
+			if d <= prevD {
+				prevD = d
+				setSize = cnt
+			}
+		}
+	}
+	return setSize
+}
+
+// possibleSetCountsWithSymmetry returns symmetrical setCounts based on the
+// input argument patterns, the symmetry calculation is to ensure that
+// we also use uniform number of drives common across all ellipses patterns.
+func possibleSetCountsWithSymmetry(setCounts []uint64, argPatterns []ellipses.ArgPattern) []uint64 {
+	var newSetCounts = make(map[uint64]struct{})
+	for _, ss := range setCounts {
+		var symmetry bool
+		for _, argPattern := range argPatterns {
+			for _, p := range argPattern {
+				if uint64(len(p.Seq)) > ss {
+					symmetry = uint64(len(p.Seq))%ss == 0
+				} else {
+					symmetry = ss%uint64(len(p.Seq)) == 0
+				}
+			}
+		}
+		// With no arg patterns, it is expected that user knows
+		// the right symmetry, so either ellipses patterns are
+		// provided (recommended) or no ellipses patterns.
+		if _, ok := newSetCounts[ss]; !ok && (symmetry || argPatterns == nil) {
+			newSetCounts[ss] = struct{}{}
+		}
+	}
+
+	setCounts = []uint64{}
+	for setCount := range newSetCounts {
+		setCounts = append(setCounts, setCount)
+	}
+
+	// Not necessarily needed but it ensures to the readers
+	// eyes that we prefer a sorted setCount slice for the
+	// subsequent function to figure out the right common
+	// divisor, it avoids loops.
+	sort.Slice(setCounts, func(i, j int) bool {
+		return setCounts[i] < setCounts[j]
+	})
+
+	return setCounts
+}
+
 // getSetIndexes returns list of indexes which provides the set size
 // on each index, this function also determines the final set size
 // The final set size has the affinity towards choosing smaller
 // indexes (total sets)
-func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint64) (setIndexes [][]uint64, err error) {
+func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint64, argPatterns []ellipses.ArgPattern) (setIndexes [][]uint64, err error) {
 	if len(totalSizes) == 0 || len(args) == 0 {
 		return nil, errInvalidArgument
 	}
@@ -81,24 +143,7 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 		}
 	}
 
-	var setSize uint64
-
 	commonSize := getDivisibleSize(totalSizes)
-	if commonSize > setSizes[len(setSizes)-1] {
-		prevD := commonSize / setSizes[0]
-		for _, i := range setSizes {
-			if commonSize%i == 0 {
-				d := commonSize / i
-				if d <= prevD {
-					prevD = d
-					setSize = i
-				}
-			}
-		}
-	} else {
-		setSize = commonSize
-	}
-
 	possibleSetCounts := func(setSize uint64) (ss []uint64) {
 		for _, s := range setSizes {
 			if setSize%s == 0 {
@@ -114,11 +159,11 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
 	}
 
+	var setSize uint64
+	// Custom set drive count allows to override automatic distribution.
+	// only meant if you want to further optimize drive distribution.
 	if customSetDriveCount > 0 {
 		msg := fmt.Sprintf("Invalid set drive count. Acceptable values for %d number drives are %d", commonSize, setCounts)
-		if customSetDriveCount > setSize {
-			return nil, config.ErrInvalidErasureSetSize(nil).Msg(msg)
-		}
 		var found bool
 		for _, ss := range setCounts {
 			if ss == customSetDriveCount {
@@ -128,7 +173,16 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 		if !found {
 			return nil, config.ErrInvalidErasureSetSize(nil).Msg(msg)
 		}
+
+		// No automatic symmetry calculation expected, user is on their own
 		setSize = customSetDriveCount
+		globalCustomErasureDriveCount = true
+	} else {
+		// Returns possible set counts with symmetry.
+		setCounts = possibleSetCountsWithSymmetry(setCounts, argPatterns)
+
+		// Final set size with all the symmetry accounted for.
+		setSize = commonSetDriveCount(commonSize, setCounts)
 	}
 
 	// Check whether setSize is with the supported range.
@@ -202,7 +256,7 @@ func parseEndpointSet(customSetDriveCount uint64, args ...string) (ep endpointSe
 		argPatterns[i] = patterns
 	}
 
-	ep.setIndexes, err = getSetIndexes(args, getTotalSizes(argPatterns), customSetDriveCount)
+	ep.setIndexes, err = getSetIndexes(args, getTotalSizes(argPatterns), customSetDriveCount, argPatterns)
 	if err != nil {
 		return endpointSet{}, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 	}
@@ -224,7 +278,7 @@ func GetAllSets(customSetDriveCount uint64, args ...string) ([][]string, error)
 		// Check if we have more one args.
 		if len(args) > 1 {
 			var err error
-			setIndexes, err = getSetIndexes(args, []uint64{uint64(len(args))}, customSetDriveCount)
+			setIndexes, err = getSetIndexes(args, []uint64{uint64(len(args))}, customSetDriveCount, nil)
 			if err != nil {
 				return nil, err
 			}
@@ -258,6 +312,15 @@ func GetAllSets(customSetDriveCount uint64, args ...string) ([][]string, error)
 	return setArgs, nil
 }
 
+// Override set drive count for manual distribution.
+const (
+	EnvErasureSetDriveCount = "MINIO_ERASURE_SET_DRIVE_COUNT"
+)
+
+var (
+	globalCustomErasureDriveCount = false
+)
+
 // CreateServerEndpoints - validates and creates new endpoints from input args, supports
 // both ellipses and without ellipses transparently.
 func createServerEndpoints(serverAddr string, args ...string) (
@@ -268,7 +331,7 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		return nil, -1, -1, errInvalidArgument
 	}
 
-	if v := env.Get("MINIO_ERASURE_SET_DRIVE_COUNT", ""); v != "" {
+	if v := env.Get(EnvErasureSetDriveCount, ""); v != "" {
 		setDriveCount, err = strconv.Atoi(v)
 		if err != nil {
 			return nil, -1, -1, config.ErrInvalidErasureSetSize(err)

cmd/endpoint-ellipses_test.go

@@ -104,11 +104,18 @@ func TestGetSetIndexesEnvOverride(t *testing.T) {
 			8,
 			true,
 		},
+		{
+			[]string{"http://host{1...2}/data{1...180}"},
+			[]uint64{360},
+			[][]uint64{{15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}},
+			15,
+			true,
+		},
 		{
 			[]string{"http://host{1...12}/data{1...12}"},
 			[]uint64{144},
-			[][]uint64{{16, 16, 16, 16, 16, 16, 16, 16, 16}},
-			16,
+			[][]uint64{{12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12}},
+			12,
 			true,
 		},
 		{
@@ -160,7 +167,16 @@ func TestGetSetIndexesEnvOverride(t *testing.T) {
 	for _, testCase := range testCases {
 		testCase := testCase
 		t.Run("", func(t *testing.T) {
-			gotIndexes, err := getSetIndexes(testCase.args, testCase.totalSizes, testCase.envOverride)
+			var argPatterns = make([]ellipses.ArgPattern, len(testCase.args))
+			for i, arg := range testCase.args {
+				patterns, err := ellipses.FindEllipsesPatterns(arg)
+				if err != nil {
+					t.Fatalf("Unexpected failure %s", err)
+				}
+				argPatterns[i] = patterns
+			}
+
+			gotIndexes, err := getSetIndexes(testCase.args, testCase.totalSizes, testCase.envOverride, argPatterns)
 			if err != nil && testCase.success {
 				t.Errorf("Expected success but failed instead %s", err)
 			}
@@ -202,6 +218,24 @@ func TestGetSetIndexes(t *testing.T) {
 			[][]uint64{{9, 9, 9}},
 			true,
 		},
+		{
+			[]string{"http://host{1...3}/data{1...180}"},
+			[]uint64{540},
+			[][]uint64{{15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}},
+			true,
+		},
+		{
+			[]string{"http://host{1...2}.rack{1...4}/data{1...180}"},
+			[]uint64{1440},
+			[][]uint64{{16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16}},
+			true,
+		},
+		{
+			[]string{"http://host{1...2}/data{1...180}"},
+			[]uint64{360},
+			[][]uint64{{12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12}},
+			true,
+		},
 		{
 			[]string{"data/controller1/export{1...4}, data/controller2/export{1...8}, data/controller3/export{1...12}"},
 			[]uint64{4, 8, 12},
@@ -243,7 +277,15 @@ func TestGetSetIndexes(t *testing.T) {
 	for _, testCase := range testCases {
 		testCase := testCase
 		t.Run("", func(t *testing.T) {
-			gotIndexes, err := getSetIndexes(testCase.args, testCase.totalSizes, 0)
+			var argPatterns = make([]ellipses.ArgPattern, len(testCase.args))
+			for i, arg := range testCase.args {
+				patterns, err := ellipses.FindEllipsesPatterns(arg)
+				if err != nil {
+					t.Fatalf("Unexpected failure %s", err)
+				}
+				argPatterns[i] = patterns
+			}
+			gotIndexes, err := getSetIndexes(testCase.args, testCase.totalSizes, 0, argPatterns)
 			if err != nil && testCase.success {
 				t.Errorf("Expected success but failed instead %s", err)
 			}

cmd/endpoint.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
 	"net"
 	"net/url"
@@ -271,7 +270,7 @@ func hostResolveToLocalhost(endpoint Endpoint) bool {
 			"host",
 			endpoint.Hostname(),
 		)
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 		logger.LogIf(ctx, err, logger.Application)
 		return false
 	}
@@ -341,7 +340,7 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 								startTime.Add(timeElapsed),
 								"elapsed",
 								""))
-						ctx := logger.SetReqInfo(context.Background(), reqInfo)
+						ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 						logger.LogIf(ctx, err, logger.Application)
 					}
 					continue
@@ -368,7 +367,7 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 								"elapsed",
 								"",
 							))
-						ctx := logger.SetReqInfo(context.Background(),
+						ctx := logger.SetReqInfo(GlobalContext,
 							reqInfo)
 						logger.LogIf(ctx, err, logger.Application)
 					}
@@ -399,7 +398,7 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 									"elapsed",
 									"",
 								))
-							ctx := logger.SetReqInfo(context.Background(),
+							ctx := logger.SetReqInfo(GlobalContext,
 								reqInfo)
 							logger.LogIf(ctx, err, logger.Application)
 						}

cmd/erasure-encode_test.go

@@ -40,6 +40,10 @@ func (a badDisk) ReadFileStream(volume, path string, offset, length int64) (io.R
 	return nil, errFaultyDisk
 }
 
+func (a badDisk) UpdateBloomFilter(ctx context.Context, oldest, current uint64) (*bloomFilterResponse, error) {
+	return nil, errFaultyDisk
+}
+
 func (a badDisk) CreateFile(volume, path string, size int64, reader io.Reader) error {
 	return errFaultyDisk
 }

cmd/format-xl.go

@@ -378,21 +378,22 @@ func saveFormatXL(disk StorageAPI, format interface{}, diskID string) error {
 	return nil
 }
 
-var ignoredHiddenDirectories = []string{
-	minioMetaBucket,
-	".snapshot",
-	"lost+found",
-	"$RECYCLE.BIN",
-	"System Volume Information",
+var ignoredHiddenDirectories = map[string]struct{}{
+	minioMetaBucket:             {},
+	".snapshot":                 {},
+	"lost+found":                {},
+	"$RECYCLE.BIN":              {},
+	"System Volume Information": {},
 }
 
-func isIgnoreHiddenDirectories(dir string) bool {
-	for _, ignDir := range ignoredHiddenDirectories {
-		if dir == ignDir {
-			return true
+func isHiddenDirectories(vols ...VolInfo) bool {
+	for _, vol := range vols {
+		if _, ok := ignoredHiddenDirectories[vol.Name]; ok {
+			continue
 		}
+		return false
 	}
-	return false
+	return true
 }
 
 // loadFormatXL - loads format.json from disk.
@@ -407,9 +408,8 @@ func loadFormatXL(disk StorageAPI) (format *formatXLV3, err error) {
 			if err != nil {
 				return nil, err
 			}
-			if len(vols) > 1 || (len(vols) == 1 && !isIgnoreHiddenDirectories(vols[0].Name)) {
-				// 'format.json' not found, but we
-				// found user data.
+			if !isHiddenDirectories(vols...) {
+				// 'format.json' not found, but we found user data, reject such disks.
 				return nil, errCorruptedFormat
 			}
 			// No other data found, its a fresh disk.
@@ -456,7 +456,10 @@ func checkFormatXLValues(formats []*formatXLV3, drivesPerSet int) error {
 			return fmt.Errorf("%s disk is already being used in another erasure deployment. (Number of disks specified: %d but the number of disks found in the %s disk's format.json: %d)",
 				humanize.Ordinal(i+1), len(formats), humanize.Ordinal(i+1), len(formatXL.XL.Sets)*len(formatXL.XL.Sets[0]))
 		}
-		if len(formatXL.XL.Sets[0]) != drivesPerSet {
+		// Only if custom erasure drive count is set,
+		// we should fail here other proceed to honor what
+		// is present on the disk.
+		if globalCustomErasureDriveCount && len(formatXL.XL.Sets[0]) != drivesPerSet {
 			return fmt.Errorf("%s disk is already formatted with %d drives per erasure set. This cannot be changed to %d, please revert your MINIO_ERASURE_SET_DRIVE_COUNT setting", humanize.Ordinal(i+1), len(formatXL.XL.Sets[0]), drivesPerSet)
 		}
 	}
@@ -529,7 +532,7 @@ func formatXLFixDeploymentID(endpoints Endpoints, storageDisks []StorageAPI, ref
 	}
 	// Deployment ID needs to be set on all the disks.
 	// Save `format.json` across all disks.
-	return saveFormatXLAll(context.Background(), storageDisks, formats)
+	return saveFormatXLAll(GlobalContext, storageDisks, formats)
 
 }
 
@@ -559,7 +562,7 @@ func formatXLFixLocalDeploymentID(endpoints Endpoints, storageDisks []StorageAPI
 				}
 				format.ID = refFormat.ID
 				if err := saveFormatXL(storageDisks[index], format, format.XL.This); err != nil {
-					logger.LogIf(context.Background(), err)
+					logger.LogIf(GlobalContext, err)
 					return fmt.Errorf("Unable to save format.json, %w", err)
 				}
 			}

cmd/fs-v1-helpers_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"io"
 	"io/ioutil"
 	"os"
@@ -36,19 +35,19 @@ func TestFSRenameFile(t *testing.T) {
 	}
 	defer os.RemoveAll(path)
 
-	if err = fsMkdir(context.Background(), pathJoin(path, "testvolume1")); err != nil {
+	if err = fsMkdir(GlobalContext, pathJoin(path, "testvolume1")); err != nil {
 		t.Fatal(err)
 	}
-	if err = fsRenameFile(context.Background(), pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != nil {
+	if err = fsRenameFile(GlobalContext, pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != nil {
 		t.Fatal(err)
 	}
-	if err = fsRenameFile(context.Background(), pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != errFileNotFound {
+	if err = fsRenameFile(GlobalContext, pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != errFileNotFound {
 		t.Fatal(err)
 	}
-	if err = fsRenameFile(context.Background(), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2")); err != errFileNameTooLong {
+	if err = fsRenameFile(GlobalContext, pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2")); err != errFileNameTooLong {
 		t.Fatal("Unexpected error", err)
 	}
-	if err = fsRenameFile(context.Background(), pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); err != errFileNameTooLong {
+	if err = fsRenameFile(GlobalContext, pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); err != errFileNameTooLong {
 		t.Fatal("Unexpected error", err)
 	}
 }
@@ -63,30 +62,30 @@ func TestFSStats(t *testing.T) {
 
 	// Setup test environment.
 
-	if err = fsMkdir(context.Background(), ""); err != errInvalidArgument {
+	if err = fsMkdir(GlobalContext, ""); err != errInvalidArgument {
 		t.Fatal("Unexpected error", err)
 	}
 
-	if err = fsMkdir(context.Background(), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); err != errFileNameTooLong {
+	if err = fsMkdir(GlobalContext, pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); err != errFileNameTooLong {
 		t.Fatal("Unexpected error", err)
 	}
 
-	if err = fsMkdir(context.Background(), pathJoin(path, "success-vol")); err != nil {
+	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol")); err != nil {
 		t.Fatalf("Unable to create volume, %s", err)
 	}
 
 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(context.Background(), pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
+	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
 	reader.Seek(0, 0)
 
-	if err = fsMkdir(context.Background(), pathJoin(path, "success-vol", "success-file")); err != errVolumeExists {
+	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol", "success-file")); err != errVolumeExists {
 		t.Fatal("Unexpected error", err)
 	}
 
-	if _, err = fsCreateFile(context.Background(), pathJoin(path, "success-vol", "path/to/success-file"), reader, nil, 0); err != nil {
+	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "path/to/success-file"), reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -169,12 +168,12 @@ func TestFSStats(t *testing.T) {
 
 	for i, testCase := range testCases {
 		if testCase.srcPath != "" {
-			if _, err := fsStatFile(context.Background(), pathJoin(testCase.srcFSPath, testCase.srcVol,
+			if _, err := fsStatFile(GlobalContext, pathJoin(testCase.srcFSPath, testCase.srcVol,
 				testCase.srcPath)); err != testCase.expectedErr {
 				t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 			}
 		} else {
-			if _, err := fsStatVolume(context.Background(), pathJoin(testCase.srcFSPath, testCase.srcVol)); err != testCase.expectedErr {
+			if _, err := fsStatVolume(GlobalContext, pathJoin(testCase.srcFSPath, testCase.srcVol)); err != testCase.expectedErr {
 				t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 			}
 		}
@@ -189,20 +188,20 @@ func TestFSCreateAndOpen(t *testing.T) {
 	}
 	defer os.RemoveAll(path)
 
-	if err = fsMkdir(context.Background(), pathJoin(path, "success-vol")); err != nil {
+	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol")); err != nil {
 		t.Fatalf("Unable to create directory, %s", err)
 	}
 
-	if _, err = fsCreateFile(context.Background(), "", nil, nil, 0); err != errInvalidArgument {
+	if _, err = fsCreateFile(GlobalContext, "", nil, nil, 0); err != errInvalidArgument {
 		t.Fatal("Unexpected error", err)
 	}
 
-	if _, _, err = fsOpenFile(context.Background(), "", -1); err != errInvalidArgument {
+	if _, _, err = fsOpenFile(GlobalContext, "", -1); err != errInvalidArgument {
 		t.Fatal("Unexpected error", err)
 	}
 
 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(context.Background(), pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
+	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -230,18 +229,18 @@ func TestFSCreateAndOpen(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		_, err = fsCreateFile(context.Background(), pathJoin(path, testCase.srcVol, testCase.srcPath), reader, nil, 0)
+		_, err = fsCreateFile(GlobalContext, pathJoin(path, testCase.srcVol, testCase.srcPath), reader, nil, 0)
 		if err != testCase.expectedErr {
 			t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}
-		_, _, err = fsOpenFile(context.Background(), pathJoin(path, testCase.srcVol, testCase.srcPath), 0)
+		_, _, err = fsOpenFile(GlobalContext, pathJoin(path, testCase.srcVol, testCase.srcPath), 0)
 		if err != testCase.expectedErr {
 			t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}
 	}
 
 	// Attempt to open a directory.
-	if _, _, err = fsOpenFile(context.Background(), pathJoin(path), 0); err != errIsNotRegular {
+	if _, _, err = fsOpenFile(GlobalContext, pathJoin(path), 0); err != errIsNotRegular {
 		t.Fatal("Unexpected error", err)
 	}
 }
@@ -255,20 +254,20 @@ func TestFSDeletes(t *testing.T) {
 	defer os.RemoveAll(path)
 
 	// Setup test environment.
-	if err = fsMkdir(context.Background(), pathJoin(path, "success-vol")); err != nil {
+	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol")); err != nil {
 		t.Fatalf("Unable to create directory, %s", err)
 	}
 
 	var buf = make([]byte, 4096)
 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(context.Background(), pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil {
+	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
 	reader.Seek(0, io.SeekStart)
 
 	// folder is not empty
-	err = fsMkdir(context.Background(), pathJoin(path, "success-vol", "not-empty"))
+	err = fsMkdir(GlobalContext, pathJoin(path, "success-vol", "not-empty"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -278,10 +277,10 @@ func TestFSDeletes(t *testing.T) {
 	}
 
 	// recursive
-	if err = fsMkdir(context.Background(), pathJoin(path, "success-vol", "parent")); err != nil {
+	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol", "parent")); err != nil {
 		t.Fatal(err)
 	}
-	if err = fsMkdir(context.Background(), pathJoin(path, "success-vol", "parent", "dir")); err != nil {
+	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol", "parent", "dir")); err != nil {
 		t.Fatal(err)
 	}
 
@@ -343,7 +342,7 @@ func TestFSDeletes(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		if err = fsDeleteFile(context.Background(), testCase.basePath, pathJoin(testCase.basePath, testCase.srcVol, testCase.srcPath)); err != testCase.expectedErr {
+		if err = fsDeleteFile(GlobalContext, testCase.basePath, pathJoin(testCase.basePath, testCase.srcVol, testCase.srcPath)); err != testCase.expectedErr {
 			t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}
 	}
@@ -358,7 +357,7 @@ func BenchmarkFSDeleteFile(b *testing.B) {
 	defer os.RemoveAll(path)
 
 	// Setup test environment.
-	if err = fsMkdir(context.Background(), pathJoin(path, "benchmark")); err != nil {
+	if err = fsMkdir(GlobalContext, pathJoin(path, "benchmark")); err != nil {
 		b.Fatalf("Unable to create directory, %s", err)
 	}
 
@@ -375,7 +374,7 @@ func BenchmarkFSDeleteFile(b *testing.B) {
 		}
 		b.StartTimer()
 
-		err = fsDeleteFile(context.Background(), benchDir, filename)
+		err = fsDeleteFile(GlobalContext, benchDir, filename)
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -392,18 +391,18 @@ func TestFSRemoves(t *testing.T) {
 	defer os.RemoveAll(path)
 
 	// Setup test environment.
-	if err = fsMkdir(context.Background(), pathJoin(path, "success-vol")); err != nil {
+	if err = fsMkdir(GlobalContext, pathJoin(path, "success-vol")); err != nil {
 		t.Fatalf("Unable to create directory, %s", err)
 	}
 
 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(context.Background(), pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
+	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
 	reader.Seek(0, 0)
 
-	if _, err = fsCreateFile(context.Background(), pathJoin(path, "success-vol", "success-file-new"), reader, nil, 0); err != nil {
+	if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file-new"), reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
@@ -477,25 +476,25 @@ func TestFSRemoves(t *testing.T) {
 
 	for i, testCase := range testCases {
 		if testCase.srcPath != "" {
-			if err = fsRemoveFile(context.Background(), pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); err != testCase.expectedErr {
+			if err = fsRemoveFile(GlobalContext, pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); err != testCase.expectedErr {
 				t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 			}
 		} else {
-			if err = fsRemoveDir(context.Background(), pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); err != testCase.expectedErr {
+			if err = fsRemoveDir(GlobalContext, pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); err != testCase.expectedErr {
 				t.Error(err)
 			}
 		}
 	}
 
-	if err = fsRemoveAll(context.Background(), pathJoin(path, "success-vol")); err != nil {
+	if err = fsRemoveAll(GlobalContext, pathJoin(path, "success-vol")); err != nil {
 		t.Fatal(err)
 	}
 
-	if err = fsRemoveAll(context.Background(), ""); err != errInvalidArgument {
+	if err = fsRemoveAll(GlobalContext, ""); err != errInvalidArgument {
 		t.Fatal(err)
 	}
 
-	if err = fsRemoveAll(context.Background(), "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"); err != errFileNameTooLong {
+	if err = fsRemoveAll(GlobalContext, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"); err != errFileNameTooLong {
 		t.Fatal(err)
 	}
 }
@@ -509,14 +508,14 @@ func TestFSRemoveMeta(t *testing.T) {
 	defer os.RemoveAll(fsPath)
 
 	// Setup test environment.
-	if err = fsMkdir(context.Background(), pathJoin(fsPath, "success-vol")); err != nil {
+	if err = fsMkdir(GlobalContext, pathJoin(fsPath, "success-vol")); err != nil {
 		t.Fatalf("Unable to create directory, %s", err)
 	}
 
 	filePath := pathJoin(fsPath, "success-vol", "success-file")
 
 	var reader = bytes.NewReader([]byte("Hello, world"))
-	if _, err = fsCreateFile(context.Background(), filePath, reader, nil, 0); err != nil {
+	if _, err = fsCreateFile(GlobalContext, filePath, reader, nil, 0); err != nil {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 
@@ -535,7 +534,7 @@ func TestFSRemoveMeta(t *testing.T) {
 		t.Fatal(tmpErr)
 	}
 
-	if err := fsRemoveMeta(context.Background(), fsPath, filePath, tmpDir); err != nil {
+	if err := fsRemoveMeta(GlobalContext, fsPath, filePath, tmpDir); err != nil {
 		t.Fatalf("Unable to remove file, %s", err)
 	}
 
@@ -561,7 +560,7 @@ func TestFSIsFile(t *testing.T) {
 		t.Fatalf("Unable to create file %s", filePath)
 	}
 
-	if !fsIsFile(context.Background(), filePath) {
+	if !fsIsFile(GlobalContext, filePath) {
 		t.Fatalf("Expected %s to be a file", filePath)
 	}
 }

cmd/fs-v1-metadata_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -54,10 +53,10 @@ func TestReadFSMetadata(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
-	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{}); err != nil {
+	if _, err := obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 
@@ -73,7 +72,7 @@ func TestReadFSMetadata(t *testing.T) {
 
 	// Regular fs metadata reading, no errors expected
 	fsMeta := fsMetaV1{}
-	if _, err = fsMeta.ReadFrom(context.Background(), rlk.LockedFile); err != nil {
+	if _, err = fsMeta.ReadFrom(GlobalContext, rlk.LockedFile); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 }
@@ -89,10 +88,10 @@ func TestWriteFSMetadata(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
-	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{}); err != nil {
+	if _, err := obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 
@@ -108,7 +107,7 @@ func TestWriteFSMetadata(t *testing.T) {
 
 	// FS metadata reading, no errors expected (healthy disk)
 	fsMeta := fsMetaV1{}
-	_, err = fsMeta.ReadFrom(context.Background(), rlk.LockedFile)
+	_, err = fsMeta.ReadFrom(GlobalContext, rlk.LockedFile)
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}

cmd/fs-v1-multipart.go

@@ -504,6 +504,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	if _, err := fs.statBucketDir(ctx, bucket); err != nil {
 		return oi, toObjectErr(err, bucket)
 	}
+	defer ObjectPathUpdated(pathutil.Join(bucket, object))
 
 	uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID)
 	// Just check if the uploadID exists to avoid copy if it doesn't.
@@ -681,13 +682,6 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		return oi, toObjectErr(err, bucket, object)
 	}
 
-	// Deny if WORM is enabled
-	if isWORMEnabled(bucket) {
-		if _, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object)); err == nil {
-			return ObjectInfo{}, ObjectAlreadyExists{Bucket: bucket, Object: object}
-		}
-	}
-
 	err = fsRenameFile(ctx, appendFilePath, pathJoin(fs.fsPath, bucket, object))
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -757,13 +751,13 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 // Removes multipart uploads if any older than `expiry` duration
 // on all buckets for every `cleanupInterval`, this function is
 // blocking and should be run in a go-routine.
-func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh <-chan struct{}) {
+func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
 	ticker := time.NewTicker(cleanupInterval)
 	defer ticker.Stop()
 
 	for {
 		select {
-		case <-doneCh:
+		case <-ctx.Done():
 			return
 		case <-ticker.C:
 			now := time.Now()

cmd/fs-v1-multipart_test.go

@@ -39,7 +39,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 	objectName := "object"
 
 	// Create a context we can cancel.
-	ctx, cancel := context.WithCancel(context.Background())
+	ctx, cancel := context.WithCancel(GlobalContext)
 	obj.MakeBucketWithLocation(ctx, bucketName, "")
 
 	uploadID, err := obj.NewMultipartUpload(ctx, bucketName, objectName, ObjectOptions{})
@@ -51,7 +51,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 	cleanupWg.Add(1)
 	go func() {
 		defer cleanupWg.Done()
-		fs.cleanupStaleMultipartUploads(context.Background(), time.Millisecond, 0, ctx.Done())
+		fs.cleanupStaleMultipartUploads(ctx, time.Millisecond, 0)
 	}()
 
 	// Wait for 100ms such that - we have given enough time for
@@ -61,7 +61,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 	cleanupWg.Wait()
 
 	// Check if upload id was already purged.
-	if err = obj.AbortMultipartUpload(context.Background(), bucketName, objectName, uploadID); err != nil {
+	if err = obj.AbortMultipartUpload(GlobalContext, bucketName, objectName, uploadID); err != nil {
 		if _, ok := err.(InvalidUploadID); !ok {
 			t.Fatal("Unexpected err: ", err)
 		}
@@ -81,13 +81,13 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, ""); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
 	// Test with disk removed.
 	os.RemoveAll(disk)
-	if _, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}}); err != nil {
+	if _, err := fs.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}}); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}
@@ -106,11 +106,11 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 	data := []byte("12345")
 	dataLen := int64(len(data))
 
-	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, ""); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
+	uploadID, err := obj.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -121,7 +121,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 	newDisk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(newDisk)
 	obj = initFSObjects(newDisk, t)
-	if _, err = obj.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum), ObjectOptions{}); err != nil {
+	if _, err = obj.PutObjectPart(GlobalContext, bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum), ObjectOptions{}); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}
@@ -139,11 +139,11 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 	objectName := "object"
 	data := []byte("12345")
 
-	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, ""); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
+	uploadID, err := obj.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -154,7 +154,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 	newDisk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(newDisk)
 	obj = initFSObjects(newDisk, t)
-	if _, err := obj.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
+	if _, err := obj.CompleteMultipartUpload(GlobalContext, bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}
@@ -172,23 +172,23 @@ func TestCompleteMultipartUpload(t *testing.T) {
 	objectName := "object"
 	data := []byte("12345")
 
-	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, ""); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
+	uploadID, err := obj.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
 	md5Hex := getMD5Hash(data)
 
-	if _, err := obj.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
+	if _, err := obj.PutObjectPart(GlobalContext, bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
 	parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
-	if _, err := obj.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
+	if _, err := obj.CompleteMultipartUpload(GlobalContext, bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 }
@@ -204,22 +204,22 @@ func TestAbortMultipartUpload(t *testing.T) {
 	objectName := "object"
 	data := []byte("12345")
 
-	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, ""); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
+	uploadID, err := obj.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
 	md5Hex := getMD5Hash(data)
 
-	if _, err := obj.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
+	if _, err := obj.PutObjectPart(GlobalContext, bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 	time.Sleep(time.Second) // Without Sleep on windows, the fs.AbortMultipartUpload() fails with "The process cannot access the file because it is being used by another process."
-	if err := obj.AbortMultipartUpload(context.Background(), bucketName, objectName, uploadID); err != nil {
+	if err := obj.AbortMultipartUpload(GlobalContext, bucketName, objectName, uploadID); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 }
@@ -235,11 +235,11 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, ""); err != nil {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	_, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
+	_, err := obj.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -247,7 +247,7 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 	newDisk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(newDisk)
 	obj = initFSObjects(newDisk, t)
-	if _, err := obj.ListMultipartUploads(context.Background(), bucketName, objectName, "", "", "", 1000); err != nil {
+	if _, err := obj.ListMultipartUploads(GlobalContext, bucketName, objectName, "", "", "", 1000); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}

cmd/fs-v1-rwpool.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"os"
 	pathutil "path"
 	"sync"
@@ -51,7 +50,7 @@ func (fsi *fsIOPool) lookupToRead(path string) (*lock.RLockedFile, bool) {
 		if rlkFile.IsClosed() {
 			// Log this as an error.
 			reqInfo := (&logger.ReqInfo{}).AppendTags("path", path)
-			ctx := logger.SetReqInfo(context.Background(), reqInfo)
+			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 			logger.LogIf(ctx, errUnexpected)
 
 			// Purge the cached lock path from map.

cmd/fs-v1.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -41,6 +42,7 @@ import (
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/bucket/object/tagging"
 	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/mimedb"
@@ -121,7 +123,7 @@ func initMetaVolumeFS(fsPath, fsUUID string) error {
 
 // NewFSObjectLayer - initialize new fs object layer.
 func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
-	ctx := context.Background()
+	ctx := GlobalContext
 	if fsPath == "" {
 		return nil, errInvalidArgument
 	}
@@ -179,7 +181,8 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 	// or cause changes on backend format.
 	fs.fsFormatRlk = rlk
 
-	go fs.cleanupStaleMultipartUploads(ctx, GlobalMultipartCleanupInterval, GlobalMultipartExpiry, GlobalServiceDoneCh)
+	go fs.cleanupStaleMultipartUploads(ctx, GlobalMultipartCleanupInterval, GlobalMultipartExpiry)
+	go intDataUpdateTracker.start(GlobalContext, fsPath)
 
 	// Return successfully initialized object layer.
 	return fs, nil
@@ -233,7 +236,7 @@ func (fs *FSObjects) waitForLowActiveIO() {
 }
 
 // CrawlAndGetDataUsage returns data usage stats of the current FS deployment
-func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, updates chan<- DataUsageInfo) error {
+func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo) error {
 	// Load bucket totals
 	var oldCache dataUsageCache
 	err := oldCache.load(ctx, fs, dataUsageCacheName)
@@ -247,6 +250,15 @@ func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, updates chan<- Da
 	if err != nil {
 		return err
 	}
+	oldCache.Info.BloomFilter = nil
+	if bf != nil {
+		oldCache.Info.BloomFilter = bf.bytes()
+	}
+
+	if false && intDataUpdateTracker.debug {
+		b, _ := json.MarshalIndent(bf, "", "  ")
+		logger.Info("Bloom filter: %v", string(b))
+	}
 	cache, err := updateUsage(ctx, fs.fsPath, oldCache, fs.waitForLowActiveIO, func(item Item) (int64, error) {
 		// Get file size, symlinks which cannot be
 		// followed are automatically filtered by fastwalk.
@@ -256,11 +268,19 @@ func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, updates chan<- Da
 		}
 		return fi.Size(), nil
 	})
+	cache.Info.BloomFilter = nil
 
 	// Even if there was an error, the new cache may have better info.
 	if cache.Info.LastUpdate.After(oldCache.Info.LastUpdate) {
+		if intDataUpdateTracker.debug {
+			logger.Info(color.Green("CrawlAndGetDataUsage:")+" Saving cache with %d entries", len(cache.Cache))
+		}
 		logger.LogIf(ctx, cache.save(ctx, fs, dataUsageCacheName))
 		updates <- cache.dui(dataUsageRoot, buckets)
+	} else {
+		if intDataUpdateTracker.debug {
+			logger.Info(color.Green("CrawlAndGetDataUsage:")+" Cache not updated, %d entries", len(cache.Cache))
+		}
 	}
 
 	return err
@@ -452,6 +472,8 @@ func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string, forceDelet
 // update metadata.
 func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) {
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
+	defer ObjectPathUpdated(path.Join(dstBucket, dstObject))
+
 	if !cpSrcDstSame {
 		objectDWLock := fs.NewNSLock(ctx, dstBucket, dstObject)
 		if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
@@ -561,7 +583,7 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 	if HasSuffix(object, SlashSeparator) {
 		// The lock taken above is released when
 		// objReader.Close() is called by the caller.
-		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
+		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts, nsUnlocker)
 	}
 	// Take a rwPool lock for NFS gateway type deployment
 	rwPoolUnlocker := func() {}
@@ -578,7 +600,7 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		rwPoolUnlocker = func() { fs.rwPool.Close(fsMetaPath) }
 	}
 
-	objReaderFn, off, length, rErr := NewGetObjectReader(rs, objInfo, opts.CheckCopyPrecondFn, nsUnlocker, rwPoolUnlocker)
+	objReaderFn, off, length, rErr := NewGetObjectReader(rs, objInfo, opts, nsUnlocker, rwPoolUnlocker)
 	if rErr != nil {
 		return nil, rErr
 	}
@@ -871,6 +893,7 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
 	if err := checkPutObjectArgs(ctx, bucket, object, fs, r.Size()); err != nil {
 		return ObjectInfo{}, err
 	}
+
 	// Lock the object.
 	objectLock := fs.NewNSLock(ctx, bucket, object)
 	if err := objectLock.GetLock(globalObjectTimeout); err != nil {
@@ -878,6 +901,7 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
 		return objInfo, err
 	}
 	defer objectLock.Unlock()
+	defer ObjectPathUpdated(path.Join(bucket, object))
 
 	atomic.AddInt64(&fs.activeIOCount, 1)
 	defer func() {
@@ -991,12 +1015,6 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 
 	// Entire object was written to the temp location, now it's safe to rename it to the actual location.
 	fsNSObjPath := pathJoin(fs.fsPath, bucket, object)
-	// Deny if WORM is enabled
-	if isWORMEnabled(bucket) {
-		if _, err := fsStatFile(ctx, fsNSObjPath); err == nil {
-			return ObjectInfo{}, ObjectAlreadyExists{Bucket: bucket, Object: object}
-		}
-	}
 	if err = fsRenameFile(ctx, fsTmpObjPath, fsNSObjPath); err != nil {
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
@@ -1042,6 +1060,8 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) er
 		return err
 	}
 
+	defer ObjectPathUpdated(path.Join(bucket, object))
+
 	atomic.AddInt64(&fs.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt64(&fs.activeIOCount, -1)
@@ -1089,7 +1109,7 @@ func (fs *FSObjects) listDirFactory() ListDirFunc {
 		var err error
 		entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 		if err != nil && err != errFileNotFound {
-			logger.LogIf(context.Background(), err)
+			logger.LogIf(GlobalContext, err)
 			return false, nil
 		}
 		if len(entries) == 0 {
@@ -1392,6 +1412,13 @@ func (fs *FSObjects) IsCompressionSupported() bool {
 
 // IsReady - Check if the backend disk is ready to accept traffic.
 func (fs *FSObjects) IsReady(_ context.Context) bool {
-	_, err := os.Stat(fs.fsPath)
-	return err == nil
+	if _, err := os.Stat(fs.fsPath); err != nil {
+		return false
+	}
+
+	globalObjLayerMutex.RLock()
+	res := globalObjectAPI != nil && !globalSafeMode
+	globalObjLayerMutex.RUnlock()
+
+	return res
 }

cmd/fs-v1_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"bytes"
-	"context"
 	"os"
 	"path/filepath"
 	"testing"
@@ -37,11 +36,11 @@ func TestFSParentDirIsObject(t *testing.T) {
 	bucketName := "testbucket"
 	objectName := "object"
 
-	if err = obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
+	if err = obj.MakeBucketWithLocation(GlobalContext, bucketName, ""); err != nil {
 		t.Fatal(err)
 	}
 	objectContent := "12345"
-	objInfo, err := obj.PutObject(context.Background(), bucketName, objectName,
+	objInfo, err := obj.PutObject(GlobalContext, bucketName, objectName,
 		mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
@@ -84,7 +83,7 @@ func TestFSParentDirIsObject(t *testing.T) {
 		},
 	}
 	for i, testCase := range testCases {
-		gotValue := fs.parentDirIsObject(context.Background(), bucketName, testCase.objectName)
+		gotValue := fs.parentDirIsObject(GlobalContext, bucketName, testCase.objectName)
 		if testCase.parentIsObject != gotValue {
 			t.Errorf("Test %d: Unexpected value returned got %t, expected %t", i+1, gotValue, testCase.parentIsObject)
 		}
@@ -125,23 +124,23 @@ func TestFSShutdown(t *testing.T) {
 		fs := obj.(*FSObjects)
 
 		objectContent := "12345"
-		obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-		obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), ObjectOptions{})
+		obj.MakeBucketWithLocation(GlobalContext, bucketName, "")
+		obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), ObjectOptions{})
 		return fs, disk
 	}
 
 	// Test Shutdown with regular conditions
 	fs, disk := prepareTest()
-	if err := fs.Shutdown(context.Background()); err != nil {
+	if err := fs.Shutdown(GlobalContext); err != nil {
 		t.Fatal("Cannot shutdown the FS object: ", err)
 	}
 	os.RemoveAll(disk)
 
 	// Test Shutdown with faulty disk
 	fs, disk = prepareTest()
-	fs.DeleteObject(context.Background(), bucketName, objectName)
+	fs.DeleteObject(GlobalContext, bucketName, objectName)
 	os.RemoveAll(disk)
-	if err := fs.Shutdown(context.Background()); err != nil {
+	if err := fs.Shutdown(GlobalContext); err != nil {
 		t.Fatal("Got unexpected fs shutdown error: ", err)
 	}
 }
@@ -156,18 +155,18 @@ func TestFSGetBucketInfo(t *testing.T) {
 	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 
-	err := obj.MakeBucketWithLocation(context.Background(), "a", "")
+	err := obj.MakeBucketWithLocation(GlobalContext, "a", "")
 	if !isSameType(err, BucketNameInvalid{}) {
 		t.Fatal("BucketNameInvalid error not returned")
 	}
 
-	err = obj.MakeBucketWithLocation(context.Background(), bucketName, "")
+	err = obj.MakeBucketWithLocation(GlobalContext, bucketName, "")
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// Test with valid parameters
-	info, err := fs.GetBucketInfo(context.Background(), bucketName)
+	info, err := fs.GetBucketInfo(GlobalContext, bucketName)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -176,7 +175,7 @@ func TestFSGetBucketInfo(t *testing.T) {
 	}
 
 	// Test with non-existent bucket
-	_, err = fs.GetBucketInfo(context.Background(), "a")
+	_, err = fs.GetBucketInfo(GlobalContext, "a")
 	if !isSameType(err, BucketNotFound{}) {
 		t.Fatal("BucketNotFound error not returned")
 	}
@@ -184,7 +183,7 @@ func TestFSGetBucketInfo(t *testing.T) {
 	// Check for buckets and should get disk not found.
 	os.RemoveAll(disk)
 
-	if _, err = fs.GetBucketInfo(context.Background(), bucketName); err != nil {
+	if _, err = fs.GetBucketInfo(GlobalContext, bucketName); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("BucketNotFound error not returned")
 		}
@@ -200,12 +199,12 @@ func TestFSPutObject(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "1/2/3/4/object"
 
-	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, ""); err != nil {
 		t.Fatal(err)
 	}
 
 	// With a regular object.
-	_, err := obj.PutObject(context.Background(), bucketName+"non-existent", objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
+	_, err := obj.PutObject(GlobalContext, bucketName+"non-existent", objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
@@ -214,7 +213,7 @@ func TestFSPutObject(t *testing.T) {
 	}
 
 	// With a directory object.
-	_, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+SlashSeparator, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), ObjectOptions{})
+	_, err = obj.PutObject(GlobalContext, bucketName+"non-existent", objectName+SlashSeparator, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
@@ -222,11 +221,11 @@ func TestFSPutObject(t *testing.T) {
 		t.Fatalf("Expected error type BucketNotFound, got %#v", err)
 	}
 
-	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
+	_, err = obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
+	_, err = obj.PutObject(GlobalContext, bucketName, objectName+"/1", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backend corruption occurred")
 	}
@@ -241,7 +240,7 @@ func TestFSPutObject(t *testing.T) {
 		}
 	}
 
-	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), ObjectOptions{})
+	_, err = obj.PutObject(GlobalContext, bucketName, objectName+"/1/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backned corruption occurred")
 	}
@@ -268,33 +267,33 @@ func TestFSDeleteObject(t *testing.T) {
 	bucketName := "bucket"
 	objectName := "object"
 
-	obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-	obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
+	obj.MakeBucketWithLocation(GlobalContext, bucketName, "")
+	obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 
 	// Test with invalid bucket name
-	if err := fs.DeleteObject(context.Background(), "fo", objectName); !isSameType(err, BucketNameInvalid{}) {
+	if err := fs.DeleteObject(GlobalContext, "fo", objectName); !isSameType(err, BucketNameInvalid{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with bucket does not exist
-	if err := fs.DeleteObject(context.Background(), "foobucket", "fooobject"); !isSameType(err, BucketNotFound{}) {
+	if err := fs.DeleteObject(GlobalContext, "foobucket", "fooobject"); !isSameType(err, BucketNotFound{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with invalid object name
-	if err := fs.DeleteObject(context.Background(), bucketName, "\\"); !isSameType(err, ObjectNotFound{}) {
+	if err := fs.DeleteObject(GlobalContext, bucketName, "\\"); !isSameType(err, ObjectNotFound{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with object does not exist.
-	if err := fs.DeleteObject(context.Background(), bucketName, "foooobject"); !isSameType(err, ObjectNotFound{}) {
+	if err := fs.DeleteObject(GlobalContext, bucketName, "foooobject"); !isSameType(err, ObjectNotFound{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with valid condition
-	if err := fs.DeleteObject(context.Background(), bucketName, objectName); err != nil {
+	if err := fs.DeleteObject(GlobalContext, bucketName, objectName); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
 	// Delete object should err disk not found.
 	os.RemoveAll(disk)
-	if err := fs.DeleteObject(context.Background(), bucketName, objectName); err != nil {
+	if err := fs.DeleteObject(GlobalContext, bucketName, objectName); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error: ", err)
 		}
@@ -312,30 +311,30 @@ func TestFSDeleteBucket(t *testing.T) {
 	fs := obj.(*FSObjects)
 	bucketName := "bucket"
 
-	err := obj.MakeBucketWithLocation(context.Background(), bucketName, "")
+	err := obj.MakeBucketWithLocation(GlobalContext, bucketName, "")
 	if err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
 	// Test with an invalid bucket name
-	if err = fs.DeleteBucket(context.Background(), "fo", false); !isSameType(err, BucketNotFound{}) {
+	if err = fs.DeleteBucket(GlobalContext, "fo", false); !isSameType(err, BucketNotFound{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
 	// Test with an inexistant bucket
-	if err = fs.DeleteBucket(context.Background(), "foobucket", false); !isSameType(err, BucketNotFound{}) {
+	if err = fs.DeleteBucket(GlobalContext, "foobucket", false); !isSameType(err, BucketNotFound{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with a valid case
-	if err = fs.DeleteBucket(context.Background(), bucketName, false); err != nil {
+	if err = fs.DeleteBucket(GlobalContext, bucketName, false); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
-	obj.MakeBucketWithLocation(context.Background(), bucketName, "")
+	obj.MakeBucketWithLocation(GlobalContext, bucketName, "")
 
 	// Delete bucket should get error disk not found.
 	os.RemoveAll(disk)
-	if err = fs.DeleteBucket(context.Background(), bucketName, false); err != nil {
+	if err = fs.DeleteBucket(GlobalContext, bucketName, false); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error: ", err)
 		}
@@ -352,7 +351,7 @@ func TestFSListBuckets(t *testing.T) {
 	fs := obj.(*FSObjects)
 
 	bucketName := "bucket"
-	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, ""); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -367,7 +366,7 @@ func TestFSListBuckets(t *testing.T) {
 	f.Close()
 
 	// Test list buckets to have only one entry.
-	buckets, err := fs.ListBuckets(context.Background())
+	buckets, err := fs.ListBuckets(GlobalContext)
 	if err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
@@ -377,7 +376,7 @@ func TestFSListBuckets(t *testing.T) {
 
 	// Test ListBuckets with disk not found.
 	os.RemoveAll(disk)
-	if _, err := fs.ListBuckets(context.Background()); err != nil {
+	if _, err := fs.ListBuckets(GlobalContext); err != nil {
 		if err != errDiskNotFound {
 			t.Fatal("Unexpected error: ", err)
 		}
@@ -390,7 +389,7 @@ func TestFSHealObject(t *testing.T) {
 	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
-	_, err := obj.HealObject(context.Background(), "bucket", "object", madmin.HealOpts{})
+	_, err := obj.HealObject(GlobalContext, "bucket", "object", madmin.HealOpts{})
 	if err == nil || !isSameType(err, NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}
@@ -402,7 +401,7 @@ func TestFSHealObjects(t *testing.T) {
 	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
-	err := obj.HealObjects(context.Background(), "bucket", "prefix", madmin.HealOpts{}, nil)
+	err := obj.HealObjects(GlobalContext, "bucket", "prefix", madmin.HealOpts{}, nil)
 	if err == nil || !isSameType(err, NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}

cmd/gateway-main.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"net"
 	"net/url"
 	"os"
 	"os/signal"
@@ -130,11 +131,19 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	globalRootCAs, err = config.GetRootCAs(globalCertsCADir.Get())
 	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
 
+	globalMinioEndpoint = func() string {
+		host := globalMinioHost
+		if host == "" {
+			host = sortIPs(localIP4.ToSlice())[0]
+		}
+		return fmt.Sprintf("%s://%s", getURLScheme(globalIsSSL), net.JoinHostPort(host, globalMinioPort))
+	}()
+
 	// Handle gateway specific env
 	gatewayHandleEnvVars()
 
 	// Set system resources to maximum.
-	logger.LogIf(context.Background(), setMaxResources())
+	logger.LogIf(GlobalContext, setMaxResources())
 
 	// Set when gateway is enabled
 	globalIsGateway = true
@@ -197,6 +206,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	httpServer := xhttp.NewServer([]string{globalCLIContext.Addr},
 		criticalErrorHandler{registerHandlers(router, globalHandlers...)}, getCert)
+	httpServer.BaseContext = func(listener net.Listener) context.Context {
+		return GlobalContext
+	}
 	go func() {
 		globalHTTPServerErrorCh <- httpServer.Start()
 	}()
@@ -237,7 +249,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// sub-systems, make sure that we do not move the above codeblock elsewhere.
 	if enableConfigOps {
 		logger.FatalIf(globalConfigSys.Init(newObject), "Unable to initialize config system")
-		buckets, err := newObject.ListBuckets(context.Background())
+		buckets, err := newObject.ListBuckets(GlobalContext)
 		if err != nil {
 			logger.Fatal(err, "Unable to list buckets")
 		}
@@ -245,7 +257,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.FatalIf(globalNotificationSys.Init(buckets, newObject), "Unable to initialize notification system")
 		// Start watching disk for reloading config, this
 		// is only enabled for "NAS" gateway.
-		globalConfigSys.WatchConfigNASDisk(newObject)
+		globalConfigSys.WatchConfigNASDisk(GlobalContext, newObject)
 	}
 	// This is only to uniquely identify each gateway deployments.
 	globalDeploymentID = env.Get("MINIO_GATEWAY_DEPLOYMENT_ID", mustGetUUID())
@@ -255,13 +267,13 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		// ****  WARNING ****
 		// Migrating to encrypted backend on etcd should happen before initialization of
 		// IAM sub-systems, make sure that we do not move the above codeblock elsewhere.
-		logger.FatalIf(migrateIAMConfigsEtcdToEncrypted(globalEtcdClient),
+		logger.FatalIf(migrateIAMConfigsEtcdToEncrypted(GlobalContext, globalEtcdClient),
 			"Unable to handle encrypted backend for iam and policies")
 	}
 
 	if enableIAMOps {
 		// Initialize IAM sys.
-		logger.FatalIf(globalIAMSys.Init(newObject), "Unable to initialize IAM system")
+		logger.FatalIf(globalIAMSys.Init(GlobalContext, newObject), "Unable to initialize IAM system")
 	}
 
 	if globalCacheConfig.Enabled {

cmd/gateway-startup-msg.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"fmt"
 	"strings"
 
@@ -30,7 +29,7 @@ func printGatewayStartupMessage(apiEndPoints []string, backendType string) {
 	// If cache layer is enabled, print cache capacity.
 	cacheAPI := newCachedObjectLayerFn()
 	if cacheAPI != nil {
-		printCacheStorageInfo(cacheAPI.StorageInfo(context.Background()))
+		printCacheStorageInfo(cacheAPI.StorageInfo(GlobalContext))
 	}
 	// Prints credential.
 	printGatewayCommonMsg(strippedAPIEndpoints)

cmd/gateway-unsupported.go

@@ -50,7 +50,7 @@ func NewGatewayLayerWithLocker(gwLayer ObjectLayer) ObjectLayer {
 type GatewayUnsupported struct{}
 
 // CrawlAndGetDataUsage - crawl is not implemented for gateway
-func (a GatewayUnsupported) CrawlAndGetDataUsage(ctx context.Context, updates chan<- DataUsageInfo) error {
+func (a GatewayUnsupported) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo) error {
 	logger.CriticalIf(ctx, errors.New("not implemented"))
 	return NotImplemented{}
 }

cmd/gateway/azure/gateway-azure.go

@@ -24,6 +24,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"github.com/minio/minio/pkg/env"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -48,16 +49,23 @@ import (
 	minio "github.com/minio/minio/cmd"
 )
 
+var (
+	azureUploadChunkSize   = getUploadChunkSizeFromEnv(azureChunkSizeEnvVar, strconv.Itoa(azureDefaultUploadChunkSize/humanize.MiByte))
+	azureSdkTimeout        = time.Duration(azureUploadChunkSize/humanize.MiByte) * azureSdkTimeoutPerMb
+	azureUploadConcurrency = azureUploadMaxMemoryUsage / azureUploadChunkSize
+)
+
 const (
 	// The defaultDialTimeout for communicating with the cloud backends is set
 	// to 30 seconds in utils.go; the Azure SDK recommends to set a timeout of 60
 	// seconds per MB of data a client expects to upload so we must transfer less
 	// than 0.5 MB per chunk to stay within the defaultDialTimeout tolerance.
 	// See https://github.com/Azure/azure-storage-blob-go/blob/fc70003/azblob/zc_policy_retry.go#L39-L44 for more details.
-	azureUploadChunkSize      = 0.25 * humanize.MiByte
-	azureSdkTimeout           = (azureUploadChunkSize / humanize.MiByte) * 60 * time.Second
-	azureUploadMaxMemoryUsage = 10 * humanize.MiByte
-	azureUploadConcurrency    = azureUploadMaxMemoryUsage / azureUploadChunkSize
+	// To change the upload chunk size, set the environmental variable MINIO_AZURE_CHUNK_SIZE_MB with a (float) value between 0 and 100
+	azureDefaultUploadChunkSize = 25 * humanize.MiByte
+	azureSdkTimeoutPerMb        = 60 * time.Second
+	azureUploadMaxMemoryUsage   = 100 * humanize.MiByte
+	azureChunkSizeEnvVar        = "MINIO_AZURE_CHUNK_SIZE_MB"
 
 	azureDownloadRetryAttempts = 5
 	azureBlockSize             = 100 * humanize.MiByte
@@ -86,6 +94,7 @@ EXAMPLES:
   1. Start minio gateway server for Azure Blob Storage backend on custom endpoint.
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}azureaccountname
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_AZURE_CHUNK_SIZE_MB {{.AssignmentOperator}}0.25
      {{.Prompt}} {{.HelpName}} https://azureaccountname.blob.custom.azure.endpoint
 
   2. Start minio gateway server for Azure Blob Storage backend with edge caching enabled.
@@ -97,6 +106,7 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_AZURE_CHUNK_SIZE_MB {{.AssignmentOperator}}25
      {{.Prompt}} {{.HelpName}}
 `
 
@@ -125,6 +135,27 @@ func azureGatewayMain(ctx *cli.Context) {
 	minio.StartGateway(ctx, &Azure{host})
 }
 
+// getUploadChunkSizeFromEnv returns the parsed chunk size from the environmental variable 'MINIO_AZURE_CHUNK_SIZE_MB'
+// The environmental variable should be a floating point number between 0 and 100 representing the MegaBytes
+// The returned value is an int representing the size in bytes
+func getUploadChunkSizeFromEnv(envvar string, defaultValue string) int {
+	envChunkSize := env.Get(envvar, defaultValue)
+
+	i, err := strconv.ParseFloat(envChunkSize, 64)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return azureDefaultUploadChunkSize
+	}
+
+	if i <= 0 || i > 100 {
+		logger.LogIf(context.Background(), fmt.Errorf("ENV '%v' should be a floating point value between 0 and 100.\n"+
+			"The upload chunk size is set to its default: %s\n", azureChunkSizeEnvVar, defaultValue))
+		return azureDefaultUploadChunkSize
+	}
+
+	return int(i * humanize.MiByte)
+}
+
 // Azure implements Gateway.
 type Azure struct {
 	host string
@@ -264,7 +295,6 @@ func s3MetaToAzureProperties(ctx context.Context, s3Metadata map[string]string)
 			// handle it for storage.
 			k = strings.Replace(k, "X-Amz-Meta-", "", 1)
 			blobMeta[encodeKey(k)] = v
-
 		// All cases below, extract common metadata that is
 		// accepted by S3 into BlobProperties for setting on
 		// Azure - see
@@ -306,6 +336,28 @@ func newPartMetaV1(uploadID string, partID int) (partMeta *partMetadataV1) {
 	return p
 }
 
+func s3StorageClassToAzureTier(sc string) azblob.AccessTierType {
+	switch sc {
+	case "REDUCED_REDUNDANCY":
+		return azblob.AccessTierCool
+	case "STANDARD":
+		return azblob.AccessTierHot
+	}
+	return azblob.AccessTierHot
+}
+
+func azureTierToS3StorageClass(tierType string) string {
+	switch azblob.AccessTierType(tierType) {
+	case azblob.AccessTierCool:
+		return "REDUCED_REDUNDANCY"
+	case azblob.AccessTierHot:
+		return "STANDARD"
+	default:
+		return "STANDARD"
+	}
+
+}
+
 // azurePropertiesToS3Meta converts Azure metadata/properties to S3
 // metadata. It is the reverse of s3MetaToAzureProperties. Azure's
 // `.GetMetadata()` lower-cases all header keys, so this is taken into
@@ -408,6 +460,8 @@ func azureCodesToObjectError(err error, serviceCode string, statusCode int, buck
 		err = minio.PartTooBig{}
 	case "InvalidMetadata":
 		err = minio.UnsupportedMetadata{}
+	case "BlobAccessTierNotSupportedForAccountType":
+		err = minio.NotImplemented{}
 	default:
 		switch statusCode {
 		case http.StatusNotFound:
@@ -639,6 +693,7 @@ func (a *azureObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
 				ETag:            etag,
 				ContentType:     *blob.Properties.ContentType,
 				ContentEncoding: *blob.Properties.ContentEncoding,
+				UserDefined:     blob.Metadata,
 			})
 		}
 
@@ -718,7 +773,7 @@ func (a *azureObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser)
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts, pipeCloser)
 }
 
 // GetObject - reads an object from azure. Supports additional
@@ -786,6 +841,7 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 		Size:            blob.ContentLength(),
 		ContentType:     blob.ContentType(),
 		ContentEncoding: blob.ContentEncoding(),
+		StorageClass:    azureTierToS3StorageClass(blob.AccessTier()),
 	}, nil
 }
 
@@ -855,7 +911,7 @@ func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, des
 	// To handle the case where the source object should be copied without its metadata,
 	// the metadata must be removed from the dest. object after the copy completes
 	if len(azureMeta) == 0 {
-		_, err := destBlob.SetMetadata(ctx, azureMeta, azblob.BlobAccessConditions{})
+		_, err = destBlob.SetMetadata(ctx, azureMeta, azblob.BlobAccessConditions{})
 		if err != nil {
 			return objInfo, azureToObjectError(err, srcBucket, srcObject)
 		}
@@ -865,6 +921,15 @@ func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, des
 	if err != nil {
 		return objInfo, azureToObjectError(err, srcBucket, srcObject)
 	}
+
+	if _, ok := srcInfo.UserDefined["x-amz-storage-class"]; ok {
+		_, err = destBlob.SetTier(ctx, s3StorageClassToAzureTier(srcInfo.UserDefined["x-amz-storage-class"]),
+			azblob.LeaseAccessConditions{})
+		if err != nil {
+			return objInfo, azureToObjectError(err, srcBucket, srcObject)
+		}
+	}
+
 	return a.GetObjectInfo(ctx, destBucket, destObject, dstOpts)
 }
 

cmd/gateway/azure/gateway-azure_test.go

@@ -17,11 +17,13 @@
 package azure
 
 import (
-	"context"
 	"encoding/base64"
 	"fmt"
+	"github.com/dustin/go-humanize"
 	"net/http"
+	"os"
 	"reflect"
+	"strconv"
 	"testing"
 
 	"github.com/Azure/azure-storage-blob-go/azblob"
@@ -84,7 +86,7 @@ func TestS3MetaToAzureProperties(t *testing.T) {
 		"X_Amz_Matdesc":    "{}",
 		"X_Amz_Iv":         "eWmyryl8kq+EVnnsE7jpOg==",
 	}
-	meta, _, err := s3MetaToAzureProperties(context.Background(), headers)
+	meta, _, err := s3MetaToAzureProperties(minio.GlobalContext, headers)
 	if err != nil {
 		t.Fatalf("Test failed, with %s", err)
 	}
@@ -94,7 +96,7 @@ func TestS3MetaToAzureProperties(t *testing.T) {
 	headers = map[string]string{
 		"invalid--meta": "value",
 	}
-	_, _, err = s3MetaToAzureProperties(context.Background(), headers)
+	_, _, err = s3MetaToAzureProperties(minio.GlobalContext, headers)
 	if err != nil {
 		if _, ok := err.(minio.UnsupportedMetadata); !ok {
 			t.Fatalf("Test failed with unexpected error %s, expected UnsupportedMetadata", err)
@@ -104,7 +106,7 @@ func TestS3MetaToAzureProperties(t *testing.T) {
 	headers = map[string]string{
 		"content-md5": "Dce7bmCX61zvxzP5QmfelQ==",
 	}
-	_, props, err := s3MetaToAzureProperties(context.Background(), headers)
+	_, props, err := s3MetaToAzureProperties(minio.GlobalContext, headers)
 	if err != nil {
 		t.Fatalf("Test failed, with %s", err)
 	}
@@ -286,7 +288,7 @@ func TestCheckAzureUploadID(t *testing.T) {
 	}
 
 	for _, uploadID := range invalidUploadIDs {
-		if err := checkAzureUploadID(context.Background(), uploadID); err == nil {
+		if err := checkAzureUploadID(minio.GlobalContext, uploadID); err == nil {
 			t.Fatalf("%s: expected: <error>, got: <nil>", uploadID)
 		}
 	}
@@ -297,8 +299,41 @@ func TestCheckAzureUploadID(t *testing.T) {
 	}
 
 	for _, uploadID := range validUploadIDs {
-		if err := checkAzureUploadID(context.Background(), uploadID); err != nil {
+		if err := checkAzureUploadID(minio.GlobalContext, uploadID); err != nil {
 			t.Fatalf("%s: expected: <nil>, got: %s", uploadID, err)
 		}
 	}
 }
+
+func TestParsingUploadChunkSize(t *testing.T) {
+	key := "MINIO_AZURE_CHUNK_SIZE_MB"
+	invalidValues := []string{
+		"",
+		"0,3",
+		"100.1",
+		"-1",
+	}
+
+	for i, chunkValue := range invalidValues {
+		os.Setenv(key, chunkValue)
+		result := getUploadChunkSizeFromEnv(key, strconv.Itoa(azureDefaultUploadChunkSize/humanize.MiByte))
+		if result != azureDefaultUploadChunkSize {
+			t.Errorf("Test %d: expected: %d, got: %d", i+1, azureDefaultUploadChunkSize, result)
+		}
+	}
+
+	validValues := []string{
+		"1",
+		"1.25",
+		"50",
+		"99",
+	}
+	for i, chunkValue := range validValues {
+		os.Setenv(key, chunkValue)
+		result := getUploadChunkSizeFromEnv(key, strconv.Itoa(azureDefaultUploadChunkSize/humanize.MiByte))
+		if result == azureDefaultUploadChunkSize {
+			t.Errorf("Test %d: expected: %d, got: %d", i+1, azureDefaultUploadChunkSize, result)
+		}
+	}
+
+}

cmd/gateway/b2/gateway-b2.go

@@ -87,8 +87,13 @@ EXAMPLES:
 
 // Handler for 'minio gateway b2' command line.
 func b2GatewayMain(ctx *cli.Context) {
+	strictS3Compat := true
+	if ctx.IsSet("no-compat") || ctx.GlobalIsSet("no-compat") {
+		strictS3Compat = false
+	}
+
 	minio.StartGateway(ctx, &B2{
-		strictS3Compat: ctx.IsSet("compat") || ctx.GlobalIsSet("compat"),
+		strictS3Compat: strictS3Compat,
 	})
 }
 
@@ -105,7 +110,7 @@ func (g *B2) Name() string {
 // NewGatewayLayer returns b2 gateway layer, implements ObjectLayer interface to
 // talk to B2 remote backend.
 func (g *B2) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
-	ctx := context.Background()
+	ctx := minio.GlobalContext
 	client, err := b2.AuthorizeAccount(ctx, creds.AccessKey, creds.SecretKey, b2.Transport(minio.NewGatewayHTTPTransport()))
 	if err != nil {
 		return nil, err
@@ -473,7 +478,7 @@ func (l *b2Objects) GetObjectNInfo(ctx context.Context, bucket, object string, r
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser)
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts, pipeCloser)
 }
 
 // GetObject reads an object from B2. Supports additional

cmd/gateway/gateway.go

@@ -22,7 +22,6 @@ import (
 	_ "github.com/minio/minio/cmd/gateway/gcs"
 	_ "github.com/minio/minio/cmd/gateway/hdfs"
 	_ "github.com/minio/minio/cmd/gateway/nas"
-	_ "github.com/minio/minio/cmd/gateway/oss"
 	_ "github.com/minio/minio/cmd/gateway/s3"
 
 	// B2 is specifically kept here to avoid re-ordering by goimports,

cmd/gateway/gcs/gateway-gcs.go

@@ -144,12 +144,12 @@ EXAMPLES:
 func gcsGatewayMain(ctx *cli.Context) {
 	projectID := ctx.Args().First()
 	if projectID == "" && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" {
-		logger.LogIf(context.Background(), errGCSProjectIDNotFound, logger.Application)
+		logger.LogIf(minio.GlobalContext, errGCSProjectIDNotFound, logger.Application)
 		cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
 	}
 	if projectID != "" && !isValidGCSProjectIDFormat(projectID) {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("projectID", ctx.Args().First())
-		contxt := logger.SetReqInfo(context.Background(), reqInfo)
+		contxt := logger.SetReqInfo(minio.GlobalContext, reqInfo)
 		logger.LogIf(contxt, errGCSInvalidProjectID, logger.Application)
 		cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
 	}
@@ -169,7 +169,7 @@ func (g *GCS) Name() string {
 
 // NewGatewayLayer returns gcs ObjectLayer.
 func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
-	ctx := context.Background()
+	ctx := minio.GlobalContext
 
 	var err error
 	if g.projectID == "" {
@@ -373,7 +373,7 @@ func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(ctx context.Context, bucket str
 		if err != nil {
 			if err != iterator.Done {
 				reqInfo := &logger.ReqInfo{BucketName: bucket}
-				ctx := logger.SetReqInfo(context.Background(), reqInfo)
+				ctx := logger.SetReqInfo(minio.GlobalContext, reqInfo)
 				logger.LogIf(ctx, err)
 			}
 			return
@@ -383,7 +383,7 @@ func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(ctx context.Context, bucket str
 			err := l.client.Bucket(bucket).Object(attrs.Name).Delete(ctx)
 			if err != nil {
 				reqInfo := &logger.ReqInfo{BucketName: bucket, ObjectName: attrs.Name}
-				ctx := logger.SetReqInfo(context.Background(), reqInfo)
+				ctx := logger.SetReqInfo(minio.GlobalContext, reqInfo)
 				logger.LogIf(ctx, err)
 				return
 			}
@@ -754,7 +754,7 @@ func (l *gcsGateway) GetObjectNInfo(ctx context.Context, bucket, object string,
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser)
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts, pipeCloser)
 }
 
 // GetObject - reads an object from GCS. Supports additional

cmd/gateway/hdfs/gateway-hdfs.go

@@ -332,13 +332,13 @@ func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 			if os.IsNotExist(err) {
 				err = nil
 			}
-			logger.LogIf(context.Background(), err)
+			logger.LogIf(minio.GlobalContext, err)
 			return
 		}
 		defer f.Close()
 		fis, err := f.Readdir(0)
 		if err != nil {
-			logger.LogIf(context.Background(), err)
+			logger.LogIf(minio.GlobalContext, err)
 			return
 		}
 		if len(fis) == 0 {
@@ -466,7 +466,7 @@ func (n *hdfsObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser)
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts, pipeCloser)
 
 }
 

cmd/gateway/oss/gateway-oss_test.go

@@ -1,190 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package oss
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"reflect"
-	"testing"
-
-	"github.com/aliyun/aliyun-oss-go-sdk/oss"
-
-	minio "github.com/minio/minio/cmd"
-)
-
-func ossErrResponse(code string) error {
-	return oss.ServiceError{
-		Code: code,
-	}
-}
-
-func TestOSSToObjectError(t *testing.T) {
-	testCases := []struct {
-		inputErr       error
-		expectedErr    error
-		bucket, object string
-	}{
-		{
-			inputErr:    ossErrResponse("BucketAlreadyExists"),
-			expectedErr: minio.BucketAlreadyOwnedByYou{},
-		},
-		{
-			inputErr:    ossErrResponse("BucketNotEmpty"),
-			expectedErr: minio.BucketNotEmpty{},
-		},
-		{
-			inputErr:    ossErrResponse("InvalidBucketName"),
-			expectedErr: minio.BucketNameInvalid{},
-		},
-		{
-			inputErr:    ossErrResponse("NoSuchBucket"),
-			expectedErr: minio.BucketNotFound{},
-		},
-		// with empty object, NoSuchKey is interpreted as BucketNotFound
-		{
-			inputErr:    ossErrResponse("NoSuchKey"),
-			expectedErr: minio.BucketNotFound{},
-		},
-		{
-			inputErr:    ossErrResponse("NoSuchUpload"),
-			expectedErr: minio.InvalidUploadID{},
-		},
-		{
-			inputErr:    ossErrResponse("InvalidObjectName"),
-			expectedErr: minio.ObjectNameInvalid{},
-		},
-		{
-			inputErr:    ossErrResponse("AccessDenied"),
-			expectedErr: minio.PrefixAccessDenied{},
-		},
-		{
-			inputErr:    ossErrResponse("NoSuchUpload"),
-			expectedErr: minio.InvalidUploadID{},
-		},
-		{
-			inputErr:    ossErrResponse("EntityTooSmall"),
-			expectedErr: minio.PartTooSmall{},
-		},
-		{
-			inputErr:    nil,
-			expectedErr: nil,
-		},
-		// Special test case for NoSuchKey with object name
-		{
-			inputErr:    ossErrResponse("NoSuchKey"),
-			expectedErr: minio.ObjectNotFound{Bucket: "bucket", Object: "object"},
-			bucket:      "bucket",
-			object:      "object",
-		},
-
-		// Special test case for error value that is not of
-		// type (*Error)
-		{
-			inputErr:    fmt.Errorf("not a *Error"),
-			expectedErr: fmt.Errorf("not a *Error"),
-		},
-	}
-
-	for i, tc := range testCases {
-		actualErr := ossToObjectError(tc.inputErr, tc.bucket, tc.object)
-		if actualErr != nil && tc.expectedErr != nil && actualErr.Error() != tc.expectedErr.Error() {
-			t.Errorf("Test case %d: Expected error '%v' but received error '%v'", i+1, tc.expectedErr, actualErr)
-		}
-	}
-}
-
-func TestS3MetaToOSSOptions(t *testing.T) {
-	var err error
-
-	headers := map[string]string{
-		"x-amz-meta-invalid_meta": "value",
-	}
-	_, err = appendS3MetaToOSSOptions(context.Background(), nil, headers)
-	if err != nil {
-		if _, ok := err.(minio.UnsupportedMetadata); !ok {
-			t.Fatalf("Test failed with unexpected error %s, expected UnsupportedMetadata", err)
-		}
-	}
-
-	headers = map[string]string{
-		"accept-encoding":          "gzip", // not this
-		"content-encoding":         "gzip",
-		"X-Amz-Meta-Hdr":           "value",
-		"X-Amz-Meta-X-test-key":    "value",
-		"X-Amz-Meta-X--test--key":  "value",
-		"X-Amz-Meta-X-Amz-Key":     "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
-		"X-Amz-Meta-X-Amz-Matdesc": "{}",
-		"X-Amz-Meta-X-Amz-Iv":      "eWmyryl8kq+EVnnsE7jpOg==",
-	}
-	opts, err := appendS3MetaToOSSOptions(context.Background(), nil, headers)
-	if err != nil {
-		t.Fatalf("Test failed, with %s", err)
-	}
-	if len(opts) != len(headers) {
-		t.Fatalf("Test failed, S3 metdata is not fully transformed. expeted: %d, actual: %d", len(headers)-1, len(opts))
-	}
-}
-
-func TestOSSHeaderToS3Meta(t *testing.T) {
-	meta := map[string]string{
-		"x-oss-meta-first_name":       "myname",
-		"X-OSS-Meta-x_test_key":       "value",
-		"X-Oss-Meta-x_test__key":      "value",
-		"X-Oss-Meta-x__test__key":     "value",
-		"X-Oss-Meta-x____test____key": "value",
-		"X-Oss-Meta-x_amz_key":        "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
-		"X-Oss-Meta-x_amz_matdesc":    "{}",
-		"x-oss-meta-x_amz_iv":         "eWmyryl8kq+EVnnsE7jpOg==",
-	}
-	header := make(http.Header)
-	for k, v := range meta {
-		header.Set(k, v)
-	}
-
-	expectedMeta := map[string]string{
-		"X-Amz-Meta-First-Name":    "myname",
-		"X-Amz-Meta-X-Test-Key":    "value",
-		"X-Amz-Meta-X-Test_key":    "value",
-		"X-Amz-Meta-X_test_key":    "value",
-		"X-Amz-Meta-X__test__key":  "value",
-		"X-Amz-Meta-X-Amz-Key":     "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
-		"X-Amz-Meta-X-Amz-Matdesc": "{}",
-		"X-Amz-Meta-X-Amz-Iv":      "eWmyryl8kq+EVnnsE7jpOg==",
-	}
-	actualMeta := ossHeaderToS3Meta(header)
-	for k, v := range expectedMeta {
-		if v2, ok := actualMeta[k]; !ok {
-			t.Errorf("Test failed for key %s: missing key", k)
-		} else if v != v2 {
-			t.Errorf("Test failed for key %s, expected '%s', got '%s'", k, v, v2)
-		}
-	}
-}
-
-func TestOSSBuildListObjectPartsParams(t *testing.T) {
-	expected := map[string]interface{}{
-		"uploadId":           "test",
-		"part-number-marker": "123",
-		"max-parts":          "456",
-	}
-	actual := ossBuildListObjectPartsParams("test", 123, 456)
-	if !reflect.DeepEqual(expected, actual) {
-		t.Fatalf("Test failed, expected %v, got %v", expected, actual)
-	}
-}

cmd/gateway/s3/gateway-s3-metadata_test.go

@@ -18,8 +18,9 @@ package s3
 
 import (
 	"bytes"
-	"context"
 	"testing"
+
+	minio "github.com/minio/minio/cmd"
 )
 
 // Tests for GW metadata format validity.
@@ -62,7 +63,7 @@ func TestReadGWMetadata(t *testing.T) {
 
 	for i, tt := range tests {
 		buf := bytes.NewBufferString(tt.metaStr)
-		m, err := readGWMetadata(context.Background(), *buf)
+		m, err := readGWMetadata(minio.GlobalContext, *buf)
 		if err != nil && tt.pass {
 			t.Errorf("Test %d: Expected parse gw metadata to succeed, but failed, %s", i+1, err)
 		}

cmd/gateway/s3/gateway-s3-sse.go

@@ -315,7 +315,7 @@ func (l *s3EncObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		return l.s3Objects.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 	}
 	objInfo.UserDefined = minio.CleanMinioInternalMetadataKeys(objInfo.UserDefined)
-	fn, off, length, err := minio.NewGetObjectReader(rs, objInfo, o.CheckCopyPrecondFn)
+	fn, off, length, err := minio.NewGetObjectReader(rs, objInfo, o)
 	if err != nil {
 		return nil, minio.ErrorRespToObjectError(err)
 	}
@@ -679,13 +679,13 @@ func getGWContentPath(object string) string {
 }
 
 // Clean-up the stale incomplete encrypted multipart uploads. Should be run in a Go routine.
-func (l *s3EncObjects) cleanupStaleEncMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh <-chan struct{}) {
+func (l *s3EncObjects) cleanupStaleEncMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
 	ticker := time.NewTicker(cleanupInterval)
 	defer ticker.Stop()
 
 	for {
 		select {
-		case <-doneCh:
+		case <-ctx.Done():
 			return
 		case <-ticker.C:
 			l.cleanupStaleEncMultipartUploadsOnGW(ctx, expiry)

cmd/gateway/s3/gateway-s3.go

@@ -241,8 +241,8 @@ func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
 		encS := s3EncObjects{s}
 
 		// Start stale enc multipart uploads cleanup routine.
-		go encS.cleanupStaleEncMultipartUploads(context.Background(),
-			minio.GlobalMultipartCleanupInterval, minio.GlobalMultipartExpiry, minio.GlobalServiceDoneCh)
+		go encS.cleanupStaleEncMultipartUploads(minio.GlobalContext,
+			minio.GlobalMultipartCleanupInterval, minio.GlobalMultipartExpiry)
 
 		return &encS, nil
 	}
@@ -401,7 +401,7 @@ func (l *s3Objects) GetObjectNInfo(ctx context.Context, bucket, object string, r
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser)
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts, pipeCloser)
 }
 
 // GetObject reads an object from S3. Supports additional

cmd/generic-handlers.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"net/http"
 	"strings"
 	"time"
@@ -723,7 +722,7 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
 		PassHost:     true,
 		RoundTripper: NewGatewayHTTPTransport(),
 		Logger: func(err error) {
-			logger.LogIf(context.Background(), err)
+			logger.LogIf(GlobalContext, err)
 		},
 	})
 	return bucketForwardingHandler{fwd, h}

cmd/globals.go

@@ -138,6 +138,8 @@ var (
 	globalMinioPort = globalMinioDefaultPort
 	// Holds the host that was passed using --address
 	globalMinioHost = ""
+	// Holds the possible host endpoint.
+	globalMinioEndpoint = ""
 
 	// globalConfigSys server config system.
 	globalConfigSys *ConfigSys
@@ -153,6 +155,10 @@ var (
 	globalLifecycleSys       *LifecycleSys
 	globalBucketSSEConfigSys *BucketSSEConfigSys
 
+	// globalAPIThrottling controls S3 requests throttling when
+	// enabled in the config or in the shell environment.
+	globalAPIThrottling apiThrottling
+
 	globalStorageClass storageclass.Config
 	globalLDAPConfig   xldap.Config
 	globalOpenIDConfig openid.Config
@@ -209,9 +215,6 @@ var (
 	globalOperationTimeout = newDynamicTimeout(10*time.Minute /*30*/, 600*time.Second)         // default timeout for general ops
 	globalHealingTimeout   = newDynamicTimeout(30*time.Minute /*1*/, 30*time.Minute)           // timeout for healing related ops
 
-	// Is worm enabled
-	globalWORMEnabled bool
-
 	globalBucketObjectLockConfig = objectlock.NewBucketObjectLockConfig()
 
 	// Disk cache drives

cmd/handler-utils.go

@@ -28,7 +28,6 @@ import (
 	"net/url"
 	"regexp"
 	"strings"
-	"time"
 
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -51,7 +50,7 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError
 	locationConstraint := createBucketLocationConfiguration{}
 	err := xmlDecoder(r.Body, &locationConstraint, r.ContentLength)
 	if err != nil && r.ContentLength != 0 {
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(GlobalContext, err)
 		// Treat all other failures as XML parsing errors.
 		return "", ErrMalformedXML
 	} // else for both err as nil or io.EOF
@@ -359,31 +358,6 @@ func httpTraceHdrs(f http.HandlerFunc) http.HandlerFunc {
 	}
 }
 
-// maxClients throttles the S3 API calls
-func maxClients(f http.HandlerFunc, enabled bool, requestsMaxCh chan struct{}, requestsDeadline time.Duration) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		if !enabled {
-			f.ServeHTTP(w, r)
-			return
-		}
-
-		select {
-		case requestsMaxCh <- struct{}{}:
-			defer func() { <-requestsMaxCh }()
-			f.ServeHTTP(w, r)
-		case <-time.NewTimer(requestsDeadline).C:
-
-			// Send a http timeout message
-			writeErrorResponse(r.Context(), w,
-				errorCodes.ToAPIErr(ErrOperationMaxedOut),
-				r.URL, guessIsBrowserReq(r))
-			return
-		case <-r.Context().Done():
-			return
-		}
-	}
-}
-
 func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 
@@ -431,7 +405,7 @@ func getResource(path string, host string, domains []string) (string, error) {
 		if host, _, err = net.SplitHostPort(host); err != nil {
 			reqInfo := (&logger.ReqInfo{}).AppendTags("host", host)
 			reqInfo.AppendTags("path", path)
-			ctx := logger.SetReqInfo(context.Background(), reqInfo)
+			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 			logger.LogIf(ctx, err)
 			return "", err
 		}

cmd/http/headers.go

@@ -106,4 +106,7 @@ const (
 
 	// Server-Status
 	MinIOServerStatus = "x-minio-server-status"
+
+	// Delete special flag
+	MinIOForceDelete = "x-minio-force-delete"
 )