Support any string as delimiter for listing (#7882)

Fixes #7836

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,33 +1,19 @@
-<!--- Provide a general summary of your changes in the Title above -->
-
 ## Description
-<!--- Describe your changes in detail -->
+
 
 ## Motivation and Context
-<!--- Why is this change required? What problem does it solve? -->
-<!--- If it fixes an open issue, please link to the issue here. -->
 
-## Regression
-<!-- Is this PR fixing a regression? (Yes / No) -->
-<!-- If Yes, optionally please include minio version or commit id or PR# that caused this regression, if you have these details. -->
 
-## How Has This Been Tested?
-<!--- Please describe in detail how you tested your changes. -->
-<!--- Include details of your testing environment, and the tests you ran to -->
-<!--- see how your change affects other areas of the code, etc. -->
+## How to test this PR?
+
 
 ## Types of changes
-<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
 - [ ] Bug fix (non-breaking change which fixes an issue)
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Breaking change (fix or feature that would cause existing functionality to change)
 
 ## Checklist:
-<!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
-<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
-- [ ] My code follows the code style of this project.
-- [ ] My change requires a change to the documentation.
-- [ ] I have updated the documentation accordingly.
-- [ ] I have added unit tests to cover my changes.
-- [ ] I have added/updated functional tests in [mint](https://github.com/minio/mint). (If yes, add `mint` PR # here: )
-- [ ] All new and existing tests passed.
+- [ ] Fixes a regression (If yes, please add `commit-id` or `PR #` here)
+- [ ] Documentation needed
+- [ ] Unit tests needed
+- [ ] Functional tests needed (If yes, add [mint](https://github.com/minio/mint) PR # here: )

Dockerfile

@@ -27,7 +27,7 @@ COPY --from=0 /usr/bin/check-user /usr/bin/check-user
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \
-     apk add --no-cache ca-certificates 'curl>7.61.0' && \
+     apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]

browser/README.md

@@ -4,10 +4,11 @@
 
 ## Installation
 
-### Install yarn
+### Install node
 ```sh
-curl -o- -L https://yarnpkg.com/install.sh | bash
-yarn
+curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+exec -l $SHELL
+nvm install stable
 ```
 
 ### Install `go-bindata` and `go-bindata-assetfs`
@@ -15,8 +16,8 @@ yarn
 If you do not have a working Golang environment, please follow [Install Golang](https://docs.min.io/docs/how-to-install-golang)
 
 ```sh
-go get github.com/jteeuwen/go-bindata/...
-go get github.com/elazarl/go-bindata-assetfs/...
+go get github.com/go-bindata/go-bindata/go-bindata
+go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs
 ```
 
 ## Generating Assets
@@ -24,7 +25,7 @@ go get github.com/elazarl/go-bindata-assetfs/...
 ### Generate ui-assets.go
 
 ```sh
-yarn release
+npm run release
 ```
 
 This generates ui-assets.go in the current directory. Now do `make` in the parent directory to build the minio binary with the newly generated ``ui-assets.go``
@@ -32,7 +33,7 @@ This generates ui-assets.go in the current directory. Now do `make` in the paren
 ### Run MinIO Browser with live reload
 
 ```sh
-yarn dev
+npm run dev
 ```
 
 Open [http://localhost:8080/minio/](http://localhost:8080/minio/) in your browser to play with the application
@@ -66,7 +67,7 @@ index 3ccdaba..9496c56 100644
 ```
 
 ```sh
-yarn dev
+npm run dev
 ```
 
 Open [http://localhost:8888/minio/](http://localhost:8888/minio/) in your browser to play with the application

browser/app/js/constants.js

@@ -31,3 +31,10 @@ export const SHARE_OBJECT_EXPIRY_MINUTES = 0
 
 export const ACCESS_KEY_MIN_LENGTH = 3
 export const SECRET_KEY_MIN_LENGTH = 8
+
+export const SORT_BY_NAME = "name"
+export const SORT_BY_SIZE = "size"
+export const SORT_BY_LAST_MODIFIED = "last-modified"
+
+export const SORT_ORDER_ASC = "asc"
+export const SORT_ORDER_DESC = "desc"

browser/app/js/objects/ObjectItem.js

@@ -51,7 +51,7 @@ export const ObjectItem = ({
       </div>
       <div className="fesl-item fesl-item-name">
         <a
-          href="#"
+          href={getDataType(name, contentType) === "folder" ? name : "#"}
           onClick={e => {
             e.preventDefault()
             if (onClick) {

browser/app/js/objects/ObjectsHeader.js

@@ -18,11 +18,19 @@ import React from "react"
 import classNames from "classnames"
 import { connect } from "react-redux"
 import * as actionsObjects from "./actions"
+import {
+  SORT_BY_NAME,
+  SORT_BY_SIZE,
+  SORT_BY_LAST_MODIFIED,
+  SORT_ORDER_DESC,
+  SORT_ORDER_ASC
+} from "../constants"
 
 export const ObjectsHeader = ({
-  sortNameOrder,
-  sortSizeOrder,
-  sortLastModifiedOrder,
+  sortedByName,
+  sortedBySize,
+  sortedByLastModified,
+  sortOrder,
   sortObjects
 }) => (
   <div className="feb-container">
@@ -31,48 +39,54 @@ export const ObjectsHeader = ({
       <div
         className="fesl-item fesl-item-name"
         id="sort-by-name"
-        onClick={() => sortObjects("name")}
+        onClick={() => sortObjects(SORT_BY_NAME)}
         data-sort="name"
       >
         Name
         <i
           className={classNames({
             "fesli-sort": true,
+            "fesli-sort--active": sortedByName,
             fa: true,
-            "fa-sort-alpha-desc": sortNameOrder,
-            "fa-sort-alpha-asc": !sortNameOrder
+            "fa-sort-alpha-desc": sortedByName && sortOrder === SORT_ORDER_DESC,
+            "fa-sort-alpha-asc": sortedByName && sortOrder === SORT_ORDER_ASC
           })}
         />
       </div>
       <div
         className="fesl-item fesl-item-size"
         id="sort-by-size"
-        onClick={() => sortObjects("size")}
+        onClick={() => sortObjects(SORT_BY_SIZE)}
         data-sort="size"
       >
         Size
         <i
           className={classNames({
             "fesli-sort": true,
+            "fesli-sort--active": sortedBySize,
             fa: true,
-            "fa-sort-amount-desc": sortSizeOrder,
-            "fa-sort-amount-asc": !sortSizeOrder
+            "fa-sort-amount-desc":
+              sortedBySize && sortOrder === SORT_ORDER_DESC,
+            "fa-sort-amount-asc": sortedBySize && sortOrder === SORT_ORDER_ASC
           })}
         />
       </div>
       <div
         className="fesl-item fesl-item-modified"
         id="sort-by-last-modified"
-        onClick={() => sortObjects("last-modified")}
+        onClick={() => sortObjects(SORT_BY_LAST_MODIFIED)}
         data-sort="last-modified"
       >
         Last Modified
         <i
           className={classNames({
             "fesli-sort": true,
+            "fesli-sort--active": sortedByLastModified,
             fa: true,
-            "fa-sort-numeric-desc": sortLastModifiedOrder,
-            "fa-sort-numeric-asc": !sortLastModifiedOrder
+            "fa-sort-numeric-desc":
+              sortedByLastModified && sortOrder === SORT_ORDER_DESC,
+            "fa-sort-numeric-asc":
+              sortedByLastModified && sortOrder === SORT_ORDER_ASC
           })}
         />
       </div>
@@ -83,10 +97,10 @@ export const ObjectsHeader = ({
 
 const mapStateToProps = state => {
   return {
-    sortNameOrder: state.objects.sortBy == "name" && state.objects.sortOrder,
-    sortSizeOrder: state.objects.sortBy == "size" && state.objects.sortOrder,
-    sortLastModifiedOrder:
-      state.objects.sortBy == "last-modified" && state.objects.sortOrder
+    sortedByName: state.objects.sortBy == SORT_BY_NAME,
+    sortedBySize: state.objects.sortBy == SORT_BY_SIZE,
+    sortedByLastModified: state.objects.sortBy == SORT_BY_LAST_MODIFIED,
+    sortOrder: state.objects.sortOrder
   }
 }
 
@@ -96,4 +110,7 @@ const mapDispatchToProps = dispatch => {
   }
 }
 
-export default connect(mapStateToProps, mapDispatchToProps)(ObjectsHeader)
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps
+)(ObjectsHeader)

browser/app/js/objects/ObjectsListContainer.js

@@ -15,32 +15,52 @@
  */
 
 import React from "react"
-import classNames from "classnames"
 import { connect } from "react-redux"
 import InfiniteScroll from "react-infinite-scroller"
-import * as actionsObjects from "./actions"
 import ObjectsList from "./ObjectsList"
 
 export class ObjectsListContainer extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      page: 1
+    }
+    this.loadNextPage = this.loadNextPage.bind(this)
+  }
+  componentWillReceiveProps(nextProps) {
+    if (
+      nextProps.currentBucket !== this.props.currentBucket ||
+      nextProps.currentPrefix !== this.props.currentPrefix ||
+      nextProps.sortBy !== this.props.sortBy ||
+      nextProps.sortOrder !== this.props.sortOrder
+    ) {
+      this.setState({
+        page: 1
+      })
+    }
+  }
+  loadNextPage() {
+    this.setState(state => {
+      return { page: state.page + 1 }
+    })
+  }
   render() {
-    const { objects, isTruncated, currentBucket, loadObjects } = this.props
+    const { objects, listLoading } = this.props
+
+    const visibleObjects = objects.slice(0, this.state.page * 100)
+
     return (
-      <div className="feb-container">
+      <div style={{ position: "relative" }}>
         <InfiniteScroll
           pageStart={0}
-          loadMore={() => loadObjects(true)}
-          hasMore={isTruncated}
+          loadMore={this.loadNextPage}
+          hasMore={objects.length > visibleObjects.length}
           useWindow={true}
           initialLoad={false}
         >
-          <ObjectsList objects={objects} />
+          <ObjectsList objects={visibleObjects} />
         </InfiniteScroll>
-        <div
-          className="text-center"
-          style={{ display: isTruncated && currentBucket ? "block" : "none" }}
-        >
-          <span>Loading...</span>
-        </div>
+        {listLoading && <div className="loading" />}
       </div>
     )
   }
@@ -51,16 +71,10 @@ const mapStateToProps = state => {
     currentBucket: state.buckets.currentBucket,
     currentPrefix: state.objects.currentPrefix,
     objects: state.objects.list,
-    isTruncated: state.objects.isTruncated
+    sortBy: state.objects.sortBy,
+    sortOrder: state.objects.sortOrder,
+    listLoading: state.objects.listLoading
   }
 }
 
-const mapDispatchToProps = dispatch => {
-  return {
-    loadObjects: append => dispatch(actionsObjects.fetchObjects(append))
-  }
-}
-
-export default connect(mapStateToProps, mapDispatchToProps)(
-  ObjectsListContainer
-)
+export default connect(mapStateToProps)(ObjectsListContainer)

browser/app/js/objects/__tests__/ObjectsHeader.test.js

@@ -17,6 +17,7 @@
 import React from "react"
 import { shallow } from "enzyme"
 import { ObjectsHeader } from "../ObjectsHeader"
+import { SORT_ORDER_ASC, SORT_ORDER_DESC } from "../../constants"
 
 describe("ObjectsHeader", () => {
   it("should render without crashing", () => {
@@ -24,44 +25,84 @@ describe("ObjectsHeader", () => {
     shallow(<ObjectsHeader sortObjects={sortObjects} />)
   })
 
-  it("should render columns with asc classes by default", () => {
+  it("should render the name column with asc class when objects are sorted by name asc", () => {
     const sortObjects = jest.fn()
-    const wrapper = shallow(<ObjectsHeader sortObjects={sortObjects} />)
+    const wrapper = shallow(
+      <ObjectsHeader
+        sortObjects={sortObjects}
+        sortedByName={true}
+        sortOrder={SORT_ORDER_ASC}
+      />
+    )
     expect(
       wrapper.find("#sort-by-name i").hasClass("fa-sort-alpha-asc")
     ).toBeTruthy()
-    expect(
-      wrapper.find("#sort-by-size i").hasClass("fa-sort-amount-asc")
-    ).toBeTruthy()
-    expect(
-      wrapper.find("#sort-by-last-modified i").hasClass("fa-sort-numeric-asc")
-    ).toBeTruthy()
   })
 
-  it("should render name column with desc class when objects are sorted by name", () => {
+  it("should render the name column with desc class when objects are sorted by name desc", () => {
     const sortObjects = jest.fn()
     const wrapper = shallow(
-      <ObjectsHeader sortObjects={sortObjects} sortNameOrder={true} />
+      <ObjectsHeader
+        sortObjects={sortObjects}
+        sortedByName={true}
+        sortOrder={SORT_ORDER_DESC}
+      />
     )
     expect(
       wrapper.find("#sort-by-name i").hasClass("fa-sort-alpha-desc")
     ).toBeTruthy()
   })
 
-  it("should render size column with desc class when objects are sorted by size", () => {
+  it("should render the size column with asc class when objects are sorted by size asc", () => {
     const sortObjects = jest.fn()
     const wrapper = shallow(
-      <ObjectsHeader sortObjects={sortObjects} sortSizeOrder={true} />
+      <ObjectsHeader
+        sortObjects={sortObjects}
+        sortedBySize={true}
+        sortOrder={SORT_ORDER_ASC}
+      />
+    )
+    expect(
+      wrapper.find("#sort-by-size i").hasClass("fa-sort-amount-asc")
+    ).toBeTruthy()
+  })
+
+  it("should render the size column with desc class when objects are sorted by size desc", () => {
+    const sortObjects = jest.fn()
+    const wrapper = shallow(
+      <ObjectsHeader
+        sortObjects={sortObjects}
+        sortedBySize={true}
+        sortOrder={SORT_ORDER_DESC}
+      />
     )
     expect(
       wrapper.find("#sort-by-size i").hasClass("fa-sort-amount-desc")
     ).toBeTruthy()
   })
 
-  it("should render last modified column with desc class when objects are sorted by last modified", () => {
+  it("should render the date column with asc class when objects are sorted by date asc", () => {
     const sortObjects = jest.fn()
     const wrapper = shallow(
-      <ObjectsHeader sortObjects={sortObjects} sortLastModifiedOrder={true} />
+      <ObjectsHeader
+        sortObjects={sortObjects}
+        sortedByLastModified={true}
+        sortOrder={SORT_ORDER_ASC}
+      />
+    )
+    expect(
+      wrapper.find("#sort-by-last-modified i").hasClass("fa-sort-numeric-asc")
+    ).toBeTruthy()
+  })
+
+  it("should render the date column with desc class when objects are sorted by date desc", () => {
+    const sortObjects = jest.fn()
+    const wrapper = shallow(
+      <ObjectsHeader
+        sortObjects={sortObjects}
+        sortedByLastModified={true}
+        sortOrder={SORT_ORDER_DESC}
+      />
     )
     expect(
       wrapper.find("#sort-by-last-modified i").hasClass("fa-sort-numeric-desc")

browser/app/js/objects/__tests__/ObjectsListContainer.test.js

@@ -20,14 +20,13 @@ import { ObjectsListContainer } from "../ObjectsListContainer"
 
 describe("ObjectsList", () => {
   it("should render without crashing", () => {
-    shallow(<ObjectsListContainer loadObjects={jest.fn()} />)
+    shallow(<ObjectsListContainer objects={[]} />)
   })
 
   it("should render ObjectsList with objects", () => {
     const wrapper = shallow(
       <ObjectsListContainer
         objects={[{ name: "test1.jpg" }, { name: "test2.jpg" }]}
-        loadObjects={jest.fn()}
       />
     )
     expect(wrapper.find("ObjectsList").length).toBe(1)
@@ -37,10 +36,14 @@ describe("ObjectsList", () => {
     ])
   })
 
-  it("should show the loading indicator at the bottom if there are more elements to display", () => {
+  it("should show the loading indicator when the objects are being loaded", () => {
     const wrapper = shallow(
-      <ObjectsListContainer currentBucket="test1" isTruncated={true} />
+      <ObjectsListContainer
+        currentBucket="test1"
+        objects={[]}
+        listLoading={true}
+      />
     )
-    expect(wrapper.find(".text-center").prop("style")).toHaveProperty("display", "block")
+    expect(wrapper.find(".loading").exists()).toBeTruthy()
   })
 })

browser/app/js/objects/__tests__/actions.test.js

@@ -18,7 +18,13 @@ import configureStore from "redux-mock-store"
 import thunk from "redux-thunk"
 import * as actionsObjects from "../actions"
 import * as alertActions from "../../alert/actions"
-import { minioBrowserPrefix } from "../../constants"
+import {
+  minioBrowserPrefix,
+  SORT_BY_NAME,
+  SORT_ORDER_ASC,
+  SORT_BY_LAST_MODIFIED,
+  SORT_ORDER_DESC
+} from "../../constants"
 import history from "../../history"
 
 jest.mock("../../web", () => ({
@@ -37,8 +43,6 @@ jest.mock("../../web", () => ({
     } else {
       return Promise.resolve({
         objects: [{ name: "test1" }, { name: "test2" }],
-        istruncated: false,
-        nextmarker: "test2",
         writable: false
       })
     }
@@ -77,17 +81,11 @@ describe("Objects actions", () => {
     const expectedActions = [
       {
         type: "objects/SET_LIST",
-        objects: [{ name: "test1" }, { name: "test2" }],
-        isTruncated: false,
-        marker: "test2"
+        objects: [{ name: "test1" }, { name: "test2" }]
       }
     ]
     store.dispatch(
-      actionsObjects.setList(
-        [{ name: "test1" }, { name: "test2" }],
-        "test2",
-        false
-      )
+      actionsObjects.setList([{ name: "test1" }, { name: "test2" }])
     )
     const actions = store.getActions()
     expect(actions).toEqual(expectedActions)
@@ -98,10 +96,10 @@ describe("Objects actions", () => {
     const expectedActions = [
       {
         type: "objects/SET_SORT_BY",
-        sortBy: "name"
+        sortBy: SORT_BY_NAME
       }
     ]
-    store.dispatch(actionsObjects.setSortBy("name"))
+    store.dispatch(actionsObjects.setSortBy(SORT_BY_NAME))
     const actions = store.getActions()
     expect(actions).toEqual(expectedActions)
   })
@@ -111,10 +109,10 @@ describe("Objects actions", () => {
     const expectedActions = [
       {
         type: "objects/SET_SORT_ORDER",
-        sortOrder: true
+        sortOrder: SORT_ORDER_ASC
       }
     ]
-    store.dispatch(actionsObjects.setSortOrder(true))
+    store.dispatch(actionsObjects.setSortOrder(SORT_ORDER_ASC))
     const actions = store.getActions()
     expect(actions).toEqual(expectedActions)
   })
@@ -126,23 +124,26 @@ describe("Objects actions", () => {
     })
     const expectedActions = [
       {
-        type: "objects/SET_LIST",
-        objects: [{ name: "test1" }, { name: "test2" }],
-        marker: "test2",
-        isTruncated: false
+        type: "objects/RESET_LIST"
       },
+      { listLoading: true, type: "objects/SET_LIST_LOADING" },
       {
         type: "objects/SET_SORT_BY",
-        sortBy: ""
+        sortBy: SORT_BY_LAST_MODIFIED
       },
       {
         type: "objects/SET_SORT_ORDER",
-        sortOrder: false
+        sortOrder: SORT_ORDER_DESC
+      },
+      {
+        type: "objects/SET_LIST",
+        objects: [{ name: "test2" }, { name: "test1" }]
       },
       {
         type: "objects/SET_PREFIX_WRITABLE",
         prefixWritable: false
-      }
+      },
+      { listLoading: false, type: "objects/SET_LIST_LOADING" }
     ]
     return store.dispatch(actionsObjects.fetchObjects()).then(() => {
       const actions = store.getActions()
@@ -150,35 +151,16 @@ describe("Objects actions", () => {
     })
   })
 
-  it("creates objects/APPEND_LIST after fetching more objects", () => {
-    const store = mockStore({
-      buckets: { currentBucket: "bk1" },
-      objects: { currentPrefix: "" }
-    })
-    const expectedActions = [
-      {
-        type: "objects/APPEND_LIST",
-        objects: [{ name: "test1" }, { name: "test2" }],
-        marker: "test2",
-        isTruncated: false
-      },
-      {
-        type: "objects/SET_PREFIX_WRITABLE",
-        prefixWritable: false
-      }
-    ]
-    return store.dispatch(actionsObjects.fetchObjects(true)).then(() => {
-      const actions = store.getActions()
-      expect(actions).toEqual(expectedActions)
-    })
-  })
-
   it("creates objects/RESET_LIST after failing to fetch the objects from bucket with ListObjects denied for LoggedIn users", () => {
     const store = mockStore({
       buckets: { currentBucket: "test-deny" },
       objects: { currentPrefix: "" }
     })
     const expectedActions = [
+      {
+        type: "objects/RESET_LIST"
+      },
+      { listLoading: true, type: "objects/SET_LIST_LOADING" },
       {
         type: "alert/SET",
         alert: {
@@ -189,8 +171,9 @@ describe("Objects actions", () => {
         }
       },
       {
-        type: "object/RESET_LIST"
-      }
+        type: "objects/RESET_LIST"
+      },
+      { listLoading: false, type: "objects/SET_LIST_LOADING" }
     ]
     return store.dispatch(actionsObjects.fetchObjects()).then(() => {
       const actions = store.getActions()
@@ -213,28 +196,24 @@ describe("Objects actions", () => {
       objects: {
         list: [],
         sortBy: "",
-        sortOrder: false,
-        isTruncated: false,
-        marker: ""
+        sortOrder: SORT_ORDER_ASC
       }
     })
     const expectedActions = [
       {
         type: "objects/SET_SORT_BY",
-        sortBy: "name"
+        sortBy: SORT_BY_NAME
       },
       {
         type: "objects/SET_SORT_ORDER",
-        sortOrder: true
+        sortOrder: SORT_ORDER_ASC
       },
       {
         type: "objects/SET_LIST",
-        objects: [],
-        isTruncated: false,
-        marker: ""
+        objects: []
       }
     ]
-    store.dispatch(actionsObjects.sortObjects("name"))
+    store.dispatch(actionsObjects.sortObjects(SORT_BY_NAME))
     const actions = store.getActions()
     expect(actions).toEqual(expectedActions)
   })
@@ -246,6 +225,10 @@ describe("Objects actions", () => {
     })
     const expectedActions = [
       { type: "objects/SET_CURRENT_PREFIX", prefix: "abc/" },
+      {
+        type: "objects/RESET_LIST"
+      },
+      { listLoading: true, type: "objects/SET_LIST_LOADING" },
       { type: "objects/CHECKED_LIST_RESET" }
     ]
     store.dispatch(actionsObjects.selectPrefix("abc/"))

browser/app/js/objects/__tests__/reducer.test.js

@@ -16,17 +16,17 @@
 
 import reducer from "../reducer"
 import * as actions from "../actions"
+import { SORT_ORDER_ASC, SORT_BY_NAME } from "../../constants"
 
 describe("objects reducer", () => {
   it("should return the initial state", () => {
     const initialState = reducer(undefined, {})
     expect(initialState).toEqual({
       list: [],
+      listLoading: false,
       sortBy: "",
-      sortOrder: false,
+      sortOrder: SORT_ORDER_ASC,
       currentPrefix: "",
-      marker: "",
-      isTruncated: false,
       prefixWritable: false,
       shareObject: {
         show: false,
@@ -40,37 +40,9 @@ describe("objects reducer", () => {
   it("should handle SET_LIST", () => {
     const newState = reducer(undefined, {
       type: actions.SET_LIST,
-      objects: [{ name: "obj1" }, { name: "obj2" }],
-      marker: "obj2",
-      isTruncated: false
+      objects: [{ name: "obj1" }, { name: "obj2" }]
     })
     expect(newState.list).toEqual([{ name: "obj1" }, { name: "obj2" }])
-    expect(newState.marker).toBe("obj2")
-    expect(newState.isTruncated).toBeFalsy()
-  })
-
-  it("should handle APPEND_LIST", () => {
-    const newState = reducer(
-      {
-        list: [{ name: "obj1" }, { name: "obj2" }],
-        marker: "obj2",
-        isTruncated: true
-      },
-      {
-        type: actions.APPEND_LIST,
-        objects: [{ name: "obj3" }, { name: "obj4" }],
-        marker: "obj4",
-        isTruncated: false
-      }
-    )
-    expect(newState.list).toEqual([
-      { name: "obj1" },
-      { name: "obj2" },
-      { name: "obj3" },
-      { name: "obj4" }
-    ])
-    expect(newState.marker).toBe("obj4")
-    expect(newState.isTruncated).toBeFalsy()
   })
 
   it("should handle REMOVE", () => {
@@ -98,30 +70,28 @@ describe("objects reducer", () => {
   it("should handle SET_SORT_BY", () => {
     const newState = reducer(undefined, {
       type: actions.SET_SORT_BY,
-      sortBy: "name"
+      sortBy: SORT_BY_NAME
     })
-    expect(newState.sortBy).toEqual("name")
+    expect(newState.sortBy).toEqual(SORT_BY_NAME)
   })
 
   it("should handle SET_SORT_ORDER", () => {
     const newState = reducer(undefined, {
       type: actions.SET_SORT_ORDER,
-      sortOrder: true
+      sortOrder: SORT_ORDER_ASC
     })
-    expect(newState.sortOrder).toEqual(true)
+    expect(newState.sortOrder).toEqual(SORT_ORDER_ASC)
   })
 
   it("should handle SET_CURRENT_PREFIX", () => {
     const newState = reducer(
-      { currentPrefix: "test1/", marker: "abc", isTruncated: true },
+      { currentPrefix: "test1/" },
       {
         type: actions.SET_CURRENT_PREFIX,
         prefix: "test2/"
       }
     )
     expect(newState.currentPrefix).toEqual("test2/")
-    expect(newState.marker).toEqual("")
-    expect(newState.isTruncated).toBeFalsy()
   })
 
   it("should handle SET_PREFIX_WRITABLE", () => {

browser/app/js/objects/actions.js

@@ -16,15 +16,26 @@
 
 import web from "../web"
 import history from "../history"
-import { sortObjectsByName, sortObjectsBySize, sortObjectsByDate } from "../utils"
+import {
+  sortObjectsByName,
+  sortObjectsBySize,
+  sortObjectsByDate
+} from "../utils"
 import { getCurrentBucket } from "../buckets/selectors"
 import { getCurrentPrefix, getCheckedList } from "./selectors"
 import * as alertActions from "../alert/actions"
 import * as bucketActions from "../buckets/actions"
-import { minioBrowserPrefix } from "../constants"
+import {
+  minioBrowserPrefix,
+  SORT_BY_NAME,
+  SORT_BY_SIZE,
+  SORT_BY_LAST_MODIFIED,
+  SORT_ORDER_ASC,
+  SORT_ORDER_DESC
+} from "../constants"
 
 export const SET_LIST = "objects/SET_LIST"
-export const RESET_LIST = "object/RESET_LIST"
+export const RESET_LIST = "objects/RESET_LIST"
 export const APPEND_LIST = "objects/APPEND_LIST"
 export const REMOVE = "objects/REMOVE"
 export const SET_SORT_BY = "objects/SET_SORT_BY"
@@ -35,53 +46,63 @@ export const SET_SHARE_OBJECT = "objects/SET_SHARE_OBJECT"
 export const CHECKED_LIST_ADD = "objects/CHECKED_LIST_ADD"
 export const CHECKED_LIST_REMOVE = "objects/CHECKED_LIST_REMOVE"
 export const CHECKED_LIST_RESET = "objects/CHECKED_LIST_RESET"
+export const SET_LIST_LOADING = "objects/SET_LIST_LOADING"
 
-export const setList = (objects, marker, isTruncated) => ({
+export const setList = objects => ({
   type: SET_LIST,
-  objects,
-  marker,
-  isTruncated
+  objects
 })
 
 export const resetList = () => ({
   type: RESET_LIST
 })
 
-export const appendList = (objects, marker, isTruncated) => ({
-  type: APPEND_LIST,
-  objects,
-  marker,
-  isTruncated
+export const setListLoading = listLoading => ({
+  type: SET_LIST_LOADING,
+  listLoading
 })
 
-export const fetchObjects = append => {
+export const fetchObjects = () => {
   return function(dispatch, getState) {
-    const {buckets: {currentBucket}, objects: {currentPrefix, marker}} = getState()
+    dispatch(resetList())
+    const {
+      buckets: { currentBucket },
+      objects: { currentPrefix }
+    } = getState()
     if (currentBucket) {
+      dispatch(setListLoading(true))
       return web
         .ListObjects({
           bucketName: currentBucket,
-          prefix: currentPrefix,
-          marker: append ? marker : ""
+          prefix: currentPrefix
         })
         .then(res => {
-          let objects = []
-          if (res.objects) {
-            objects = res.objects.map(object => {
-              return {
-                ...object,
-                name: object.name.replace(currentPrefix, "")
-              }
-            })
+          // we need to check if the bucket name and prefix are the same as
+          // when the request was made before updating the displayed objects
+          if (
+            currentBucket === getCurrentBucket(getState()) &&
+            currentPrefix === getCurrentPrefix(getState())
+          ) {
+            let objects = []
+            if (res.objects) {
+              objects = res.objects.map(object => {
+                return {
+                  ...object,
+                  name: object.name.replace(currentPrefix, "")
+                }
+              })
+            }
+
+            const sortBy = SORT_BY_LAST_MODIFIED
+            const sortOrder = SORT_ORDER_DESC
+            dispatch(setSortBy(sortBy))
+            dispatch(setSortOrder(sortOrder))
+            const sortedList = sortObjectsList(objects, sortBy, sortOrder)
+            dispatch(setList(sortedList))
+
+            dispatch(setPrefixWritable(res.writable))
+            dispatch(setListLoading(false))
           }
-          if (append) {
-            dispatch(appendList(objects, res.nextmarker, res.istruncated))
-          } else {
-            dispatch(setList(objects, res.nextmarker, res.istruncated))
-            dispatch(setSortBy(""))
-            dispatch(setSortOrder(false))
-          }
-          dispatch(setPrefixWritable(res.writable))
         })
         .catch(err => {
           if (web.LoggedIn()) {
@@ -96,6 +117,7 @@ export const fetchObjects = append => {
           } else {
             history.push("/login")
           }
+          dispatch(setListLoading(false))
         })
     }
   }
@@ -103,26 +125,27 @@ export const fetchObjects = append => {
 
 export const sortObjects = sortBy => {
   return function(dispatch, getState) {
-    const {objects} = getState()
-    const sortOrder = objects.sortBy == sortBy ? !objects.sortOrder : true
+    const { objects } = getState()
+    let sortOrder = SORT_ORDER_ASC
+    // Reverse sort order if the list is already sorted on same field
+    if (objects.sortBy === sortBy && objects.sortOrder === SORT_ORDER_ASC) {
+      sortOrder = SORT_ORDER_DESC
+    }
     dispatch(setSortBy(sortBy))
     dispatch(setSortOrder(sortOrder))
-    let list
-    switch (sortBy) {
-      case "name":
-        list = sortObjectsByName(objects.list, sortOrder)
-        break
-      case "size":
-        list = sortObjectsBySize(objects.list, sortOrder)
-        break
-      case "last-modified":
-        list = sortObjectsByDate(objects.list, sortOrder)
-        break
-      default:
-        list = objects.list
-        break
-    }
-    dispatch(setList(list, objects.marker, objects.isTruncated))
+    const sortedList = sortObjectsList(objects.list, sortBy, sortOrder)
+    dispatch(setList(sortedList))
+  }
+}
+
+const sortObjectsList = (list, sortBy, sortOrder) => {
+  switch (sortBy) {
+    case SORT_BY_NAME:
+      return sortObjectsByName(list, sortOrder)
+    case SORT_BY_SIZE:
+      return sortObjectsBySize(list, sortOrder)
+    case SORT_BY_LAST_MODIFIED:
+      return sortObjectsByDate(list, sortOrder)
   }
 }
 
@@ -229,7 +252,16 @@ export const shareObject = (object, days, hours, minutes) => {
           )
         })
     } else {
-      dispatch(showShareObject(object, `${location.host}` + '/' + `${currentBucket}` + '/' + encodeURI(objectName)))
+      dispatch(
+        showShareObject(
+          object,
+          `${location.host}` +
+            "/" +
+            `${currentBucket}` +
+            "/" +
+            encodeURI(objectName)
+        )
+      )
       dispatch(
         alertActions.set({
           type: "success",
@@ -322,13 +354,14 @@ export const downloadCheckedObjects = () => {
           }${minioBrowserPrefix}/zip?token=${res.token}`
           downloadZip(requestUrl, req, dispatch)
         })
-        .catch(err => dispatch(
-          alertActions.set({
-            type: "danger",
-            message: err.message
-          })
+        .catch(err =>
+          dispatch(
+            alertActions.set({
+              type: "danger",
+              message: err.message
+            })
+          )
         )
-      )
     }
   }
 }
@@ -351,7 +384,8 @@ const downloadZip = (url, req, dispatch) => {
       var separator = req.prefix.length > 1 ? "-" : ""
 
       anchor.href = blobUrl
-      anchor.download = req.bucketName + separator + req.prefix.slice(0, -1) + ".zip"
+      anchor.download =
+        req.bucketName + separator + req.prefix.slice(0, -1) + ".zip"
 
       anchor.click()
       window.URL.revokeObjectURL(blobUrl)

browser/app/js/objects/reducer.js

@@ -15,6 +15,7 @@
  */
 
 import * as actionsObjects from "./actions"
+import { SORT_ORDER_ASC } from "../constants"
 
 const removeObject = (list, objectToRemove, lookup) => {
   const idx = list.findIndex(object => lookup(object) === objectToRemove)
@@ -27,11 +28,10 @@ const removeObject = (list, objectToRemove, lookup) => {
 export default (
   state = {
     list: [],
+    listLoading: false,
     sortBy: "",
-    sortOrder: false,
+    sortOrder: SORT_ORDER_ASC,
     currentPrefix: "",
-    marker: "",
-    isTruncated: false,
     prefixWritable: false,
     shareObject: {
       show: false,
@@ -46,23 +46,17 @@ export default (
     case actionsObjects.SET_LIST:
       return {
         ...state,
-        list: action.objects,
-        marker: action.marker,
-        isTruncated: action.isTruncated
+        list: action.objects
       }
     case actionsObjects.RESET_LIST:
       return {
         ...state,
-        list: [],
-        marker: "",
-        isTruncated: false
+        list: []
       }
-    case actionsObjects.APPEND_LIST:
+    case actionsObjects.SET_LIST_LOADING:
       return {
         ...state,
-        list: [...state.list, ...action.objects],
-        marker: action.marker,
-        isTruncated: action.isTruncated
+        listLoading: action.listLoading
       }
     case actionsObjects.REMOVE:
       return {
@@ -82,9 +76,7 @@ export default (
     case actionsObjects.SET_CURRENT_PREFIX:
       return {
         ...state,
-        currentPrefix: action.prefix,
-        marker: "",
-        isTruncated: false
+        currentPrefix: action.prefix
       }
     case actionsObjects.SET_PREFIX_WRITABLE:
       return {

browser/app/js/utils.js

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { minioBrowserPrefix } from "./constants.js"
+import { minioBrowserPrefix, SORT_ORDER_DESC } from "./constants.js"
 
 export const sortObjectsByName = (objects, order) => {
   let folders = objects.filter(object => object.name.endsWith("/"))
@@ -29,7 +29,7 @@ export const sortObjectsByName = (objects, order) => {
     if (a.name.toLowerCase() > b.name.toLowerCase()) return 1
     return 0
   })
-  if (order) {
+  if (order === SORT_ORDER_DESC) {
     folders = folders.reverse()
     files = files.reverse()
   }
@@ -40,7 +40,7 @@ export const sortObjectsBySize = (objects, order) => {
   let folders = objects.filter(object => object.name.endsWith("/"))
   let files = objects.filter(object => !object.name.endsWith("/"))
   files = files.sort((a, b) => a.size - b.size)
-  if (order) files = files.reverse()
+  if (order === SORT_ORDER_DESC) files = files.reverse()
   return [...folders, ...files]
 }
 
@@ -51,7 +51,7 @@ export const sortObjectsByDate = (objects, order) => {
     (a, b) =>
       new Date(a.lastModified).getTime() - new Date(b.lastModified).getTime()
   )
-  if (order) files = files.reverse()
+  if (order === SORT_ORDER_DESC) files = files.reverse()
   return [...folders, ...files]
 }
 

browser/app/less/inc/list.less

@@ -43,6 +43,9 @@ header.fesl-row {
                 color: @dark-gray;
                 font-size: 14px;
             }
+            & > .fesli-sort--active {
+                .opacity(0.5);
+            }
 
             &:hover:not(.fesl-item-actions) {
                 background: lighten(@text-muted-color, 22%);

browser/app/less/inc/misc.less

@@ -113,4 +113,41 @@
 		margin: 0;
 		vertical-align: top;
 	}
+}
+
+.loading {
+  position: absolute;
+  margin: auto;
+  left: 0;
+  right: 0;
+  top: 0;
+  bottom: 0;
+  border-top: 1px solid @loading-track-bg;
+  border-right: 1px solid @loading-track-bg;
+  border-bottom: 1px solid @loading-track-bg;
+  border-left: 1px solid @loading-point-bg;
+  transform: translateZ(0);
+  animation: loading 1.1s infinite linear;
+  border-radius: 50%;
+	width: 35px;
+	height: 35px;
+	margin-top: 30px;
+}
+
+@-webkit-keyframes loading {
+  0% {
+    transform: rotate(0deg);
+  }
+  100% {
+    transform: rotate(360deg);
+  }
+}
+
+@keyframes loading {
+  0% {
+    transform: rotate(0deg);
+  }
+  100% {
+    transform: rotate(360deg);
+  }
 }

browser/app/less/inc/variables.less

@@ -100,4 +100,10 @@
     List
 --------------------------*/
 @list-row-selected-bg: #fbf2bf;
-@list-row-even-bg:          #fafafa;
+@list-row-even-bg:          #fafafa;
+
+/*--------------------------
+    Loading
+---------------------------*/
+@loading-track-bg: #eeeeee;
+@loading-point-bg: #00303f;

browser/package.json

@@ -29,8 +29,8 @@
   "homepage": "https://github.com/minio/minio",
   "devDependencies": {
     "async": "^1.5.2",
-    "babel-cli": "^6.14.0",
-    "babel-core": "^6.14.0",
+    "babel-cli": "^6.26.0",
+    "babel-core": "^6.26.3",
     "babel-jest": "^22.1.0",
     "babel-loader": "^7.1.2",
     "babel-plugin-syntax-object-rest-spread": "^6.13.0",
@@ -38,10 +38,10 @@
     "babel-polyfill": "^6.23.0",
     "babel-preset-es2015": "^6.14.0",
     "babel-preset-react": "^6.11.1",
-    "babel-register": "^6.14.0",
+    "babel-register": "^6.26.0",
     "copy-webpack-plugin": "^4.6.0",
     "css-loader": "^0.23.1",
-    "enzyme": "^3.3.0",
+    "enzyme": "^3.10.0",
     "enzyme-adapter-react-16": "^1.1.1",
     "esformatter": "^0.10.0",
     "esformatter-jsx": "^7.4.1",
@@ -73,7 +73,7 @@
     "material-design-iconic-font": "^2.2.0",
     "mime-db": "^1.25.0",
     "mime-types": "^2.1.13",
-    "moment": "^2.15.1",
+    "moment": "^2.24.0",
     "react": "^16.2.0",
     "react-addons-test-utils": "^0.14.8",
     "react-bootstrap": "^0.32.0",

browser/yarn.lock

@@ -366,6 +366,11 @@ array-equal@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/array-equal/-/array-equal-1.0.0.tgz#8c2a5ef2472fd9ea742b04c77a75093ba2757c93"
 
+array-filter@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/array-filter/-/array-filter-1.0.0.tgz#baf79e62e6ef4c2a4c0b831232daffec251f9d83"
+  integrity sha1-uveeYubvTCpMC4MSMtr/7CUfnYM=
+
 array-flatten@1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2"
@@ -398,6 +403,15 @@ array-unique@^0.3.2:
   resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428"
   integrity sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=
 
+array.prototype.flat@^1.2.1:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/array.prototype.flat/-/array.prototype.flat-1.2.1.tgz#812db8f02cad24d3fab65dd67eabe3b8903494a4"
+  integrity sha512-rVqIs330nLJvfC7JqYvEWwqVr5QjYF1ib02i3YJtR/fICO6527Tjpc/e4Mvmxh3GIePPreRXMdaGyC99YphWEw==
+  dependencies:
+    define-properties "^1.1.2"
+    es-abstract "^1.10.0"
+    function-bind "^1.1.1"
+
 arrify@^1.0.0, arrify@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d"
@@ -508,24 +522,25 @@ aws4@^1.6.0:
   version "1.6.0"
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e"
 
-babel-cli@^6.14.0:
-  version "6.24.1"
-  resolved "https://registry.yarnpkg.com/babel-cli/-/babel-cli-6.24.1.tgz#207cd705bba61489b2ea41b5312341cf6aca2283"
+babel-cli@^6.26.0:
+  version "6.26.0"
+  resolved "https://registry.yarnpkg.com/babel-cli/-/babel-cli-6.26.0.tgz#502ab54874d7db88ad00b887a06383ce03d002f1"
+  integrity sha1-UCq1SHTX24itALiHoGODzgPQAvE=
   dependencies:
-    babel-core "^6.24.1"
-    babel-polyfill "^6.23.0"
-    babel-register "^6.24.1"
-    babel-runtime "^6.22.0"
-    commander "^2.8.1"
-    convert-source-map "^1.1.0"
+    babel-core "^6.26.0"
+    babel-polyfill "^6.26.0"
+    babel-register "^6.26.0"
+    babel-runtime "^6.26.0"
+    commander "^2.11.0"
+    convert-source-map "^1.5.0"
     fs-readdir-recursive "^1.0.0"
-    glob "^7.0.0"
-    lodash "^4.2.0"
-    output-file-sync "^1.1.0"
-    path-is-absolute "^1.0.0"
+    glob "^7.1.2"
+    lodash "^4.17.4"
+    output-file-sync "^1.1.2"
+    path-is-absolute "^1.0.1"
     slash "^1.0.0"
-    source-map "^0.5.0"
-    v8flags "^2.0.10"
+    source-map "^0.5.6"
+    v8flags "^2.1.1"
   optionalDependencies:
     chokidar "^1.6.1"
 
@@ -569,29 +584,30 @@ babel-core@^6.0.0, babel-core@^6.26.0:
     slash "^1.0.0"
     source-map "^0.5.6"
 
-babel-core@^6.14.0, babel-core@^6.24.1:
-  version "6.25.0"
-  resolved "https://registry.yarnpkg.com/babel-core/-/babel-core-6.25.0.tgz#7dd42b0463c742e9d5296deb3ec67a9322dad729"
+babel-core@^6.26.3:
+  version "6.26.3"
+  resolved "https://registry.yarnpkg.com/babel-core/-/babel-core-6.26.3.tgz#b2e2f09e342d0f0c88e2f02e067794125e75c207"
+  integrity sha512-6jyFLuDmeidKmUEb3NM+/yawG0M2bDZ9Z1qbZP59cyHLz8kYGKYwpJP0UwUKKUiTRNvxfLesJnTedqczP7cTDA==
   dependencies:
-    babel-code-frame "^6.22.0"
-    babel-generator "^6.25.0"
+    babel-code-frame "^6.26.0"
+    babel-generator "^6.26.0"
     babel-helpers "^6.24.1"
     babel-messages "^6.23.0"
-    babel-register "^6.24.1"
-    babel-runtime "^6.22.0"
-    babel-template "^6.25.0"
-    babel-traverse "^6.25.0"
-    babel-types "^6.25.0"
-    babylon "^6.17.2"
-    convert-source-map "^1.1.0"
-    debug "^2.1.1"
-    json5 "^0.5.0"
-    lodash "^4.2.0"
-    minimatch "^3.0.2"
-    path-is-absolute "^1.0.0"
-    private "^0.1.6"
+    babel-register "^6.26.0"
+    babel-runtime "^6.26.0"
+    babel-template "^6.26.0"
+    babel-traverse "^6.26.0"
+    babel-types "^6.26.0"
+    babylon "^6.18.0"
+    convert-source-map "^1.5.1"
+    debug "^2.6.9"
+    json5 "^0.5.1"
+    lodash "^4.17.4"
+    minimatch "^3.0.4"
+    path-is-absolute "^1.0.1"
+    private "^0.1.8"
     slash "^1.0.0"
-    source-map "^0.5.0"
+    source-map "^0.5.7"
 
 babel-generator@^6.18.0, babel-generator@^6.26.0:
   version "6.26.0"
@@ -606,19 +622,6 @@ babel-generator@^6.18.0, babel-generator@^6.26.0:
     source-map "^0.5.6"
     trim-right "^1.0.1"
 
-babel-generator@^6.25.0:
-  version "6.25.0"
-  resolved "https://registry.yarnpkg.com/babel-generator/-/babel-generator-6.25.0.tgz#33a1af70d5f2890aeb465a4a7793c1df6a9ea9fc"
-  dependencies:
-    babel-messages "^6.23.0"
-    babel-runtime "^6.22.0"
-    babel-types "^6.25.0"
-    detect-indent "^4.0.0"
-    jsesc "^1.3.0"
-    lodash "^4.2.0"
-    source-map "^0.5.0"
-    trim-right "^1.0.1"
-
 babel-helper-builder-react-jsx@^6.24.1:
   version "6.24.1"
   resolved "https://registry.yarnpkg.com/babel-helper-builder-react-jsx/-/babel-helper-builder-react-jsx-6.24.1.tgz#0ad7917e33c8d751e646daca4e77cc19377d2cbc"
@@ -985,6 +988,15 @@ babel-polyfill@6.23.0, babel-polyfill@^6.23.0:
     core-js "^2.4.0"
     regenerator-runtime "^0.10.0"
 
+babel-polyfill@^6.26.0:
+  version "6.26.0"
+  resolved "https://registry.yarnpkg.com/babel-polyfill/-/babel-polyfill-6.26.0.tgz#379937abc67d7895970adc621f284cd966cf2153"
+  integrity sha1-N5k3q8Z9eJWXCtxiHyhM2WbPIVM=
+  dependencies:
+    babel-runtime "^6.26.0"
+    core-js "^2.5.0"
+    regenerator-runtime "^0.10.5"
+
 babel-preset-es2015@^6.14.0:
   version "6.24.1"
   resolved "https://registry.yarnpkg.com/babel-preset-es2015/-/babel-preset-es2015-6.24.1.tgz#d44050d6bc2c9feea702aaf38d727a0210538939"
@@ -1038,18 +1050,6 @@ babel-preset-react@^6.11.1:
     babel-plugin-transform-react-jsx-source "^6.22.0"
     babel-preset-flow "^6.23.0"
 
-babel-register@^6.14.0, babel-register@^6.24.1:
-  version "6.24.1"
-  resolved "https://registry.yarnpkg.com/babel-register/-/babel-register-6.24.1.tgz#7e10e13a2f71065bdfad5a1787ba45bca6ded75f"
-  dependencies:
-    babel-core "^6.24.1"
-    babel-runtime "^6.22.0"
-    core-js "^2.4.0"
-    home-or-tmp "^2.0.0"
-    lodash "^4.2.0"
-    mkdirp "^0.5.1"
-    source-map-support "^0.4.2"
-
 babel-register@^6.26.0:
   version "6.26.0"
   resolved "https://registry.yarnpkg.com/babel-register/-/babel-register-6.26.0.tgz#6ed021173e2fcb486d7acb45c6009a856f647071"
@@ -1086,7 +1086,7 @@ babel-template@^6.16.0, babel-template@^6.26.0:
     babylon "^6.18.0"
     lodash "^4.17.4"
 
-babel-template@^6.24.1, babel-template@^6.25.0:
+babel-template@^6.24.1:
   version "6.25.0"
   resolved "https://registry.yarnpkg.com/babel-template/-/babel-template-6.25.0.tgz#665241166b7c2aa4c619d71e192969552b10c071"
   dependencies:
@@ -1771,17 +1771,17 @@ commander@2.17.x, commander@~2.17.1:
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.17.1.tgz#bd77ab7de6de94205ceacc72f1716d29f20a77bf"
   integrity sha512-wPMUt6FnH2yzG95SA6mzjQOEKUU3aLaDEmzs1ti+1E9h+CsrZghRlqEM/EJ4KscsQVG8uNN4uVreUeT8+drlgg==
 
-commander@^2.2.0, commander@^2.8.1, commander@^2.9.0:
+commander@^2.11.0, commander@~2.20.0:
+  version "2.20.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.0.tgz#d58bb2b5c1ee8f87b0d340027e9e94e222c5a422"
+  integrity sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==
+
+commander@^2.2.0, commander@^2.9.0:
   version "2.10.0"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.10.0.tgz#e1f5d3245de246d1a5ca04702fa1ad1bd7e405fe"
   dependencies:
     graceful-readlink ">= 1.0.0"
 
-commander@~2.20.0:
-  version "2.20.0"
-  resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.0.tgz#d58bb2b5c1ee8f87b0d340027e9e94e222c5a422"
-  integrity sha512-7j2y+40w61zy6YC2iRNpUe/NwhNyoXrYpHMrSunaMG64nRnaf96zO/KMQR4OyN/UnE5KLyEBnKHd4aG3rskjpQ==
-
 commondir@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b"
@@ -1867,14 +1867,17 @@ content-type@~1.0.4:
   resolved "https://registry.yarnpkg.com/content-type/-/content-type-1.0.4.tgz#e138cc75e040c727b1966fe5e5f8c9aee256fe3b"
   integrity sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==
 
-convert-source-map@^1.1.0:
-  version "1.5.0"
-  resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.5.0.tgz#9acd70851c6d5dfdd93d9282e5edf94a03ff46b5"
-
 convert-source-map@^1.4.0, convert-source-map@^1.5.0:
   version "1.5.1"
   resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.5.1.tgz#b8278097b9bc229365de5c62cf5fcaed8b5599e5"
 
+convert-source-map@^1.5.1:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-1.6.0.tgz#51b537a8c43e0f04dec1993bffcdd504e758ac20"
+  integrity sha512-eFu7XigvxdZ1ETfbgPBohgyQ/Z++C0eEhTor0qRwBw9unw+L0/6V8wkSuGgzdThkiS5lSpdptOQPD8Ak40a+7A==
+  dependencies:
+    safe-buffer "~5.1.1"
+
 cookie-signature@1.0.6:
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c"
@@ -2144,7 +2147,7 @@ date-now@^0.1.4:
   version "0.1.4"
   resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b"
 
-debug@2.6.9, debug@^2.1.2, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8:
+debug@2.6.9, debug@^2.1.2, debug@^2.2.0, debug@^2.3.3, debug@^2.6.8, debug@^2.6.9:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
   dependencies:
@@ -2161,7 +2164,7 @@ debug@^0.7.4:
   version "0.7.4"
   resolved "https://registry.yarnpkg.com/debug/-/debug-0.7.4.tgz#06e1ea8082c2cb14e39806e22e2f6f757f92af39"
 
-debug@^2.1.1, debug@^2.1.3:
+debug@^2.1.3:
   version "2.6.8"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.8.tgz#e731531ca2ede27d188222427da17821d68ff4fc"
   dependencies:
@@ -2601,26 +2604,32 @@ enzyme-to-json@^3.3.0:
   dependencies:
     lodash "^4.17.4"
 
-enzyme@^3.3.0:
-  version "3.3.0"
-  resolved "https://registry.yarnpkg.com/enzyme/-/enzyme-3.3.0.tgz#0971abd167f2d4bf3f5bd508229e1c4b6dc50479"
+enzyme@^3.10.0:
+  version "3.10.0"
+  resolved "https://registry.yarnpkg.com/enzyme/-/enzyme-3.10.0.tgz#7218e347c4a7746e133f8e964aada4a3523452f6"
+  integrity sha512-p2yy9Y7t/PFbPoTvrWde7JIYB2ZyGC+NgTNbVEGvZ5/EyoYSr9aG/2rSbVvyNvMHEhw9/dmGUJHWtfQIEiX9pg==
   dependencies:
+    array.prototype.flat "^1.2.1"
     cheerio "^1.0.0-rc.2"
-    function.prototype.name "^1.0.3"
-    has "^1.0.1"
+    function.prototype.name "^1.1.0"
+    has "^1.0.3"
+    html-element-map "^1.0.0"
     is-boolean-object "^1.0.0"
-    is-callable "^1.1.3"
+    is-callable "^1.1.4"
     is-number-object "^1.0.3"
+    is-regex "^1.0.4"
     is-string "^1.0.4"
     is-subset "^0.1.1"
-    lodash "^4.17.4"
-    object-inspect "^1.5.0"
+    lodash.escape "^4.0.1"
+    lodash.isequal "^4.5.0"
+    object-inspect "^1.6.0"
     object-is "^1.0.1"
     object.assign "^4.1.0"
     object.entries "^1.0.4"
     object.values "^1.0.4"
     raf "^3.4.0"
     rst-selector-parser "^2.2.3"
+    string.prototype.trim "^1.1.2"
 
 errno@^0.1.1, errno@^0.1.3, errno@~0.1.7:
   version "0.1.7"
@@ -2636,6 +2645,18 @@ error-ex@^1.2.0:
   dependencies:
     is-arrayish "^0.2.1"
 
+es-abstract@^1.10.0, es-abstract@^1.5.0:
+  version "1.13.0"
+  resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.13.0.tgz#ac86145fdd5099d8dd49558ccba2eaf9b88e24e9"
+  integrity sha512-vDZfg/ykNxQVwup/8E1BZhVzFfBxs9NqMzGcvIJrqg5k2/5Za2bWo40dK2J1pgLngZ7c+Shh8lwYtLGyrwPutg==
+  dependencies:
+    es-to-primitive "^1.2.0"
+    function-bind "^1.1.1"
+    has "^1.0.3"
+    is-callable "^1.1.4"
+    is-regex "^1.0.4"
+    object-keys "^1.0.12"
+
 es-abstract@^1.5.1:
   version "1.10.0"
   resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.10.0.tgz#1ecb36c197842a00d8ee4c2dfd8646bb97d60864"
@@ -2655,7 +2676,7 @@ es-abstract@^1.6.1:
     is-callable "^1.1.3"
     is-regex "^1.0.3"
 
-es-to-primitive@^1.1.1:
+es-to-primitive@^1.1.1, es-to-primitive@^1.2.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.0.tgz#edf72478033456e8dda8ef09e00ad9650707f377"
   integrity sha512-qZryBOJjV//LaxLTV6UC//WewneB3LcXOL9NP++ozKVXsIIIpm/2c13UDiD9Jp2eThsecw9m3jPqDwTyobcdbg==
@@ -3347,18 +3368,19 @@ fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2:
     mkdirp ">=0.5 0"
     rimraf "2"
 
-function-bind@^1.1.0:
-  version "1.1.0"
-  resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.0.tgz#16176714c801798e4e8f2cf7f7529467bb4a5771"
-
-function-bind@^1.1.1:
+function-bind@^1.0.2, function-bind@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d"
   integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==
 
-function.prototype.name@^1.0.3:
+function-bind@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.0.tgz#16176714c801798e4e8f2cf7f7529467bb4a5771"
+
+function.prototype.name@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.0.tgz#8bd763cc0af860a859cc5d49384d74b932cd2327"
+  integrity sha512-Bs0VRrTz4ghD8pTmbJQD1mZ8A/mN0ur/jGz+A6FBxPDUPkm1tNfF6bhTYPA7i7aF4lZJVr+OXTNNrnnIl58Wfg==
   dependencies:
     define-properties "^1.1.2"
     function-bind "^1.1.1"
@@ -3445,9 +3467,10 @@ glob-parent@^3.1.0:
     is-glob "^3.1.0"
     path-dirname "^1.0.0"
 
-glob@^7.0.0, glob@^7.1.1, glob@^7.1.2:
-  version "7.1.2"
-  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
+glob@^7.0.3, glob@^7.0.5:
+  version "7.1.3"
+  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.3.tgz#3960832d3f1574108342dafd3a67b332c0969df1"
+  integrity sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==
   dependencies:
     fs.realpath "^1.0.0"
     inflight "^1.0.4"
@@ -3456,10 +3479,9 @@ glob@^7.0.0, glob@^7.1.1, glob@^7.1.2:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
-glob@^7.0.3, glob@^7.0.5:
-  version "7.1.3"
-  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.3.tgz#3960832d3f1574108342dafd3a67b332c0969df1"
-  integrity sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==
+glob@^7.1.1, glob@^7.1.2:
+  version "7.1.2"
+  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
   dependencies:
     fs.realpath "^1.0.0"
     inflight "^1.0.4"
@@ -3650,7 +3672,7 @@ has-values@^1.0.0:
     is-number "^3.0.0"
     kind-of "^4.0.0"
 
-has@^1.0.1:
+has@^1.0.1, has@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796"
   integrity sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==
@@ -3764,6 +3786,13 @@ html-comment-regex@^1.1.0:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/html-comment-regex/-/html-comment-regex-1.1.1.tgz#668b93776eaae55ebde8f3ad464b307a4963625e"
 
+html-element-map@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/html-element-map/-/html-element-map-1.0.1.tgz#3c4fcb4874ebddfe4283b51c8994e7713782b592"
+  integrity sha512-BZSfdEm6n706/lBfXKWa4frZRZcT5k1cOusw95ijZsHlI+GdgY0v95h6IzO3iIDf2ROwq570YTwqNPqHcNMozw==
+  dependencies:
+    array-filter "^1.0.0"
+
 html-encoding-sniffer@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-1.0.2.tgz#e70d84b94da53aa375e11fe3a351be6642ca46f8"
@@ -5093,6 +5122,11 @@ lodash.deburr@^3.0.0:
   dependencies:
     lodash._root "^3.0.0"
 
+lodash.escape@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/lodash.escape/-/lodash.escape-4.0.1.tgz#c9044690c21e04294beaa517712fded1fa88de98"
+  integrity sha1-yQRGkMIeBClL6qUXcS/e0fqI3pg=
+
 lodash.flattendeep@^4.4.0:
   version "4.4.0"
   resolved "https://registry.yarnpkg.com/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz#fb030917f86a3134e5bc9bec0d69e0013ddfedb2"
@@ -5112,6 +5146,11 @@ lodash.isequal@^3.0:
     lodash._baseisequal "^3.0.0"
     lodash._bindcallback "^3.0.0"
 
+lodash.isequal@^4.5.0:
+  version "4.5.0"
+  resolved "https://registry.yarnpkg.com/lodash.isequal/-/lodash.isequal-4.5.0.tgz#415c4478f2bcc30120c22ce10ed3226f7d3e18e0"
+  integrity sha1-QVxEePK8wwEgwizhDtMib30+GOA=
+
 lodash.isplainobject@^4.0.6:
   version "4.0.6"
   resolved "https://registry.yarnpkg.com/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz#7c526a52d89b45c45cc690b88163be0497f550cb"
@@ -5200,8 +5239,9 @@ lru-cache@^5.1.1:
     yallist "^3.0.2"
 
 macaddress@^0.2.8:
-  version "0.2.8"
-  resolved "https://registry.yarnpkg.com/macaddress/-/macaddress-0.2.8.tgz#5904dc537c39ec6dbefeae902327135fa8511f12"
+  version "0.2.9"
+  resolved "https://registry.yarnpkg.com/macaddress/-/macaddress-0.2.9.tgz#3579b8b9acd5b96b4553abf0f394185a86813cb3"
+  integrity sha512-k4F1JUof6cQXxNFzx3thLby4oJzXTXQueAOOts944Vqizn+Rjc2QNFenT9FJSLU1CH3PmrHRSyZs2E+Cqw+P2w==
 
 make-dir@^1.0.0:
   version "1.3.0"
@@ -5502,9 +5542,10 @@ mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkd
   dependencies:
     minimist "0.0.8"
 
-moment@^2.15.1:
-  version "2.18.1"
-  resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f"
+moment@^2.24.0:
+  version "2.24.0"
+  resolved "https://registry.yarnpkg.com/moment/-/moment-2.24.0.tgz#0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b"
+  integrity sha512-bV7f+6l2QigeBBZSM/6yTNq4P2fNpSWj/0e7jQcy87A8e7o2nAfP/34/2ky5Vw4B9S446EtIhodAzkFCcR4dQg==
 
 "mout@>=0.9 <2.0":
   version "1.0.0"
@@ -5858,9 +5899,10 @@ object-inspect@^1.1.0:
   version "1.2.2"
   resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.2.2.tgz#c82115e4fcc888aea14d64c22e4f17f6a70d5e5a"
 
-object-inspect@^1.5.0:
-  version "1.5.0"
-  resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.5.0.tgz#9d876c11e40f485c79215670281b767488f9bfe3"
+object-inspect@^1.6.0:
+  version "1.6.0"
+  resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.6.0.tgz#c70b6cbf72f274aab4c34c0c82f5167bf82cf15b"
+  integrity sha512-GJzfBZ6DgDAmnuaM3104jR4s1Myxr3Y3zfIyN4z3UdqN69oSRacNK8UhnobDdC+7J2AHCjGwxQubNJfE70SXXQ==
 
 object-is@^1.0.1:
   version "1.0.1"
@@ -6054,9 +6096,10 @@ osenv@^0.1.4:
     os-homedir "^1.0.0"
     os-tmpdir "^1.0.0"
 
-output-file-sync@^1.1.0:
+output-file-sync@^1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/output-file-sync/-/output-file-sync-1.1.2.tgz#d0a33eefe61a205facb90092e826598d5245ce76"
+  integrity sha1-0KM+7+YaIF+suQCS6CZZjVJFznY=
   dependencies:
     graceful-fs "^4.1.4"
     mkdirp "^0.5.1"
@@ -6630,7 +6673,7 @@ private@^0.1.6:
   version "0.1.7"
   resolved "https://registry.yarnpkg.com/private/-/private-0.1.7.tgz#68ce5e8a1ef0a23bb570cc28537b5332aba63ef1"
 
-private@^0.1.7:
+private@^0.1.7, private@^0.1.8:
   version "0.1.8"
   resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff"
 
@@ -7141,7 +7184,7 @@ regenerate@^1.2.1:
   version "1.3.2"
   resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.3.2.tgz#d1941c67bad437e1be76433add5b385f95b19260"
 
-regenerator-runtime@^0.10.0:
+regenerator-runtime@^0.10.0, regenerator-runtime@^0.10.5:
   version "0.10.5"
   resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.10.5.tgz#336c3efc1220adcedda2c9fab67b5a7955a33658"
 
@@ -7750,12 +7793,6 @@ source-map-support@^0.4.15:
   dependencies:
     source-map "^0.5.6"
 
-source-map-support@^0.4.2:
-  version "0.4.15"
-  resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.4.15.tgz#03202df65c06d2bd8c7ec2362a193056fef8d3b1"
-  dependencies:
-    source-map "^0.5.6"
-
 source-map-support@^0.5.0:
   version "0.5.3"
   resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.3.tgz#2b3d5fff298cfa4d1afd7d4352d569e9a0158e76"
@@ -7775,7 +7812,7 @@ source-map-url@^0.4.0:
   resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.4.0.tgz#3e935d7ddd73631b97659956d55128e87b5084a3"
   integrity sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=
 
-source-map@^0.5.0, source-map@^0.5.3, source-map@^0.5.6, source-map@~0.5.6:
+source-map@^0.5.3, source-map@^0.5.6, source-map@^0.5.7, source-map@~0.5.6:
   version "0.5.7"
   resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc"
   integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=
@@ -7969,6 +8006,15 @@ string-width@^2.0.0:
     is-fullwidth-code-point "^2.0.0"
     strip-ansi "^3.0.0"
 
+string.prototype.trim@^1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.1.2.tgz#d04de2c89e137f4d7d206f086b5ed2fae6be8cea"
+  integrity sha1-0E3iyJ4Tf019IG8Ia17S+ua+jOo=
+  dependencies:
+    define-properties "^1.1.2"
+    es-abstract "^1.5.0"
+    function-bind "^1.0.2"
+
 string_decoder@^1.0.0, string_decoder@~1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.3.tgz#0fc67d7c141825de94282dd536bec6b9bce860ab"
@@ -7995,15 +8041,11 @@ string_decoder@~1.1.1:
   dependencies:
     safe-buffer "~5.1.0"
 
-stringstream@~0.0.4:
+stringstream@~0.0.4, stringstream@~0.0.5:
   version "0.0.6"
   resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.6.tgz#7880225b0d4ad10e30927d167a1d6f2fd3b33a72"
   integrity sha512-87GEBAkegbBcweToUrdzf3eLhWNg06FJTebl4BVJz/JgWy8CvEr9dRtX5qWphiynMSQlxxi+QqN0z5T32SLlhA==
 
-stringstream@~0.0.5:
-  version "0.0.5"
-  resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878"
-
 strip-ansi@^3.0.0, strip-ansi@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf"
@@ -8546,9 +8588,10 @@ v8-compile-cache@^2.0.2:
   resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.0.2.tgz#a428b28bb26790734c4fc8bc9fa106fccebf6a6c"
   integrity sha512-1wFuMUIM16MDJRCrpbpuEPTUGmM5QMUg0cr3KFwra2XgOgFcPGDQHDh3CszSCD2Zewc/dh/pamNEW8CbfDebUw==
 
-v8flags@^2.0.10:
+v8flags@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/v8flags/-/v8flags-2.1.1.tgz#aab1a1fa30d45f88dd321148875ac02c0b55e5b4"
+  integrity sha1-qrGh+jDUX4jdMhFIh1rALAtV5bQ=
   dependencies:
     user-home "^1.1.1"
 

cmd/admin-handlers.go

@@ -34,6 +34,7 @@ import (
 	"github.com/tidwall/gjson"
 	"github.com/tidwall/sjson"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/disk"
@@ -43,6 +44,7 @@ import (
 	"github.com/minio/minio/pkg/mem"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/quick"
+	trace "github.com/minio/minio/pkg/trace"
 )
 
 const (
@@ -681,7 +683,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 					// Start writing response to client
 					started = true
 					setCommonHeaders(w)
-					w.Header().Set("Content-Type", string(mimeJSON))
+					w.Header().Set(xhttp.ContentType, string(mimeJSON))
 					// Set 200 OK status
 					w.WriteHeader(200)
 				}
@@ -701,20 +703,20 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 					var errorRespJSON []byte
 					if hr.errBody == "" {
 						errorRespJSON = encodeResponseJSON(getAPIErrorResponse(ctx, hr.apiErr,
-							r.URL.Path, w.Header().Get(responseRequestIDKey),
-							w.Header().Get(responseDeploymentIDKey)))
+							r.URL.Path, w.Header().Get(xhttp.AmzRequestID),
+							globalDeploymentID))
 					} else {
 						errorRespJSON = encodeResponseJSON(APIErrorResponse{
 							Code:      hr.apiErr.Code,
 							Message:   hr.errBody,
 							Resource:  r.URL.Path,
-							RequestID: w.Header().Get(responseRequestIDKey),
-							HostID:    w.Header().Get(responseDeploymentIDKey),
+							RequestID: w.Header().Get(xhttp.AmzRequestID),
+							HostID:    globalDeploymentID,
 						})
 					}
 					if !started {
 						setCommonHeaders(w)
-						w.Header().Set("Content-Type", string(mimeJSON))
+						w.Header().Set(xhttp.ContentType, string(mimeJSON))
 						w.WriteHeader(hr.apiErr.HTTPStatusCode)
 					}
 					w.Write(errorRespJSON)
@@ -785,6 +787,49 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	keepConnLive(w, respCh)
 }
 
+func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "HealBackgroundStatus")
+
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	// Check if this setup has an erasure coded backend.
+	if !globalIsXL {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrHealNotImplemented), r.URL)
+		return
+	}
+
+	var bgHealStates []madmin.BgHealState
+
+	// Get local heal status first
+	bgHealStates = append(bgHealStates, getLocalBackgroundHealStatus())
+
+	if globalIsDistXL {
+		// Get heal status from other peers
+		peersHealStates := globalNotificationSys.BackgroundHealStatus()
+		bgHealStates = append(bgHealStates, peersHealStates...)
+	}
+
+	// Aggregate healing result
+	var aggregatedHealStateResult = madmin.BgHealState{}
+	for _, state := range bgHealStates {
+		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
+		if aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
+			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
+		}
+
+	}
+
+	if err := json.NewEncoder(w).Encode(aggregatedHealStateResult); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
 // GetConfigHandler - GET /minio/admin/v1/config
 // Get config.json of this minio setup.
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
@@ -940,6 +985,12 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 
 	vars := mux.Vars(r)
 	accessKey := vars["accessKey"]
+
+	if err := globalIAMSys.DeleteUser(accessKey); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
 	// Notify all other MinIO peers to delete user.
 	for _, nerr := range globalNotificationSys.DeleteUser(accessKey) {
 		if nerr.Err != nil {
@@ -1426,26 +1477,49 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HTTPTrace")
 	trcAll := r.URL.Query().Get("all") == "true"
-	objectAPI := validateAdminReq(ctx, w, r)
-	if objectAPI == nil {
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
 	}
+
 	// Avoid reusing tcp connection if read timeout is hit
 	// This is needed to make r.Context().Done() work as
 	// expected in case of read timeout
-	w.Header().Add("Connection", "close")
+	w.Header().Add(xhttp.Connection, "close")
 
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 
-	traceCh := globalTrace.Trace(doneCh, trcAll)
+	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
+	// Use buffered channel to take care of burst sends or slow w.Write()
+	traceCh := make(chan interface{}, 4000)
+
+	filter := func(entry interface{}) bool {
+		if trcAll {
+			return true
+		}
+		trcInfo := entry.(trace.Info)
+		return !strings.HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath)
+	}
+	remoteHosts := getRemoteHosts(globalEndpoints)
+	peers, err := getRestClients(remoteHosts)
+	if err != nil {
+		return
+	}
+	globalHTTPTrace.Subscribe(traceCh, doneCh, filter)
+
+	for _, peer := range peers {
+		peer.Trace(traceCh, doneCh, trcAll)
+	}
+
+	enc := json.NewEncoder(w)
 	for {
 		select {
 		case entry := <-traceCh:
-			if _, err := w.Write(entry); err != nil {
-				return
-			}
-			if _, err := w.Write([]byte("\n")); err != nil {
+			if err := enc.Encode(entry); err != nil {
 				return
 			}
 			w.(http.Flusher).Flush()

cmd/admin-handlers_test.go

@@ -194,7 +194,9 @@ var (
     "webhook": {
       "1": {
         "enable": false,
-        "endpoint": ""
+        "endpoint": "",
+        "queueDir": "",
+        "queueLimit": 0
       }
     }
   },

cmd/admin-heal-ops.go

@@ -131,6 +131,19 @@ func (ahs *allHealState) periodicHealSeqsClean() {
 	}
 }
 
+// getHealSequenceByToken - Retrieve a heal sequence by token. The second
+// argument returns if a heal sequence actually exists.
+func (ahs *allHealState) getHealSequenceByToken(token string) (h *healSequence, exists bool) {
+	ahs.Lock()
+	defer ahs.Unlock()
+	for _, healSeq := range ahs.healSeqMap {
+		if healSeq.clientToken == token {
+			return healSeq, true
+		}
+	}
+	return nil, false
+}
+
 // getHealSequence - Retrieve a heal sequence by path. The second
 // argument returns if a heal sequence actually exists.
 func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists bool) {
@@ -335,6 +348,12 @@ type healSequence struct {
 	// the last result index sent to client
 	lastSentResultIndex int64
 
+	// Number of total items scanned
+	scannedItemsCount int64
+
+	// The time of the last scan/heal activity
+	lastHealActivity time.Time
+
 	// Holds the request-info for logging
 	ctx context.Context
 }
@@ -552,17 +571,20 @@ func (h *healSequence) queueHealTask(path string, healType madmin.HealItemType)
 }
 
 func (h *healSequence) healItemsFromSourceCh() error {
+	h.lastHealActivity = UTCNow()
+
 	// Start healing the config prefix.
 	if err := h.healMinioSysMeta(minioConfigPrefix)(); err != nil {
-		return err
+		logger.LogIf(h.ctx, err)
 	}
 
 	// Start healing the bucket config prefix.
 	if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil {
-		return err
+		logger.LogIf(h.ctx, err)
 	}
 
 	for path := range h.sourceCh {
+
 		var itemType madmin.HealItemType
 		switch {
 		case path == "/":
@@ -574,8 +596,11 @@ func (h *healSequence) healItemsFromSourceCh() error {
 		}
 
 		if err := h.queueHealTask(path, itemType); err != nil {
-			return err
+			logger.LogIf(h.ctx, err)
 		}
+
+		h.scannedItemsCount++
+		h.lastHealActivity = UTCNow()
 	}
 
 	return nil

cmd/admin-router.go

@@ -61,6 +61,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 
+		adminV1Router.Methods(http.MethodPost).Path("/background-heal/status").HandlerFunc(httpTraceAll(adminAPI.BackgroundHealStatusHandler))
+
 		/// Health operations
 
 	}

cmd/api-datatypes.go

@@ -20,13 +20,6 @@ import (
 	"encoding/xml"
 )
 
-const (
-	// Response request id.
-	responseRequestIDKey = "x-amz-request-id"
-	// Deployment id.
-	responseDeploymentIDKey = "x-minio-deployment-id"
-)
-
 // ObjectIdentifier carries key name for the object to delete.
 type ObjectIdentifier struct {
 	ObjectName string `xml:"Key"`

cmd/api-headers.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
 // Returns a hexadecimal representation of time at the
@@ -36,13 +37,13 @@ func mustGetRequestID(t time.Time) string {
 
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
-	w.Header().Set("Server", "MinIO/"+ReleaseTag)
+	w.Header().Set(xhttp.ServerInfo, "MinIO/"+ReleaseTag)
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.
 	if region := globalServerConfig.GetRegion(); region != "" {
-		w.Header().Set("X-Amz-Bucket-Region", region)
+		w.Header().Set(xhttp.AmzBucketRegion, region)
 	}
-	w.Header().Set("Accept-Ranges", "bytes")
+	w.Header().Set(xhttp.AcceptRanges, "bytes")
 
 	// Remove sensitive information
 	crypto.RemoveSensitiveHeaders(w.Header())
@@ -72,23 +73,23 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 
 	// Set last modified time.
 	lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat)
-	w.Header().Set("Last-Modified", lastModified)
+	w.Header().Set(xhttp.LastModified, lastModified)
 
 	// Set Etag if available.
 	if objInfo.ETag != "" {
-		w.Header()["ETag"] = []string{"\"" + objInfo.ETag + "\""}
+		w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
 	}
 
 	if objInfo.ContentType != "" {
-		w.Header().Set("Content-Type", objInfo.ContentType)
+		w.Header().Set(xhttp.ContentType, objInfo.ContentType)
 	}
 
 	if objInfo.ContentEncoding != "" {
-		w.Header().Set("Content-Encoding", objInfo.ContentEncoding)
+		w.Header().Set(xhttp.ContentEncoding, objInfo.ContentEncoding)
 	}
 
 	if !objInfo.Expires.IsZero() {
-		w.Header().Set("Expires", objInfo.Expires.UTC().Format(http.TimeFormat))
+		w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
 	}
 
 	// Set all other user defined metadata.
@@ -124,10 +125,10 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	}
 
 	// Set content length.
-	w.Header().Set("Content-Length", strconv.FormatInt(rangeLen, 10))
+	w.Header().Set(xhttp.ContentLength, strconv.FormatInt(rangeLen, 10))
 	if rs != nil {
 		contentRange := fmt.Sprintf("bytes %d-%d/%d", start, start+rangeLen-1, totalObjectSize)
-		w.Header().Set("Content-Range", contentRange)
+		w.Header().Set(xhttp.ContentRange, contentRange)
 	}
 
 	return nil

cmd/api-response.go

@@ -22,9 +22,11 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"strconv"
 	"strings"
 	"time"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/handlers"
 )
@@ -521,8 +523,9 @@ func generateMultiDeleteResponse(quiet bool, deletedObjects []ObjectIdentifier,
 func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
 	setCommonHeaders(w)
 	if mType != mimeNone {
-		w.Header().Set("Content-Type", string(mType))
+		w.Header().Set(xhttp.ContentType, string(mType))
 	}
+	w.Header().Set(xhttp.ContentLength, strconv.Itoa(len(response)))
 	w.WriteHeader(statusCode)
 	if response != nil {
 		w.Write(response)
@@ -561,7 +564,7 @@ func writeSuccessNoContent(w http.ResponseWriter) {
 
 // writeRedirectSeeOther writes Location header with http status 303
 func writeRedirectSeeOther(w http.ResponseWriter, location string) {
-	w.Header().Set("Location", location)
+	w.Header().Set(xhttp.Location, location)
 	writeResponse(w, http.StatusSeeOther, nil, mimeNone)
 }
 
@@ -575,12 +578,12 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 	case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
 		// Set retry-after header to indicate user-agents to retry request after 120secs.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
-		w.Header().Set("Retry-After", "120")
+		w.Header().Set(xhttp.RetryAfter, "120")
 	case "AccessDenied":
 		// The request is from browser and also if browser
 		// is enabled we need to redirect.
 		if browser {
-			w.Header().Set("Location", minioReservedBucketPath+reqURL.Path)
+			w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
 			w.WriteHeader(http.StatusTemporaryRedirect)
 			return
 		}
@@ -588,7 +591,7 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 
 	// Generate error response.
 	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path,
-		w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
+		w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
 	encodedErrorResponse := encodeResponse(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }
@@ -601,7 +604,7 @@ func writeErrorResponseHeadersOnly(w http.ResponseWriter, err APIError) {
 // useful for admin APIs.
 func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
+	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }
@@ -619,8 +622,8 @@ func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, er
 		Resource:   reqURL.Path,
 		BucketName: reqInfo.BucketName,
 		Key:        reqInfo.ObjectName,
-		RequestID:  w.Header().Get(responseRequestIDKey),
-		HostID:     w.Header().Get(responseDeploymentIDKey),
+		RequestID:  w.Header().Get(xhttp.AmzRequestID),
+		HostID:     globalDeploymentID,
 	}
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
@@ -635,12 +638,12 @@ func writeCustomErrorResponseXML(ctx context.Context, w http.ResponseWriter, err
 	case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
 		// Set retry-after header to indicate user-agents to retry request after 120secs.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
-		w.Header().Set("Retry-After", "120")
+		w.Header().Set(xhttp.RetryAfter, "120")
 	case "AccessDenied":
 		// The request is from browser and also if browser
 		// is enabled we need to redirect.
 		if browser && globalIsBrowserEnabled {
-			w.Header().Set("Location", minioReservedBucketPath+reqURL.Path)
+			w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
 			w.WriteHeader(http.StatusTemporaryRedirect)
 			return
 		}
@@ -653,8 +656,8 @@ func writeCustomErrorResponseXML(ctx context.Context, w http.ResponseWriter, err
 		Resource:   reqURL.Path,
 		BucketName: reqInfo.BucketName,
 		Key:        reqInfo.ObjectName,
-		RequestID:  w.Header().Get(responseRequestIDKey),
-		HostID:     w.Header().Get(responseDeploymentIDKey),
+		RequestID:  w.Header().Get(xhttp.AmzRequestID),
+		HostID:     globalDeploymentID,
 	}
 
 	encodedErrorResponse := encodeResponse(errorResponse)

cmd/api-router.go

@@ -20,6 +20,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
 // objectAPIHandler implements and provides http handlers for S3 API.
@@ -28,10 +29,12 @@ type objectAPIHandlers struct {
 	CacheAPI  func() CacheObjectLayer
 	// Returns true of handlers should interpret encryption.
 	EncryptionEnabled func() bool
+	// Returns true if handlers allow SSE-KMS encryption headers.
+	AllowSSEKMS func() bool
 }
 
 // registerAPIRouter - registers S3 compatible APIs.
-func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
+func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool) {
 	// Initialize API.
 	api := objectAPIHandlers{
 		ObjectAPI: newObjectLayerFn,
@@ -39,6 +42,9 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
 		EncryptionEnabled: func() bool {
 			return encryptionEnabled
 		},
+		AllowSSEKMS: func() bool {
+			return allowSSEKMS
+		},
 	}
 
 	// API Router
@@ -53,98 +59,98 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
 	for _, bucket := range routers {
 		// Object operations
 		// HeadObject
-		bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
+		bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
 		// CopyObjectPart
-		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
-		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// ListObjectPxarts
-		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
 		// CompleteMultipartUpload
-		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
 		// NewMultipartUpload
-		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
 		// AbortMultipartUpload
-		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
-		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
 		// GetObjectTagging - this is a dummy call.
-		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectTaggingHandler)).Queries("tagging", "")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectTaggingHandler)).Queries("tagging", "")
 		// SelectObjectContent
-		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.SelectObjectContentHandler)).Queries("select", "").Queries("select-type", "2")
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.SelectObjectContentHandler)).Queries("select", "").Queries("select-type", "2")
 		// GetObject
-		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
 		// CopyObject
-		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
 		// PutObject
-		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
 		// DeleteObject
-		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
+		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
 
 		/// Bucket operations
 		// GetBucketLocation
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
 		// GetBucketPolicy
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketACLHandler)).Queries("acl", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketACLHandler)).Queries("acl", "")
 		// GetBucketCors - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketCorsHandler)).Queries("cors", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketCorsHandler)).Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketWebsiteHandler)).Queries("website", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketWebsiteHandler)).Queries("website", "")
 		// GetBucketVersioningHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketVersioningHandler)).Queries("versioning", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketVersioningHandler)).Queries("versioning", "")
 		// GetBucketAccelerateHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketAccelerateHandler)).Queries("accelerate", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketAccelerateHandler)).Queries("accelerate", "")
 		// GetBucketRequestPaymentHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketRequestPaymentHandler)).Queries("requestPayment", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketRequestPaymentHandler)).Queries("requestPayment", "")
 		// GetBucketLoggingHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLoggingHandler)).Queries("logging", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLoggingHandler)).Queries("logging", "")
 		// GetBucketLifecycleHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
 		// GetBucketReplicationHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketReplicationHandler)).Queries("replication", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketReplicationHandler)).Queries("replication", "")
 		// GetBucketTaggingHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketTaggingHandler)).Queries("tagging", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketTaggingHandler)).Queries("tagging", "")
 		//DeleteBucketWebsiteHandler
-		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketWebsiteHandler)).Queries("website", "")
+		bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketWebsiteHandler)).Queries("website", "")
 		// DeleteBucketTaggingHandler
-		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketTaggingHandler)).Queries("tagging", "")
+		bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketTaggingHandler)).Queries("tagging", "")
 
 		// GetBucketNotification
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
 		// ListenBucketNotification
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
 		// ListMultipartUploads
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
 		// ListObjectsV2
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
 		// ListObjectsV1 (Legacy)
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
 		// PutBucketPolicy
-		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
+		bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
 		// PutBucketNotification
-		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
+		bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
 		// PutBucket
-		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler))
+		bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketHandler))
 		// HeadBucket
-		bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
+		bucket.Methods(http.MethodHead).HandlerFunc(httpTraceAll(api.HeadBucketHandler))
 		// PostPolicy
-		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceHdrs(api.PostPolicyBucketHandler))
+		bucket.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(httpTraceHdrs(api.PostPolicyBucketHandler))
 		// DeleteMultipleObjects
-		bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
+		bucket.Methods(http.MethodPost).HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
 		// DeleteBucketPolicy
-		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
+		bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
 		// DeleteBucket
-		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
+		bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
 	}
 
 	/// Root operation
 
 	// ListBuckets
-	apiRouter.Methods("GET").Path("/").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
+	apiRouter.Methods(http.MethodGet).Path("/").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
 
 	// If none of the routes match.
 	apiRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))

cmd/auth-handler.go

@@ -29,6 +29,7 @@ import (
 	"strings"
 
 	jwtgo "github.com/dgrijalva/jwt-go"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/hash"
@@ -38,41 +39,41 @@ import (
 
 // Verify if request has JWT.
 func isRequestJWT(r *http.Request) bool {
-	return strings.HasPrefix(r.Header.Get("Authorization"), jwtAlgorithm)
+	return strings.HasPrefix(r.Header.Get(xhttp.Authorization), jwtAlgorithm)
 }
 
 // Verify if request has AWS Signature Version '4'.
 func isRequestSignatureV4(r *http.Request) bool {
-	return strings.HasPrefix(r.Header.Get("Authorization"), signV4Algorithm)
+	return strings.HasPrefix(r.Header.Get(xhttp.Authorization), signV4Algorithm)
 }
 
 // Verify if request has AWS Signature Version '2'.
 func isRequestSignatureV2(r *http.Request) bool {
-	return (!strings.HasPrefix(r.Header.Get("Authorization"), signV4Algorithm) &&
-		strings.HasPrefix(r.Header.Get("Authorization"), signV2Algorithm))
+	return (!strings.HasPrefix(r.Header.Get(xhttp.Authorization), signV4Algorithm) &&
+		strings.HasPrefix(r.Header.Get(xhttp.Authorization), signV2Algorithm))
 }
 
 // Verify if request has AWS PreSign Version '4'.
 func isRequestPresignedSignatureV4(r *http.Request) bool {
-	_, ok := r.URL.Query()["X-Amz-Credential"]
+	_, ok := r.URL.Query()[xhttp.AmzCredential]
 	return ok
 }
 
 // Verify request has AWS PreSign Version '2'.
 func isRequestPresignedSignatureV2(r *http.Request) bool {
-	_, ok := r.URL.Query()["AWSAccessKeyId"]
+	_, ok := r.URL.Query()[xhttp.AmzAccessKeyID]
 	return ok
 }
 
 // Verify if request has AWS Post policy Signature Version '4'.
 func isRequestPostPolicySignatureV4(r *http.Request) bool {
-	return strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data") &&
+	return strings.Contains(r.Header.Get(xhttp.ContentType), "multipart/form-data") &&
 		r.Method == http.MethodPost
 }
 
 // Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
 func isRequestSignStreamingV4(r *http.Request) bool {
-	return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 &&
+	return r.Header.Get(xhttp.AmzContentSha256) == streamingContentSHA256 &&
 		r.Method == http.MethodPut
 }
 
@@ -109,9 +110,9 @@ func getRequestAuthType(r *http.Request) authType {
 		return authTypeJWT
 	} else if isRequestPostPolicySignatureV4(r) {
 		return authTypePostPolicy
-	} else if _, ok := r.URL.Query()["Action"]; ok {
+	} else if _, ok := r.URL.Query()[xhttp.Action]; ok {
 		return authTypeSTS
-	} else if _, ok := r.Header["Authorization"]; !ok {
+	} else if _, ok := r.Header[xhttp.Authorization]; !ok {
 		return authTypeAnonymous
 	}
 	return authTypeUnknown
@@ -121,7 +122,7 @@ func getRequestAuthType(r *http.Request) authType {
 // It does not accept presigned or JWT or anonymous requests.
 func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region string) APIErrorCode {
 	s3Err := ErrAccessDenied
-	if _, ok := r.Header["X-Amz-Content-Sha256"]; ok &&
+	if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
 		getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) {
 		// We only support admin credentials to access admin APIs.
 
@@ -148,11 +149,11 @@ func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region stri
 
 // Fetch the security token set by the client.
 func getSessionToken(r *http.Request) (token string) {
-	token = r.Header.Get("X-Amz-Security-Token")
+	token = r.Header.Get(xhttp.AmzSecurityToken)
 	if token != "" {
 		return token
 	}
-	return r.URL.Query().Get("X-Amz-Security-Token")
+	return r.URL.Query().Get(xhttp.AmzSecurityToken)
 }
 
 // Fetch claims in the security token returned by the client, doesn't return
@@ -200,6 +201,39 @@ func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
 	if _, ok = v.(string); !ok {
 		return nil, errInvalidAccessKeyID
 	}
+
+	if globalPolicyOPA == nil {
+		// If OPA is not set, session token should
+		// have a policy and its mandatory, reject
+		// requests without policy claim.
+		p, pok := claims[iampolicy.PolicyName]
+		if !pok {
+			return nil, errAuthentication
+		}
+		if _, pok = p.(string); !pok {
+			return nil, errAuthentication
+		}
+		sp, spok := claims[iampolicy.SessionPolicyName]
+		// Sub policy is optional, if not set return success.
+		if !spok {
+			return claims, nil
+		}
+		// Sub policy is set but its not a string, reject such requests
+		spStr, spok := sp.(string)
+		if !spok {
+			return nil, errAuthentication
+		}
+		// Looks like subpolicy is set and is a string, if set then its
+		// base64 encoded, decode it. Decoding fails reject such requests.
+		spBytes, err := base64.StdEncoding.DecodeString(spStr)
+		if err != nil {
+			// Base64 decoding fails, we should log to indicate
+			// something is malforming the request sent by client.
+			logger.LogIf(context.Background(), err)
+			return nil, errAuthentication
+		}
+		claims[iampolicy.SessionPolicyName] = string(spBytes)
+	}
 	return claims, nil
 }
 
@@ -337,8 +371,8 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 		contentMD5, contentSHA256 []byte
 	)
 	// Extract 'Content-Md5' if present.
-	if _, ok := r.Header["Content-Md5"]; ok {
-		contentMD5, err = base64.StdEncoding.Strict().DecodeString(r.Header.Get("Content-Md5"))
+	if _, ok := r.Header[xhttp.ContentMD5]; ok {
+		contentMD5, err = base64.StdEncoding.Strict().DecodeString(r.Header.Get(xhttp.ContentMD5))
 		if err != nil || len(contentMD5) == 0 {
 			return ErrInvalidDigest
 		}
@@ -347,14 +381,14 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 	// Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned)
 	// Do not verify 'X-Amz-Content-Sha256' if skipSHA256.
 	if skipSHA256 := skipContentSha256Cksum(r); !skipSHA256 && isRequestPresignedSignatureV4(r) {
-		if sha256Sum, ok := r.URL.Query()["X-Amz-Content-Sha256"]; ok && len(sha256Sum) > 0 {
+		if sha256Sum, ok := r.URL.Query()[xhttp.AmzContentSha256]; ok && len(sha256Sum) > 0 {
 			contentSHA256, err = hex.DecodeString(sha256Sum[0])
 			if err != nil {
 				return ErrContentSHA256Mismatch
 			}
 		}
-	} else if _, ok := r.Header["X-Amz-Content-Sha256"]; !skipSHA256 && ok {
-		contentSHA256, err = hex.DecodeString(r.Header.Get("X-Amz-Content-Sha256"))
+	} else if _, ok := r.Header[xhttp.AmzContentSha256]; !skipSHA256 && ok {
+		contentSHA256, err = hex.DecodeString(r.Header.Get(xhttp.AmzContentSha256))
 		if err != nil || len(contentSHA256) == 0 {
 			return ErrContentSHA256Mismatch
 		}
@@ -362,7 +396,8 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 
 	// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
 	// The verification happens implicit during reading.
-	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256), -1, globalCLIContext.StrictS3Compat)
+	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5),
+		hex.EncodeToString(contentSHA256), -1, globalCLIContext.StrictS3Compat)
 	if err != nil {
 		return toAPIErrorCode(ctx, err)
 	}

cmd/bucket-handlers-listobjects.go

@@ -46,13 +46,6 @@ func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, max
 		}
 	}
 
-	/// MinIO special conditions for ListObjects.
-
-	// Verify if delimiter is anything other than '/', which we do not support.
-	if delimiter != "" && delimiter != "/" {
-		return ErrNotImplemented
-	}
-	// Success.
 	return ErrNone
 }
 

cmd/bucket-handlers.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
@@ -445,7 +446,8 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				}
 
 				// Make sure to add Location information here only for bucket
-				w.Header().Set("Location", getObjectLocation(r, globalDomainNames, bucket, ""))
+				w.Header().Set(xhttp.Location,
+					getObjectLocation(r, globalDomainNames, bucket, ""))
 
 				writeSuccessResponseHeadersOnly(w)
 				return
@@ -466,7 +468,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Make sure to add Location information here only for bucket
-	w.Header().Set("Location", path.Clean(r.URL.Path)) // Clean any trailing slashes.
+	w.Header().Set(xhttp.Location, path.Clean(r.URL.Path)) // Clean any trailing slashes.
 
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -498,6 +500,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	bucket := mux.Vars(r)["bucket"]
 
+	// To detect if the client has disconnected.
+	r.Body = &detectDisconnect{r.Body, r.Context().Done()}
+
 	// Require Content-Length to be set in the request
 	size := r.ContentLength
 	if size < 0 {
@@ -678,8 +683,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 
 	location := getObjectLocation(r, globalDomainNames, bucket, object)
-	w.Header()["ETag"] = []string{`"` + objInfo.ETag + `"`}
-	w.Header().Set("Location", location)
+	w.Header()[xhttp.ETag] = []string{`"` + objInfo.ETag + `"`}
+	w.Header().Set(xhttp.Location, location)
 
 	// Notify object created event.
 	defer sendEvent(eventArgs{

cmd/config-common.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"time"
 
 	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/minio/minio/cmd/logger"
@@ -53,8 +52,19 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
 }
 
 func deleteConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
-	_, err := client.Delete(ctx, configFile)
-	return err
+	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+
+	_, err := client.Delete(timeoutCtx, configFile)
+	if err != nil {
+		if err == context.DeadlineExceeded {
+			return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s",
+				client.Endpoints())
+		}
+		return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s",
+			err, client.Endpoints())
+	}
+	return nil
 }
 
 func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
@@ -89,9 +99,11 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
 	resp, err := client.Get(timeoutCtx, configFile)
 	if err != nil {
 		if err == context.DeadlineExceeded {
-			return nil, fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
+			return nil, fmt.Errorf("etcd setup is unreachable, please check your endpoints %s",
+				client.Endpoints())
 		}
-		return nil, fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
+		return nil, fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s",
+			err, client.Endpoints())
 	}
 	if resp.Count == 0 {
 		return nil, errConfigNotFound
@@ -104,54 +116,7 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
 	return nil, errConfigNotFound
 }
 
-// watchConfigEtcd - watches for changes on `configFile` on etcd and loads them.
-func watchConfigEtcd(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
-	for {
-		watchCh := globalEtcdClient.Watch(context.Background(), configFile)
-		select {
-		case <-GlobalServiceDoneCh:
-			return
-		case watchResp, ok := <-watchCh:
-			if !ok {
-				time.Sleep(1 * time.Second)
-				continue
-			}
-			if err := watchResp.Err(); err != nil {
-				logger.LogIf(context.Background(), err)
-				// log and retry.
-				time.Sleep(1 * time.Second)
-				continue
-			}
-			for _, event := range watchResp.Events {
-				if event.IsModify() || event.IsCreate() {
-					loadCfgFn(objAPI)
-				}
-			}
-		}
-	}
-}
-
-func checkConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
-	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
-	defer cancel()
-	resp, err := client.Get(timeoutCtx, configFile)
-	if err != nil {
-		if err == context.DeadlineExceeded {
-			return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
-		}
-		return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
-	}
-	if resp.Count == 0 {
-		return errConfigNotFound
-	}
-	return nil
-}
-
 func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
-	if globalEtcdClient != nil {
-		return checkConfigEtcd(ctx, globalEtcdClient, configFile)
-	}
-
 	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
 		// Treat object not found as config not found.
 		if isErrObjectNotFound(err) {

cmd/config-current.go

@@ -767,7 +767,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 	for id, args := range config.Notify.Webhook {
 		if args.Enable {
 			args.RootCAs = globalRootCAs
-			newTarget := target.NewWebhookTarget(id, args)
+			newTarget := target.NewWebhookTarget(id, args, GlobalServiceDoneCh)
 			if err := targetList.Add(newTarget); err != nil {
 				logger.LogIf(context.Background(), err)
 				continue

cmd/config-current_test.go

@@ -203,7 +203,7 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "", "queueDir": "", "queueLimit": 0 } }}}`, false},
 
 		// Test 17 - Test Webhook
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "", "queueDir": "", "queueLimit": 0} }}}`, false},
 
 		// Test 18 - Test MySQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},

cmd/config-migrate.go

@@ -29,7 +29,7 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
-	"github.com/minio/minio/pkg/iam/policy"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/iam/validator"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/quick"
@@ -2413,6 +2413,7 @@ func migrateV27ToV28() error {
 }
 
 // Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
+// if etcd is configured then migrates /config/config.json to '<export_path>/.minio.sys/config/config.json'
 func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	// Construct path to config.json for the given bucket.
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
@@ -2423,10 +2424,14 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	}
 
 	defer func() {
-		// Rename config.json to config.json.deprecated only upon
-		// success of this function.
 		if err == nil {
-			os.Rename(getConfigFile(), getConfigFile()+".deprecated")
+			if globalEtcdClient != nil {
+				deleteConfigEtcd(context.Background(), globalEtcdClient, configFile)
+			} else {
+				// Rename config.json to config.json.deprecated only upon
+				// success of this function.
+				os.Rename(getConfigFile(), getConfigFile()+".deprecated")
+			}
 		}
 	}()
 
@@ -2435,7 +2440,7 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
 	// and configFile, take a transaction lock to avoid data race between readConfig()
 	// and saveConfig().
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
+	objLock := globalNSMutex.NewNSLock(context.Background(), minioMetaBucket, transactionConfigFile)
 	if err = objLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}
@@ -2446,20 +2451,24 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 		return err
 	} // if errConfigNotFound proceed to migrate..
 
+	var configFiles = []string{
+		getConfigFile(),
+		getConfigFile() + ".deprecated",
+		configFile,
+	}
 	var config = &serverConfig{}
-	if _, err = Load(getConfigFile(), config); err != nil {
-		if !os.IsNotExist(err) {
-			return err
-		}
-		// Read from deprecate file as well if necessary.
-		if _, err = Load(getConfigFile()+".deprecated", config); err != nil {
+	for _, cfgFile := range configFiles {
+		if _, err = Load(cfgFile, config); err != nil {
 			if !os.IsNotExist(err) {
 				return err
 			}
-			// If all else fails simply initialize the server config.
-			return newSrvConfig(objAPI)
+			continue
 		}
-
+		break
+	}
+	if os.IsNotExist(err) {
+		// Initialize the server config, if no config exists.
+		return newSrvConfig(objAPI)
 	}
 	return saveServerConfig(context.Background(), objAPI, config)
 }
@@ -2483,7 +2492,7 @@ func migrateMinioSysConfig(objAPI ObjectLayer) error {
 	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
 	// and configFile, take a transaction lock to avoid data race between readConfig()
 	// and saveConfig().
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
+	objLock := globalNSMutex.NewNSLock(context.Background(), minioMetaBucket, transactionConfigFile)
 	if err := objLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}

cmd/config-versions.go

@@ -884,7 +884,7 @@ type serverConfigV32 struct {
 	} `json:"policy"`
 }
 
-// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in MQTT and kafka.
+// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in notification targets.
 type serverConfigV33 struct {
 	quick.Config `json:"-"` // ignore interfaces
 

cmd/config.go

@@ -49,10 +49,6 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
 	}
 
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	if globalEtcdClient != nil {
-		return saveConfigEtcd(ctx, globalEtcdClient, configFile, data)
-	}
-
 	// Create a backup of the current config
 	oldData, err := readConfig(ctx, objAPI, configFile)
 	if err == nil {
@@ -71,15 +67,8 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
 }
 
 func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, error) {
-	var configData []byte
-	var err error
-
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	if globalEtcdClient != nil {
-		configData, err = readConfigEtcd(ctx, globalEtcdClient, configFile)
-	} else {
-		configData, err = readConfig(ctx, objAPI, configFile)
-	}
+	configData, err := readConfig(ctx, objAPI, configFile)
 	if err != nil {
 		return nil, err
 	}
@@ -152,45 +141,25 @@ func initConfig(objAPI ObjectLayer) error {
 		return errServerNotInitialized
 	}
 
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	if globalEtcdClient != nil {
-		if err := checkConfigEtcd(context.Background(), globalEtcdClient, getConfigFile()); err != nil {
-			if err == errConfigNotFound {
-				// Migrates all configs at old location.
-				if err = migrateConfig(); err != nil {
-					return err
-				}
-				// Migrates etcd ${HOME}/.minio/config.json to '/config/config.json'
-				if err = migrateConfigToMinioSys(objAPI); err != nil {
-					return err
-				}
-			} else {
-				return err
-			}
-		}
-
-		// Watch config for changes and reloads them.
-		go watchConfigEtcd(objAPI, configFile, loadConfig)
-
-	} else {
-		if isFile(getConfigFile()) {
-			if err := migrateConfig(); err != nil {
-				return err
-			}
-		}
-		// Migrates ${HOME}/.minio/config.json or config.json.deprecated
-		// to '<export_path>/.minio.sys/config/config.json'
-		// ignore if the file doesn't exist.
-		if err := migrateConfigToMinioSys(objAPI); err != nil {
-			return err
-		}
-
-		// Migrates backend '<export_path>/.minio.sys/config/config.json' to latest version.
-		if err := migrateMinioSysConfig(objAPI); err != nil {
+	if isFile(getConfigFile()) {
+		if err := migrateConfig(); err != nil {
 			return err
 		}
 	}
 
+	// Migrates ${HOME}/.minio/config.json or config.json.deprecated
+	// to '<export_path>/.minio.sys/config/config.json'
+	// ignore if the file doesn't exist.
+	// If etcd is set then migrates /config/config.json
+	// to '<export_path>/.minio.sys/config/config.json'
+	if err := migrateConfigToMinioSys(objAPI); err != nil {
+		return err
+	}
+
+	// Migrates backend '<export_path>/.minio.sys/config/config.json' to latest version.
+	if err := migrateMinioSysConfig(objAPI); err != nil {
+		return err
+	}
+
 	return loadConfig(objAPI)
 }

cmd/crypto/header.go

@@ -18,6 +18,7 @@ import (
 	"bytes"
 	"crypto/md5"
 	"encoding/base64"
+	"encoding/json"
 	"net/http"
 	"strings"
 )
@@ -125,6 +126,25 @@ func (s3KMS) IsRequested(h http.Header) bool {
 	return false
 }
 
+// ParseHTTP parses the SSE-KMS headers and returns the SSE-KMS key ID
+// and context, if present, on success.
+func (s3KMS) ParseHTTP(h http.Header) (string, interface{}, error) {
+	algorithm := h.Get(SSEHeader)
+	if algorithm != SSEAlgorithmKMS {
+		return "", nil, ErrInvalidEncryptionMethod
+	}
+
+	contextStr, ok := h[SSEKmsContext]
+	if ok {
+		var context map[string]interface{}
+		if err := json.Unmarshal([]byte(contextStr[0]), &context); err != nil {
+			return "", nil, err
+		}
+		return h.Get(SSEKmsID), context, nil
+	}
+	return h.Get(SSEKmsID), nil, nil
+}
+
 var (
 	// SSEC represents AWS SSE-C. It provides functionality to handle
 	// SSE-C requests.

cmd/crypto/header_test.go

@@ -54,6 +54,56 @@ func TestKMSIsRequested(t *testing.T) {
 	}
 }
 
+var kmsParseHTTPTests = []struct {
+	Header     http.Header
+	ShouldFail bool
+}{
+	{Header: http.Header{}, ShouldFail: true},                                                     // 0
+	{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"aws:kms"}}, ShouldFail: false}, // 1
+	{Header: http.Header{
+		"X-Amz-Server-Side-Encryption":                []string{"aws:kms"},
+		"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"},
+	}, ShouldFail: false}, // 2
+	{Header: http.Header{
+		"X-Amz-Server-Side-Encryption":                []string{"aws:kms"},
+		"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"},
+		"X-Amz-Server-Side-Encryption-Context":        []string{"{}"},
+	}, ShouldFail: false}, // 3
+	{Header: http.Header{
+		"X-Amz-Server-Side-Encryption":                []string{"aws:kms"},
+		"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"},
+		"X-Amz-Server-Side-Encryption-Context":        []string{"{\"bucket\": \"some-bucket\"}"},
+	}, ShouldFail: false}, // 4
+	{Header: http.Header{
+		"X-Amz-Server-Side-Encryption":                []string{"aws:kms"},
+		"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"},
+		"X-Amz-Server-Side-Encryption-Context":        []string{"{\"bucket\": \"some-bucket\"}"},
+	}, ShouldFail: false}, // 5
+	{Header: http.Header{
+		"X-Amz-Server-Side-Encryption":                []string{"AES256"},
+		"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"},
+		"X-Amz-Server-Side-Encryption-Context":        []string{"{\"bucket\": \"some-bucket\"}"},
+	}, ShouldFail: true}, // 6
+	{Header: http.Header{
+		"X-Amz-Server-Side-Encryption":                []string{"aws:kms"},
+		"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"},
+		"X-Amz-Server-Side-Encryption-Context":        []string{"{\"bucket\": \"some-bucket\""}, // invalid JSON
+	}, ShouldFail: true}, // 7
+
+}
+
+func TestKMSParseHTTP(t *testing.T) {
+	for i, test := range kmsParseHTTPTests {
+		_, _, err := S3KMS.ParseHTTP(test.Header)
+		if err == nil && test.ShouldFail {
+			t.Errorf("Test %d: should fail but succeeded", i)
+		}
+		if err != nil && !test.ShouldFail {
+			t.Errorf("Test %d: should pass but failed with: %v", i, err)
+		}
+	}
+}
+
 var s3IsRequestedTests = []struct {
 	Header   http.Header
 	Expected bool

cmd/daily-heal-ops.go

@@ -39,7 +39,7 @@ func newBgHealSequence(numDisks int) *healSequence {
 	hs := madmin.HealOpts{
 		// Remove objects that do not have read-quorum
 		Remove:   true,
-		ScanMode: madmin.HealDeepScan,
+		ScanMode: madmin.HealNormalScan,
 	}
 
 	return &healSequence{
@@ -60,6 +60,18 @@ func newBgHealSequence(numDisks int) *healSequence {
 	}
 }
 
+func getLocalBackgroundHealStatus() madmin.BgHealState {
+	backgroundSequence, ok := globalSweepHealState.getHealSequenceByToken(bgHealingUUID)
+	if !ok {
+		return madmin.BgHealState{}
+	}
+
+	return madmin.BgHealState{
+		ScannedItemsCount: backgroundSequence.scannedItemsCount,
+		LastHealActivity:  backgroundSequence.lastHealActivity,
+	}
+}
+
 func initDailyHeal() {
 	go startDailyHeal()
 }

cmd/daily-sweeper.go

@@ -55,7 +55,7 @@ func sweepRound(ctx context.Context, objAPI ObjectLayer) error {
 	zeroDynamicTimeout := newDynamicTimeout(zeroDuration, zeroDuration)
 
 	// General lock so we avoid parallel daily sweep by different instances.
-	sweepLock := globalNSMutex.NewNSLock("system", "daily-sweep")
+	sweepLock := globalNSMutex.NewNSLock(ctx, "system", "daily-sweep")
 	if err := sweepLock.GetLock(zeroDynamicTimeout); err != nil {
 		return err
 	}

cmd/disk-cache-fs.go

@@ -303,7 +303,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 	data := r.Reader
 	fs := cfs.FSObjects
 	// Lock the object.
-	objectLock := fs.nsMutex.NewNSLock(bucket, object)
+	objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
 	if err := objectLock.GetLock(globalObjectTimeout); err != nil {
 		return objInfo, err
 	}
@@ -489,7 +489,7 @@ func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, objec
 // moveBucketToTrash clears cacheFSObjects of bucket contents and moves it to trash folder.
 func (cfs *cacheFSObjects) moveBucketToTrash(ctx context.Context, bucket string) (err error) {
 	fs := cfs.FSObjects
-	bucketLock := fs.nsMutex.NewNSLock(bucket, "")
+	bucketLock := fs.nsMutex.NewNSLock(ctx, bucket, "")
 	if err = bucketLock.GetLock(globalObjectTimeout); err != nil {
 		return err
 	}

cmd/encryption-v1.go

@@ -1238,6 +1238,17 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
 		opts.UserDefined = metadata
 		return
 	}
+	if crypto.S3KMS.IsRequested(r.Header) {
+		keyID, context, err := crypto.S3KMS.ParseHTTP(r.Header)
+		if err != nil {
+			return ObjectOptions{}, err
+		}
+		sseKms, err := encrypt.NewSSEKMS(keyID, context)
+		if err != nil {
+			return ObjectOptions{}, err
+		}
+		return ObjectOptions{ServerSideEncryption: sseKms, UserDefined: metadata}, nil
+	}
 	// default case of passing encryption headers and UserDefined metadata to backend
 	return getDefaultOpts(r.Header, false, metadata)
 }

cmd/erasure.go

@@ -77,10 +77,7 @@ func (e *Erasure) DecodeDataBlocks(data [][]byte) error {
 	if !needsReconstruction {
 		return nil
 	}
-	if err := e.encoder.ReconstructData(data); err != nil {
-		return err
-	}
-	return nil
+	return e.encoder.ReconstructData(data)
 }
 
 // DecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it.

cmd/format-xl.go

@@ -491,7 +491,7 @@ func formatXLGetDeploymentID(refFormat *formatXLV3, formats []*formatXLV3) (stri
 func formatXLFixDeploymentID(ctx context.Context, endpoints EndpointList, storageDisks []StorageAPI, refFormat *formatXLV3) (err error) {
 	// Acquire lock on format.json
 	mutex := newNSLock(globalIsDistXL)
-	formatLock := mutex.NewNSLock(minioMetaBucket, formatConfigFile)
+	formatLock := mutex.NewNSLock(ctx, minioMetaBucket, formatConfigFile)
 	if err = formatLock.GetLock(globalHealingTimeout); err != nil {
 		return err
 	}

cmd/fs-v1-multipart.go

@@ -634,7 +634,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	// Hold write lock on the object.
-	destLock := fs.nsMutex.NewNSLock(bucket, object)
+	destLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
 	if err = destLock.GetLock(globalObjectTimeout); err != nil {
 		return oi, err
 	}

cmd/fs-v1.go

@@ -286,7 +286,7 @@ func (fs *FSObjects) statBucketDir(ctx context.Context, bucket string) (os.FileI
 // MakeBucketWithLocation - create a new bucket, returns if it
 // already exists.
 func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
-	bucketLock := fs.nsMutex.NewNSLock(bucket, "")
+	bucketLock := fs.nsMutex.NewNSLock(ctx, bucket, "")
 	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
 		return err
 	}
@@ -309,7 +309,7 @@ func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket, locatio
 
 // GetBucketInfo - fetch bucket metadata info.
 func (fs *FSObjects) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
-	bucketLock := fs.nsMutex.NewNSLock(bucket, "")
+	bucketLock := fs.nsMutex.NewNSLock(ctx, bucket, "")
 	if e := bucketLock.GetRLock(globalObjectTimeout); e != nil {
 		return bi, e
 	}
@@ -372,7 +372,7 @@ func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
 // DeleteBucket - delete a bucket and all the metadata associated
 // with the bucket including pending multipart, object metadata.
 func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string) error {
-	bucketLock := fs.nsMutex.NewNSLock(bucket, "")
+	bucketLock := fs.nsMutex.NewNSLock(ctx, bucket, "")
 	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
 		logger.LogIf(ctx, err)
 		return err
@@ -408,7 +408,7 @@ func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string) error {
 func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) {
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
 	if !cpSrcDstSame {
-		objectDWLock := fs.nsMutex.NewNSLock(dstBucket, dstObject)
+		objectDWLock := fs.nsMutex.NewNSLock(ctx, dstBucket, dstObject)
 		if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
 			return oi, err
 		}
@@ -431,8 +431,9 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 
 		// Save objects' metadata in `fs.json`.
 		fsMeta := newFSMetaV1()
-		if _, err = fsMeta.ReadFrom(ctx, wlk); err != nil && err != io.EOF {
-			return oi, toObjectErr(err, srcBucket, srcObject)
+		if _, err = fsMeta.ReadFrom(ctx, wlk); err != nil {
+			// For any error to read fsMeta, set default ETag and proceed.
+			fsMeta = fs.defaultFsJSON(srcObject)
 		}
 
 		fsMeta.Meta = srcInfo.UserDefined
@@ -479,7 +480,7 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 
 	if lockType != noLock {
 		// Lock the object before reading.
-		lock := fs.nsMutex.NewNSLock(bucket, object)
+		lock := fs.nsMutex.NewNSLock(ctx, bucket, object)
 		switch lockType {
 		case writeLock:
 			if err = lock.GetLock(globalObjectTimeout); err != nil {
@@ -566,7 +567,7 @@ func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offse
 	}
 
 	// Lock the object before reading.
-	objectLock := fs.nsMutex.NewNSLock(bucket, object)
+	objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
 	if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
 		logger.LogIf(ctx, err)
 		return err
@@ -710,10 +711,7 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
 		_, rerr := fsMeta.ReadFrom(ctx, rlk.LockedFile)
 		fs.rwPool.Close(fsMetaPath)
 		if rerr != nil {
-			if rerr != io.EOF {
-				return oi, rerr
-			}
-			// Set Default ETag, if fs.json is empty
+			// For any error to read fsMeta, set default ETag and proceed.
 			fsMeta = fs.defaultFsJSON(object)
 		}
 	}
@@ -741,7 +739,7 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
 // getObjectInfoWithLock - reads object metadata and replies back ObjectInfo.
 func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
 	// Lock the object before reading.
-	objectLock := fs.nsMutex.NewNSLock(bucket, object)
+	objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
 	if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
 		return oi, err
 	}
@@ -766,7 +764,7 @@ func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object s
 func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (oi ObjectInfo, e error) {
 	oi, err := fs.getObjectInfoWithLock(ctx, bucket, object)
 	if err == errCorruptedFormat || err == io.EOF {
-		objectLock := fs.nsMutex.NewNSLock(bucket, object)
+		objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
 		if err = objectLock.GetLock(globalObjectTimeout); err != nil {
 			return oi, toObjectErr(err, bucket, object)
 		}
@@ -812,7 +810,7 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
 		return ObjectInfo{}, err
 	}
 	// Lock the object.
-	objectLock := fs.nsMutex.NewNSLock(bucket, object)
+	objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
 	if err := objectLock.GetLock(globalObjectTimeout); err != nil {
 		logger.LogIf(ctx, err)
 		return objInfo, err
@@ -967,7 +965,7 @@ func (fs *FSObjects) DeleteObjects(ctx context.Context, bucket string, objects [
 // and there are no rollbacks supported.
 func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) error {
 	// Acquire a write lock before deleting the object.
-	objectLock := fs.nsMutex.NewNSLock(bucket, object)
+	objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
 	if err := objectLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}

cmd/gateway-common.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"strings"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
 
@@ -166,7 +167,7 @@ func FromMinioClientListMultipartsInfo(lmur minio.ListMultipartUploadsResult) Li
 // FromMinioClientObjectInfo converts minio ObjectInfo to gateway ObjectInfo
 func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
 	userDefined := FromMinioClientMetadata(oi.Metadata)
-	userDefined["Content-Type"] = oi.ContentType
+	userDefined[xhttp.ContentType] = oi.ContentType
 
 	return ObjectInfo{
 		Bucket:          bucket,
@@ -176,7 +177,7 @@ func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
 		ETag:            canonicalizeETag(oi.ETag),
 		UserDefined:     userDefined,
 		ContentType:     oi.ContentType,
-		ContentEncoding: oi.Metadata.Get("Content-Encoding"),
+		ContentEncoding: oi.Metadata.Get(xhttp.ContentEncoding),
 		StorageClass:    oi.StorageClass,
 		Expires:         oi.Expires,
 	}

cmd/gateway-main.go

@@ -48,7 +48,7 @@ var (
 
 // RegisterGatewayCommand registers a new command for gateway.
 func RegisterGatewayCommand(cmd cli.Command) error {
-	cmd.Flags = append(append(cmd.Flags, append(cmd.Flags, ServerFlags...)...), GlobalFlags...)
+	cmd.Flags = append(append(cmd.Flags, ServerFlags...), GlobalFlags...)
 	gatewayCmd.Subcommands = append(gatewayCmd.Subcommands, cmd)
 	return nil
 }
@@ -121,7 +121,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// to IPv6 address ie minio will start listening on IPv6 address whereas another
 	// (non-)minio process is listening on IPv4 of given port.
 	// To avoid this error situation we check for port availability.
-	logger.FatalIf(checkPortAvailability(globalMinioPort), "Unable to start the gateway")
+	logger.FatalIf(checkPortAvailability(globalMinioHost, globalMinioPort), "Unable to start the gateway")
 
 	// Check and load TLS certificates.
 	var err error
@@ -158,9 +158,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		registerSTSRouter(router)
 	}
 
-	// initialize globalTrace system
-	globalTrace = NewTraceSys(context.Background(), globalEndpoints)
-
 	enableConfigOps := globalEtcdClient != nil && gatewayName == "nas"
 	enableIAMOps := globalEtcdClient != nil
 
@@ -181,9 +178,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	// Currently only NAS and S3 gateway support encryption headers.
 	encryptionEnabled := gatewayName == "s3" || gatewayName == "nas"
+	allowSSEKMS := gatewayName == "s3" // Only S3 can support SSE-KMS (as pass-through)
 
 	// Add API router.
-	registerAPIRouter(router, encryptionEnabled)
+	registerAPIRouter(router, encryptionEnabled, allowSSEKMS)
 
 	var getCert certs.GetCertificateFunc
 	if globalTLSCerts != nil {
@@ -245,6 +243,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	// This is only to uniquely identify each gateway deployments.
 	globalDeploymentID = os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID")
+	logger.SetDeploymentID(globalDeploymentID)
 
 	var cacheConfig = globalServerConfig.GetCacheConfig()
 	if len(cacheConfig.Drives) > 0 {

cmd/gateway-main_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"fmt"
 	"strings"
 	"testing"
 
@@ -34,6 +35,36 @@ func TestRegisterGatewayCommand(t *testing.T) {
 	}
 }
 
+// Test running a registered gateway command with a flag
+func TestRunRegisteredGatewayCommand(t *testing.T) {
+	var err error
+
+	flagName := "test-flag"
+	flagValue := "foo"
+
+	cmd := cli.Command{
+		Name: "test-run-with-flag",
+		Flags: []cli.Flag{
+			cli.StringFlag{Name: flagName},
+		},
+		Action: func(ctx *cli.Context) {
+			if actual := ctx.String(flagName); actual != flagValue {
+				t.Errorf("value of %s expects %s, but got %s", flagName, flagValue, actual)
+			}
+		},
+	}
+
+	err = RegisterGatewayCommand(cmd)
+	if err != nil {
+		t.Errorf("RegisterGatewayCommand got unexpected error: %s", err)
+	}
+
+	if err = newApp("minio").Run(
+		[]string{"minio", "gateway", cmd.Name, fmt.Sprintf("--%s", flagName), flagValue}); err != nil {
+		t.Errorf("running registered gateway command got unexpected error: %s", err)
+	}
+}
+
 // Test parseGatewayEndpoint
 func TestParseGatewayEndpoint(t *testing.T) {
 	testCases := []struct {

cmd/gateway/s3/gateway-s3-sse.go

@@ -442,7 +442,8 @@ func (l *s3EncObjects) PutObject(ctx context.Context, bucket string, object stri
 	// Decide if sse options needed to be passed to backend
 	if opts.ServerSideEncryption != nil &&
 		((minio.GlobalGatewaySSE.SSEC() && opts.ServerSideEncryption.Type() == encrypt.SSEC) ||
-			(minio.GlobalGatewaySSE.SSES3() && opts.ServerSideEncryption.Type() == encrypt.S3)) {
+			(minio.GlobalGatewaySSE.SSES3() && opts.ServerSideEncryption.Type() == encrypt.S3) ||
+			opts.ServerSideEncryption.Type() == encrypt.KMS) {
 		sseOpts = opts.ServerSideEncryption
 	}
 	if opts.ServerSideEncryption == nil {

cmd/gateway/s3/gateway-s3.go

@@ -240,9 +240,6 @@ func newS3(urlStr string) (*miniogo.Core, error) {
 
 	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
 
-	// Set maximum retry operations.
-	miniogo.MaxRetry = 1
-
 	// Check if the provided keys are valid.
 	if _, err = clnt.BucketExists(probeBucketName); err != nil {
 		if miniogo.ToErrorResponse(err).Code != "AccessDenied" {

cmd/generic-handlers.go

@@ -28,6 +28,7 @@ import (
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/handlers"
@@ -261,10 +262,10 @@ func (h cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			if hasSuffix(r.URL.Path, ".js") || r.URL.Path == minioReservedBucketPath+"/favicon.ico" {
 				// For assets set cache expiry of one year. For each release, the name
 				// of the asset name will change and hence it can not be served from cache.
-				w.Header().Set("Cache-Control", "max-age=31536000")
+				w.Header().Set(xhttp.CacheControl, "max-age=31536000")
 			} else {
 				// For non asset requests we serve index.html which will never be cached.
-				w.Header().Set("Cache-Control", "no-store")
+				w.Header().Set(xhttp.CacheControl, "no-store")
 			}
 		}
 	}
@@ -294,7 +295,7 @@ func (h minioReservedBucketHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
 	default:
 		// For all other requests reject access to reserved
 		// buckets
-		bucketName, _ := urlPath2BucketObjectName(r.URL.Path)
+		bucketName, _ := request2BucketObjectName(r)
 		if isMinioReservedBucket(bucketName) || isMinioMetaBucket(bucketName) {
 			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrAllAccessDisabled), r.URL, guessIsBrowserReq(r))
 			return
@@ -380,15 +381,15 @@ type resourceHandler struct {
 // setCorsHandler handler for CORS (Cross Origin Resource Sharing)
 func setCorsHandler(h http.Handler) http.Handler {
 	commonS3Headers := []string{
-		"Date",
-		"ETag",
-		"Server",
-		"Connection",
-		"Accept-Ranges",
-		"Content-Range",
-		"Content-Encoding",
-		"Content-Length",
-		"Content-Type",
+		xhttp.Date,
+		xhttp.ETag,
+		xhttp.ServerInfo,
+		xhttp.Connection,
+		xhttp.AcceptRanges,
+		xhttp.ContentRange,
+		xhttp.ContentEncoding,
+		xhttp.ContentLength,
+		xhttp.ContentType,
 		"X-Amz*",
 		"x-amz*",
 		"*",
@@ -494,7 +495,7 @@ var notimplementedObjectResourceNames = map[string]bool{
 
 // Resource handler ServeHTTP() wrapper
 func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	bucketName, objectName := urlPath2BucketObjectName(r.URL.Path)
+	bucketName, objectName := request2BucketObjectName(r)
 
 	// If bucketName is present and not objectName check for bucket level resource queries.
 	if bucketName != "" && objectName == "" {
@@ -696,7 +697,8 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	bucket, object := urlPath2BucketObjectName(r.URL.Path)
+	bucket, object := request2BucketObjectName(r)
+
 	// ListBucket requests should be handled at current endpoint as
 	// all buckets data can be fetched from here.
 	if r.Method == http.MethodGet && bucket == "" && object == "" {
@@ -749,7 +751,7 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
 	return bucketForwardingHandler{fwd, h}
 }
 
-// customHeaderHandler sets x-amz-request-id, x-minio-deployment-id header.
+// customHeaderHandler sets x-amz-request-id header.
 // Previously, this value was set right before a response was sent to
 // the client. So, logger and Error response XML were not using this
 // value. This is set here so that this header can be logged as
@@ -763,12 +765,8 @@ func addCustomHeaders(h http.Handler) http.Handler {
 }
 
 func (s customHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	// Set custom headers such as x-amz-request-id and x-minio-deployment-id
-	// for each request.
-	w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
-	if globalDeploymentID != "" {
-		w.Header().Set(responseDeploymentIDKey, globalDeploymentID)
-	}
+	// Set custom headers such as x-amz-request-id for each request.
+	w.Header().Set(xhttp.AmzRequestID, mustGetRequestID(UTCNow()))
 	s.handler.ServeHTTP(logger.NewResponseWriter(w), r)
 }
 

cmd/globals.go

@@ -35,6 +35,7 @@ import (
 	"github.com/minio/minio/pkg/dns"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/iam/validator"
+	"github.com/minio/minio/pkg/pubsub"
 )
 
 // minio configuration related constants.
@@ -161,7 +162,7 @@ var (
 
 	// global Trace system to send HTTP request/response logs to
 	// registered listeners
-	globalTrace *HTTPTraceSys
+	globalHTTPTrace = pubsub.New()
 
 	globalEndpoints EndpointList
 

cmd/handler-utils.go

@@ -27,6 +27,7 @@ import (
 	"net/url"
 	"strings"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/handlers"
@@ -219,8 +220,8 @@ func extractReqParams(r *http.Request) map[string]string {
 func extractRespElements(w http.ResponseWriter) map[string]string {
 
 	return map[string]string{
-		"requestId":      w.Header().Get(responseRequestIDKey),
-		"content-length": w.Header().Get("Content-Length"),
+		"requestId":      w.Header().Get(xhttp.AmzRequestID),
+		"content-length": w.Header().Get(xhttp.ContentLength),
 		// Add more fields here.
 	}
 }
@@ -326,24 +327,24 @@ func extractPostPolicyFormValues(ctx context.Context, form *multipart.Form) (fil
 // Log headers and body.
 func httpTraceAll(f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		if !globalTrace.HasTraceListeners() {
+		if !globalHTTPTrace.HasSubscribers() {
 			f.ServeHTTP(w, r)
 			return
 		}
 		trace := Trace(f, true, w, r)
-		globalTrace.Publish(trace)
+		globalHTTPTrace.Publish(trace)
 	}
 }
 
 // Log only the headers.
 func httpTraceHdrs(f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		if !globalTrace.HasTraceListeners() {
+		if !globalHTTPTrace.HasSubscribers() {
 			f.ServeHTTP(w, r)
 			return
 		}
 		trace := Trace(f, false, w, r)
-		globalTrace.Publish(trace)
+		globalHTTPTrace.Publish(trace)
 	}
 }
 

cmd/http-tracer.go

@@ -118,12 +118,26 @@ func (r *recordResponseWriter) Body() []byte {
 	return r.body.Bytes()
 }
 
+// getOpName sanitizes the operation name for mc
+func getOpName(name string) (op string) {
+	op = strings.TrimPrefix(name, "github.com/minio/minio/cmd.")
+	op = strings.TrimSuffix(op, "Handler-fm")
+	op = strings.Replace(op, "objectAPIHandlers", "s3", 1)
+	op = strings.Replace(op, "webAPIHandlers", "s3", 1)
+	op = strings.Replace(op, "adminAPIHandlers", "admin", 1)
+	op = strings.Replace(op, "(*storageRESTServer)", "internal", 1)
+	op = strings.Replace(op, "(*peerRESTServer)", "internal", 1)
+	op = strings.Replace(op, "(*lockRESTServer)", "internal", 1)
+	op = strings.Replace(op, "stsAPIHandlers", "sts", 1)
+	op = strings.Replace(op, "LivenessCheckHandler", "healthcheck", 1)
+	op = strings.Replace(op, "ReadinessCheckHandler", "healthcheck", 1)
+	return op
+}
+
 // Trace gets trace of http request
 func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Request) trace.Info {
 
-	name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
-	name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.")
-	name = strings.TrimSuffix(name, "Handler-fm")
+	name := getOpName(runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name())
 
 	bodyPlaceHolder := []byte("<BODY>")
 	var reqBodyRecorder *recordRequest
@@ -136,7 +150,7 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 	if err == nil {
 		t.NodeName = host.Name
 	}
-	rq := trace.RequestInfo{Time: time.Now().UTC(), Method: r.Method, Path: r.URL.Path, RawQuery: r.URL.RawQuery}
+	rq := trace.RequestInfo{Time: time.Now().UTC(), Method: r.Method, Path: r.URL.Path, RawQuery: r.URL.RawQuery, Client: r.RemoteAddr}
 	rq.Headers = cloneHeader(r.Header)
 	rq.Headers.Set("Content-Length", strconv.Itoa(int(r.ContentLength)))
 	rq.Headers.Set("Host", r.Host)

cmd/http/headers.go

@@ -0,0 +1,81 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package http
+
+// Standard S3 HTTP response constants
+const (
+	LastModified       = "Last-Modified"
+	Date               = "Date"
+	ETag               = "ETag"
+	ContentType        = "Content-Type"
+	ContentMD5         = "Content-Md5"
+	ContentEncoding    = "Content-Encoding"
+	Expires            = "Expires"
+	ContentLength      = "Content-Length"
+	ContentLanguage    = "Content-Language"
+	ContentRange       = "Content-Range"
+	Connection         = "Connection"
+	AcceptRanges       = "Accept-Ranges"
+	AmzBucketRegion    = "X-Amz-Bucket-Region"
+	ServerInfo         = "Server"
+	RetryAfter         = "Retry-After"
+	Location           = "Location"
+	CacheControl       = "Cache-Control"
+	ContentDisposition = "Content-Disposition"
+	Authorization      = "Authorization"
+	Action             = "Action"
+)
+
+// Standard S3 HTTP request constants
+const (
+	IfModifiedSince   = "If-Modified-Since"
+	IfUnmodifiedSince = "If-Unmodified-Since"
+	IfMatch           = "If-Match"
+	IfNoneMatch       = "If-None-Match"
+
+	// S3 extensions
+	AmzCopySourceIfModifiedSince   = "x-amz-copy-source-if-modified-since"
+	AmzCopySourceIfUnmodifiedSince = "x-amz-copy-source-if-unmodified-since"
+
+	AmzCopySourceIfNoneMatch = "x-amz-copy-source-if-none-match"
+	AmzCopySourceIfMatch     = "x-amz-copy-source-if-match"
+
+	AmzCopySource          = "X-Amz-Copy-Source"
+	AmzCopySourceVersionID = "X-Amz-Copy-Source-Version-Id"
+	AmzCopySourceRange     = "X-Amz-Copy-Source-Range"
+
+	// Signature V4 related contants.
+	AmzContentSha256        = "X-Amz-Content-Sha256"
+	AmzDate                 = "X-Amz-Date"
+	AmzAlgorithm            = "X-Amz-Algorithm"
+	AmzExpires              = "X-Amz-Expires"
+	AmzSignedHeaders        = "X-Amz-SignedHeaders"
+	AmzSignature            = "X-Amz-Signature"
+	AmzCredential           = "X-Amz-Credential"
+	AmzSecurityToken        = "X-Amz-Security-Token"
+	AmzDecodedContentLength = "X-Amz-Decoded-Content-Length"
+
+	// Signature v2 related constants
+	AmzSignatureV2 = "Signature"
+	AmzAccessKeyID = "AWSAccessKeyId"
+
+	// Response request id.
+	AmzRequestID = "x-amz-request-id"
+
+	// Deployment id.
+	MinioDeploymentID = "x-minio-deployment-id"
+)

cmd/httptrace.go

@@ -1,115 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"strings"
-	"sync"
-
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/pubsub"
-	"github.com/minio/minio/pkg/trace"
-)
-
-//HTTPTraceSys holds global trace state
-type HTTPTraceSys struct {
-	peers  []*peerRESTClient
-	pubsub *pubsub.PubSub
-}
-
-// NewTraceSys - creates new HTTPTraceSys with all nodes subscribed to
-// the trace pub sub system
-func NewTraceSys(ctx context.Context, endpoints EndpointList) *HTTPTraceSys {
-	remoteHosts := getRemoteHosts(endpoints)
-	remoteClients, err := getRestClients(remoteHosts)
-	if err != nil {
-		logger.FatalIf(err, "Unable to start httptrace sub system")
-	}
-
-	ps := pubsub.New()
-	return &HTTPTraceSys{
-		remoteClients, ps,
-	}
-}
-
-// HasTraceListeners returns true if trace listeners are registered
-// for this node or peers
-func (sys *HTTPTraceSys) HasTraceListeners() bool {
-	return sys != nil && sys.pubsub.HasSubscribers()
-}
-
-// Publish - publishes trace message to the http trace pubsub system
-func (sys *HTTPTraceSys) Publish(traceMsg trace.Info) {
-	sys.pubsub.Publish(traceMsg)
-}
-
-// Trace writes http trace to writer
-func (sys *HTTPTraceSys) Trace(doneCh chan struct{}, trcAll bool) chan []byte {
-	traceCh := make(chan []byte)
-	go func() {
-		defer close(traceCh)
-
-		var wg = &sync.WaitGroup{}
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-
-			buf := &bytes.Buffer{}
-			ch := sys.pubsub.Subscribe()
-			defer sys.pubsub.Unsubscribe(ch)
-			for {
-				select {
-				case entry := <-ch:
-					trcInfo := entry.(trace.Info)
-					path := strings.TrimPrefix(trcInfo.ReqInfo.Path, "/")
-					// omit inter-node traffic if trcAll is false
-					if !trcAll && strings.HasPrefix(path, minioReservedBucket) {
-						continue
-					}
-					buf.Reset()
-					enc := json.NewEncoder(buf)
-					enc.SetEscapeHTML(false)
-					if err := enc.Encode(trcInfo); err != nil {
-						continue
-					}
-					traceCh <- buf.Bytes()
-				case <-doneCh:
-					return
-				}
-			}
-		}()
-
-		for _, peer := range sys.peers {
-			wg.Add(1)
-			go func(peer *peerRESTClient) {
-				defer wg.Done()
-				ch, err := peer.Trace(doneCh, trcAll)
-				if err != nil {
-					return
-				}
-				for entry := range ch {
-					traceCh <- entry
-				}
-			}(peer)
-		}
-		wg.Wait()
-	}()
-	return traceCh
-}

cmd/iam.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"path"
@@ -649,16 +650,87 @@ func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 	return cred, ok && cred.IsValid()
 }
 
-// IsAllowed - checks given policy args is allowed to continue the Rest API.
-func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
+// IsAllowedSTS is meant for STS based temporary credentials,
+// which implements claims validation and verification other than
+// applying policies.
+func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args) bool {
+	pname, ok := args.Claims[iampolicy.PolicyName]
+	if !ok {
+		// When claims are set, it should have a "policy" field.
+		return false
+	}
+	pnameStr, ok := pname.(string)
+	if !ok {
+		// When claims has "policy" field, it should be string.
+		return false
+	}
+
 	sys.RLock()
 	defer sys.RUnlock()
 
+	// If policy is available for given user, check the policy.
+	name, ok := sys.iamPolicyMap[args.AccountName]
+	if !ok {
+		// No policy available reject.
+		return false
+	}
+
+	if pnameStr != name {
+		// When claims has a policy, it should match the
+		// policy of args.AccountName which server remembers.
+		// if not reject such requests.
+		return false
+	}
+
+	// Now check if we have a sessionPolicy.
+	spolicy, ok := args.Claims[iampolicy.SessionPolicyName]
+	if !ok {
+		// Sub policy not set, this is most common since subPolicy
+		// is optional, use the top level policy only.
+		p, ok := sys.iamCannedPolicyMap[pnameStr]
+		return ok && p.IsAllowed(args)
+	}
+
+	spolicyStr, ok := spolicy.(string)
+	if !ok {
+		// Sub policy if set, should be a string reject
+		// malformed/malicious requests.
+		return false
+	}
+
+	// Check if policy is parseable.
+	subPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
+	if err != nil {
+		// Log any error in input session policy config.
+		logger.LogIf(context.Background(), err)
+		return false
+	}
+
+	// Policy without Version string value reject it.
+	if subPolicy.Version == "" {
+		return false
+	}
+
+	// Sub policy is set and valid.
+	p, ok := sys.iamCannedPolicyMap[pnameStr]
+	return ok && p.IsAllowed(args) && subPolicy.IsAllowed(args)
+}
+
+// IsAllowed - checks given policy args is allowed to continue the Rest API.
+func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
 	// If opa is configured, use OPA always.
 	if globalPolicyOPA != nil {
 		return globalPolicyOPA.IsAllowed(args)
 	}
 
+	// With claims set, we should do STS related checks and validation.
+	if len(args.Claims) > 0 {
+		return sys.IsAllowedSTS(args)
+	}
+
+	sys.RLock()
+	defer sys.RUnlock()
+
 	// If policy is available for given user, check the policy.
 	if name, found := sys.iamPolicyMap[args.AccountName]; found {
 		p, ok := sys.iamCannedPolicyMap[name]
@@ -763,7 +835,7 @@ func reloadPolicy(ctx context.Context, objectAPI ObjectLayer, prefix string,
 	if err = json.Unmarshal(pdata, &p); err != nil {
 		return err
 	}
-	cannedPolicyMap[path.Base(prefix)] = p
+	cannedPolicyMap[policyName] = p
 	return nil
 }
 
@@ -778,7 +850,8 @@ func reloadPolicies(objectAPI ObjectLayer, prefix string, cannedPolicyMap map[st
 		}
 		marker = lo.NextMarker
 		for _, prefix := range lo.Prefixes {
-			if err = reloadPolicy(context.Background(), objectAPI, "", prefix, cannedPolicyMap); err != nil {
+			if err = reloadPolicy(context.Background(), objectAPI, iamConfigPoliciesPrefix,
+				path.Base(prefix), cannedPolicyMap); err != nil {
 				return err
 			}
 		}

cmd/local-locker.go

@@ -21,7 +21,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/dsync"
+	"github.com/minio/dsync/v2"
 )
 
 // lockRequesterInfo stores various info from the client for each lock that is requested.

cmd/lock-rest-client.go

@@ -28,7 +28,7 @@ import (
 
 	"net/url"
 
-	"github.com/minio/dsync"
+	"github.com/minio/dsync/v2"
 	"github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"

cmd/lock-rest-client_test.go

@@ -19,7 +19,7 @@ package cmd
 import (
 	"testing"
 
-	"github.com/minio/dsync"
+	"github.com/minio/dsync/v2"
 	xnet "github.com/minio/minio/pkg/net"
 )
 

cmd/lock-rest-server.go

@@ -25,7 +25,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/dsync"
+	"github.com/minio/dsync/v2"
 	"github.com/minio/minio/cmd/logger"
 	xnet "github.com/minio/minio/pkg/net"
 )

cmd/logger/audit.go

@@ -63,6 +63,6 @@ func AuditLog(w http.ResponseWriter, r *http.Request, api string, reqClaims map[
 	}
 	// Send audit logs only to http targets.
 	for _, t := range AuditTargets {
-		_ = t.Send(audit.ToEntry(w, r, api, statusCode, reqClaims))
+		_ = t.Send(audit.ToEntry(w, r, api, statusCode, reqClaims, globalDeploymentID))
 	}
 }

cmd/logger/logger.go

@@ -55,6 +55,8 @@ const (
 
 var trimStrings []string
 
+var globalDeploymentID string
+
 // TimeFormat - logging time format.
 const TimeFormat string = "15:04:05 MST 01/02/2006"
 
@@ -154,6 +156,11 @@ func uniqueEntries(paths []string) []string {
 	return m.ToSlice()
 }
 
+// SetDeploymentID -- Deployment Id from the main package is set here
+func SetDeploymentID(deploymentID string) {
+	globalDeploymentID = deploymentID
+}
+
 // Init sets the trimStrings to possible GOPATHs
 // and GOROOT directories. Also append github.com/minio/minio
 // This is done to clean up the filename, when stack trace is
@@ -317,7 +324,9 @@ func logIf(ctx context.Context, err error) {
 
 	// Get the cause for the Error
 	message := err.Error()
-
+	if req.DeploymentID == "" {
+		req.DeploymentID = globalDeploymentID
+	}
 	entry := log.Entry{
 		DeploymentID: req.DeploymentID,
 		Level:        ErrorLvl.String(),

cmd/logger/message/audit/entry.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/pkg/handlers"
 )
 
@@ -50,7 +51,7 @@ type Entry struct {
 }
 
 // ToEntry - constructs an audit entry object.
-func ToEntry(w http.ResponseWriter, r *http.Request, api string, statusCode int, reqClaims map[string]interface{}) Entry {
+func ToEntry(w http.ResponseWriter, r *http.Request, api string, statusCode int, reqClaims map[string]interface{}, deploymentID string) Entry {
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]
@@ -67,13 +68,13 @@ func ToEntry(w http.ResponseWriter, r *http.Request, api string, statusCode int,
 	for k, v := range w.Header() {
 		respHeader[k] = strings.Join(v, ",")
 	}
-	respHeader["Etag"] = strings.Trim(respHeader["Etag"], `"`)
+	respHeader[xhttp.ETag] = strings.Trim(respHeader[xhttp.ETag], `"`)
 
 	entry := Entry{
 		Version:      Version,
-		DeploymentID: w.Header().Get("x-minio-deployment-id"),
+		DeploymentID: deploymentID,
 		RemoteHost:   handlers.GetSourceIP(r),
-		RequestID:    w.Header().Get("x-amz-request-id"),
+		RequestID:    w.Header().Get(xhttp.AmzRequestID),
 		UserAgent:    r.UserAgent(),
 		Time:         time.Now().UTC().Format(time.RFC3339Nano),
 		ReqQuery:     reqQuery,

cmd/logger/target/http/http.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"net/http"
 	gohttp "net/http"
 
 	xhttp "github.com/minio/minio/cmd/http"
@@ -49,11 +50,11 @@ func (h *Target) startHTTPLogger() {
 				continue
 			}
 
-			req, err := gohttp.NewRequest("POST", h.endpoint, bytes.NewBuffer(logJSON))
+			req, err := gohttp.NewRequest(http.MethodPost, h.endpoint, bytes.NewBuffer(logJSON))
 			if err != nil {
 				continue
 			}
-			req.Header.Set("Content-Type", "application/json")
+			req.Header.Set(xhttp.ContentType, "application/json")
 
 			resp, err := h.client.Do(req)
 			if err != nil {

cmd/metrics.go

@@ -34,11 +34,25 @@ var (
 		},
 		[]string{"request_type"},
 	)
+	minioVersionInfo = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: "minio",
+			Name:      "version_info",
+			Help:      "Version of current MinIO server instance",
+		},
+		[]string{
+			// current version
+			"version",
+			// commit-id of the current version
+			"commit",
+		},
+	)
 )
 
 func init() {
 	prometheus.MustRegister(httpRequestsDuration)
 	prometheus.MustRegister(newMinioCollector())
+	prometheus.MustRegister(minioVersionInfo)
 }
 
 // newMinioCollector describes the collector
@@ -64,6 +78,9 @@ func (c *minioCollector) Describe(ch chan<- *prometheus.Desc) {
 // Collect is called by the Prometheus registry when collecting metrics.
 func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 
+	// Expose MinIO's version information
+	minioVersionInfo.WithLabelValues(Version, CommitID).Add(1)
+
 	// Always expose network stats
 
 	// Network Sent/Received Bytes
@@ -184,7 +201,10 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 func metricsHandler() http.Handler {
 	registry := prometheus.NewRegistry()
 
-	err := registry.Register(httpRequestsDuration)
+	err := registry.Register(minioVersionInfo)
+	logger.LogIf(context.Background(), err)
+
+	err = registry.Register(httpRequestsDuration)
 	logger.LogIf(context.Background(), err)
 
 	err = registry.Register(newMinioCollector())

cmd/namespace-lock.go

@@ -27,7 +27,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/minio/dsync"
+	"github.com/minio/dsync/v2"
 	"github.com/minio/lsync"
 	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/logger"
@@ -127,7 +127,7 @@ type nsLockMap struct {
 }
 
 // Lock the namespace resource.
-func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock bool, timeout time.Duration) (locked bool) {
+func (n *nsLockMap) lock(ctx context.Context, volume, path string, lockSource, opsID string, readLock bool, timeout time.Duration) (locked bool) {
 	var nsLk *nsLock
 
 	n.lockMapMutex.Lock()
@@ -135,7 +135,7 @@ func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock
 	nsLk, found := n.lockMap[param]
 	if !found {
 		n.lockMap[param] = &nsLock{
-			LRWMutex: &lsync.LRWMutex{},
+			LRWMutex: lsync.NewLRWMutex(ctx),
 			ref:      1,
 		}
 		nsLk = n.lockMap[param]
@@ -199,7 +199,7 @@ func (n *nsLockMap) Lock(volume, path, opsID string, timeout time.Duration) (loc
 	readLock := false // This is a write lock.
 
 	lockSource := getSource() // Useful for debugging
-	return n.lock(volume, path, lockSource, opsID, readLock, timeout)
+	return n.lock(context.Background(), volume, path, lockSource, opsID, readLock, timeout)
 }
 
 // Unlock - unlocks any previously acquired write locks.
@@ -213,7 +213,7 @@ func (n *nsLockMap) RLock(volume, path, opsID string, timeout time.Duration) (lo
 	readLock := true
 
 	lockSource := getSource() // Useful for debugging
-	return n.lock(volume, path, lockSource, opsID, readLock, timeout)
+	return n.lock(context.Background(), volume, path, lockSource, opsID, readLock, timeout)
 }
 
 // RUnlock - unlocks any previously acquired read locks.
@@ -241,7 +241,7 @@ func (n *nsLockMap) ForceUnlock(volume, path string) {
 	//   are blocking can now proceed as normal and any new locks will also
 	//   participate normally.
 	if n.isDistXL { // For distributed mode, broadcast ForceUnlock message.
-		dsync.NewDRWMutex(pathJoin(volume, path), globalDsync).ForceUnlock()
+		dsync.NewDRWMutex(context.Background(), pathJoin(volume, path), globalDsync).ForceUnlock()
 	}
 
 	// Remove lock from the map.
@@ -291,6 +291,7 @@ func (di *distLockInstance) RUnlock() {
 
 // localLockInstance - frontend/top-level interface for namespace locks.
 type localLockInstance struct {
+	ctx                 context.Context
 	ns                  *nsLockMap
 	volume, path, opsID string
 }
@@ -298,12 +299,12 @@ type localLockInstance struct {
 // NewNSLock - returns a lock instance for a given volume and
 // path. The returned lockInstance object encapsulates the nsLockMap,
 // volume, path and operation ID.
-func (n *nsLockMap) NewNSLock(volume, path string) RWLocker {
+func (n *nsLockMap) NewNSLock(ctx context.Context, volume, path string) RWLocker {
 	opsID := mustGetUUID()
 	if n.isDistXL {
-		return &distLockInstance{dsync.NewDRWMutex(pathJoin(volume, path), globalDsync), volume, path, opsID}
+		return &distLockInstance{dsync.NewDRWMutex(ctx, pathJoin(volume, path), globalDsync), volume, path, opsID}
 	}
-	return &localLockInstance{n, volume, path, opsID}
+	return &localLockInstance{ctx, n, volume, path, opsID}
 }
 
 // Lock - block until write lock is taken or timeout has occurred.
@@ -311,7 +312,7 @@ func (li *localLockInstance) GetLock(timeout *dynamicTimeout) (timedOutErr error
 	lockSource := getSource()
 	start := UTCNow()
 	readLock := false
-	if !li.ns.lock(li.volume, li.path, lockSource, li.opsID, readLock, timeout.Timeout()) {
+	if !li.ns.lock(li.ctx, li.volume, li.path, lockSource, li.opsID, readLock, timeout.Timeout()) {
 		timeout.LogFailure()
 		return OperationTimedOut{Path: li.path}
 	}
@@ -330,7 +331,7 @@ func (li *localLockInstance) GetRLock(timeout *dynamicTimeout) (timedOutErr erro
 	lockSource := getSource()
 	start := UTCNow()
 	readLock := true
-	if !li.ns.lock(li.volume, li.path, lockSource, li.opsID, readLock, timeout.Timeout()) {
+	if !li.ns.lock(li.ctx, li.volume, li.path, lockSource, li.opsID, readLock, timeout.Timeout()) {
 		timeout.LogFailure()
 		return OperationTimedOut{Path: li.path}
 	}

cmd/namespace-lock_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"testing"
 	"time"
 )
@@ -31,7 +32,7 @@ func TestGetSource(t *testing.T) {
 	currentSource := func() string { return getSource() }
 	gotSource := currentSource()
 	// Hard coded line number, 32, in the "expectedSource" value
-	expectedSource := "[namespace-lock_test.go:32:TestGetSource()]"
+	expectedSource := "[namespace-lock_test.go:33:TestGetSource()]"
 	if gotSource != expectedSource {
 		t.Errorf("expected : %s, got : %s", expectedSource, gotSource)
 	}
@@ -204,7 +205,7 @@ func TestNamespaceForceUnlockTest(t *testing.T) {
 	isDistXL := false
 	initNSLock(isDistXL)
 	// Create lock.
-	lock := globalNSMutex.NewNSLock("bucket", "object")
+	lock := globalNSMutex.NewNSLock(context.Background(), "bucket", "object")
 	if lock.GetLock(newDynamicTimeout(60*time.Second, time.Second)) != nil {
 		t.Fatalf("Failed to get lock")
 	}
@@ -215,7 +216,7 @@ func TestNamespaceForceUnlockTest(t *testing.T) {
 
 	go func() {
 		// Try to claim lock again.
-		anotherLock := globalNSMutex.NewNSLock("bucket", "object")
+		anotherLock := globalNSMutex.NewNSLock(context.Background(), "bucket", "object")
 		if anotherLock.GetLock(newDynamicTimeout(60*time.Second, time.Second)) != nil {
 			t.Errorf("Failed to get lock")
 			return

cmd/net.go

@@ -190,10 +190,10 @@ func isHostIP(ipAddress string) bool {
 	return net.ParseIP(host) != nil
 }
 
-// checkPortAvailability - check if given port is already in use.
+// checkPortAvailability - check if given host and port is already in use.
 // Note: The check method tries to listen on given port and closes it.
 // It is possible to have a disconnected client in this tiny window of time.
-func checkPortAvailability(port string) (err error) {
+func checkPortAvailability(host, port string) (err error) {
 	// Return true if err is "address already in use" error.
 	isAddrInUseErr := func(err error) (b bool) {
 		if opErr, ok := err.(*net.OpError); ok {
@@ -209,7 +209,7 @@ func checkPortAvailability(port string) (err error) {
 
 	network := []string{"tcp", "tcp4", "tcp6"}
 	for _, n := range network {
-		l, err := net.Listen(n, net.JoinHostPort("", port))
+		l, err := net.Listen(n, net.JoinHostPort(host, port))
 		if err == nil {
 			// As we are able to listen on this network, the port is not in use.
 			// Close the listener and continue check other networks.

cmd/net_test.go

@@ -206,11 +206,13 @@ func TestCheckPortAvailability(t *testing.T) {
 	defer listener.Close()
 
 	testCases := []struct {
+		host        string
 		port        string
 		expectedErr error
 	}{
-		{port, fmt.Errorf("listen tcp :%v: bind: address already in use", port)},
-		{getFreePort(), nil},
+		{"", port, fmt.Errorf("listen tcp :%v: bind: address already in use", port)},
+		{"127.0.0.1", port, fmt.Errorf("listen tcp 127.0.0.1:%v: bind: address already in use", port)},
+		{"", getFreePort(), nil},
 	}
 
 	for _, testCase := range testCases {
@@ -219,7 +221,7 @@ func TestCheckPortAvailability(t *testing.T) {
 			continue
 		}
 
-		err := checkPortAvailability(testCase.port)
+		err := checkPortAvailability(testCase.host, testCase.port)
 		if testCase.expectedErr == nil {
 			if err != nil {
 				t.Fatalf("error: expected = <nil>, got = %v", err)

cmd/notification.go

@@ -34,6 +34,7 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/madmin"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/policy"
 )
@@ -230,6 +231,24 @@ func (sys *NotificationSys) LoadUsers() []NotificationPeerErr {
 	return ng.Wait()
 }
 
+// BackgroundHealStatus - returns background heal status of all peers
+func (sys *NotificationSys) BackgroundHealStatus() []madmin.BgHealState {
+	states := make([]madmin.BgHealState, len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		st, err := client.BackgroundHealStatus()
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+		} else {
+			states[idx] = st
+		}
+	}
+
+	return states
+}
+
 // StartProfiling - start profiling on remote peers, by initiating a remote RPC.
 func (sys *NotificationSys) StartProfiling(profiler string) []NotificationPeerErr {
 	ng := WithNPeers(len(sys.peerClients))
@@ -583,7 +602,7 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
 	// and configFile, take a transaction lock to avoid data race between readConfig()
 	// and saveConfig().
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
+	objLock := globalNSMutex.NewNSLock(ctx, minioMetaBucket, transactionConfigFile)
 	if err := objLock.GetRLock(globalOperationTimeout); err != nil {
 		return err
 	}
@@ -656,6 +675,9 @@ func (sys *NotificationSys) refresh(objAPI ObjectLayer) error {
 		ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{BucketName: bucket.Name})
 		config, err := readNotificationConfig(ctx, objAPI, bucket.Name)
 		if err != nil && err != errNoSuchNotifications {
+			if _, ok := err.(*event.ErrARNNotFound); ok {
+				continue
+			}
 			return err
 		}
 		if err == errNoSuchNotifications {
@@ -1036,7 +1058,7 @@ func SaveListener(objAPI ObjectLayer, bucketName string, eventNames []event.Name
 	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
 	// and configFile, take a transaction lock to avoid data race between readConfig()
 	// and saveConfig().
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
+	objLock := globalNSMutex.NewNSLock(ctx, minioMetaBucket, transactionConfigFile)
 	if err := objLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}
@@ -1087,7 +1109,7 @@ func RemoveListener(objAPI ObjectLayer, bucketName string, targetID event.Target
 	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
 	// and configFile, take a transaction lock to avoid data race between readConfig()
 	// and saveConfig().
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
+	objLock := globalNSMutex.NewNSLock(ctx, minioMetaBucket, transactionConfigFile)
 	if err := objLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}

cmd/object-api-common.go

@@ -21,6 +21,8 @@ import (
 	"path"
 	"sync"
 
+	"strings"
+
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/logger"
 )
@@ -234,7 +236,94 @@ func removeListenerConfig(ctx context.Context, objAPI ObjectLayer, bucket string
 	return objAPI.DeleteObject(ctx, minioMetaBucket, lcPath)
 }
 
+func listObjectsNonSlash(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, delimiter string, maxKeys int, tpool *TreeWalkPool, listDir ListDirFunc, getObjInfo func(context.Context, string, string) (ObjectInfo, error), getObjectInfoDirs ...func(context.Context, string, string) (ObjectInfo, error)) (loi ListObjectsInfo, err error) {
+	endWalkCh := make(chan struct{})
+	defer close(endWalkCh)
+	recursive := true
+	walkResultCh := startTreeWalk(ctx, bucket, prefix, "", recursive, listDir, endWalkCh)
+
+	var objInfos []ObjectInfo
+	var eof bool
+	var prevPrefix string
+
+	for {
+		if len(objInfos) == maxKeys {
+			break
+		}
+		result, ok := <-walkResultCh
+		if !ok {
+			eof = true
+			break
+		}
+
+		var objInfo ObjectInfo
+		var err error
+
+		index := strings.Index(strings.TrimPrefix(result.entry, prefix), delimiter)
+		if index == -1 {
+			objInfo, err = getObjInfo(ctx, bucket, result.entry)
+			if err != nil {
+				// Ignore errFileNotFound as the object might have got
+				// deleted in the interim period of listing and getObjectInfo(),
+				// ignore quorum error as it might be an entry from an outdated disk.
+				if IsErrIgnored(err, []error{
+					errFileNotFound,
+					errXLReadQuorum,
+				}...) {
+					continue
+				}
+				return loi, toObjectErr(err, bucket, prefix)
+			}
+		} else {
+			index = len(prefix) + index + len(delimiter)
+			currPrefix := result.entry[:index]
+			if currPrefix == prevPrefix {
+				continue
+			}
+			prevPrefix = currPrefix
+
+			objInfo = ObjectInfo{
+				Bucket: bucket,
+				Name:   currPrefix,
+				IsDir:  true,
+			}
+		}
+
+		if objInfo.Name <= marker {
+			continue
+		}
+
+		objInfos = append(objInfos, objInfo)
+		if result.end {
+			eof = true
+			break
+		}
+	}
+
+	result := ListObjectsInfo{}
+	for _, objInfo := range objInfos {
+		if objInfo.IsDir {
+			result.Prefixes = append(result.Prefixes, objInfo.Name)
+			continue
+		}
+		result.Objects = append(result.Objects, objInfo)
+	}
+
+	if !eof {
+		result.IsTruncated = true
+		if len(objInfos) > 0 {
+			result.NextMarker = objInfos[len(objInfos)-1].Name
+		}
+	}
+
+	return result, nil
+}
+
 func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, delimiter string, maxKeys int, tpool *TreeWalkPool, listDir ListDirFunc, getObjInfo func(context.Context, string, string) (ObjectInfo, error), getObjectInfoDirs ...func(context.Context, string, string) (ObjectInfo, error)) (loi ListObjectsInfo, err error) {
+	if delimiter != slashSeparator && delimiter != "" {
+		return listObjectsNonSlash(ctx, obj, bucket, prefix, marker, delimiter, maxKeys, tpool, listDir, getObjInfo, getObjectInfoDirs...)
+	}
+
 	if err := checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, obj); err != nil {
 		return loi, err
 	}

cmd/object-api-listobjects_test.go

@@ -471,10 +471,6 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 		{"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
 		{"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
 		{"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
-		// Valid, existing bucket, but sending invalid delimeter values (9-10).
-		// Empty string < "" > and forward slash < / > are the ony two valid arguments for delimeter.
-		{"test-bucket-list-object", "", "", "*", 0, ListObjectsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "*"), false},
-		{"test-bucket-list-object", "", "", "-", 0, ListObjectsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "-"), false},
 		// Testing for failure cases with both perfix and marker (11).
 		// The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix).
 		{"test-bucket-list-object", "asia", "europe-object", "", 0, ListObjectsInfo{}, fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false},

cmd/object-api-utils.go

@@ -36,6 +36,7 @@ import (
 	snappy "github.com/golang/snappy"
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/hash"
@@ -346,7 +347,7 @@ func isCompressible(header http.Header, object string) bool {
 // Eliminate the non-compressible objects.
 func excludeForCompression(header http.Header, object string) bool {
 	objStr := object
-	contentType := header.Get("Content-Type")
+	contentType := header.Get(xhttp.ContentType)
 	if globalIsCompressionEnabled {
 		// We strictly disable compression for standard extensions/content-types (`compressed`).
 		if hasStringSuffixInSlice(objStr, standardExcludeCompressExtensions) || hasPattern(standardExcludeCompressContentTypes, contentType) {
@@ -793,3 +794,18 @@ func (cr *snappyCompressReader) Read(p []byte) (int, error) {
 	}
 	return n, err
 }
+
+// Returns error if the cancelCh has been closed (indicating that S3 client has disconnected)
+type detectDisconnect struct {
+	io.ReadCloser
+	cancelCh <-chan struct{}
+}
+
+func (d *detectDisconnect) Read(p []byte) (int, error) {
+	select {
+	case <-d.cancelCh:
+		return 0, io.ErrUnexpectedEOF
+	default:
+		return d.ReadCloser.Read(p)
+	}
+}

cmd/object-handlers-common.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 )
@@ -49,7 +50,7 @@ func checkCopyObjectPartPreconditions(ctx context.Context, w http.ResponseWriter
 //  x-amz-copy-source-if-none-match
 func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo, encETag string) bool {
 	// Return false for methods other than GET and HEAD.
-	if r.Method != "PUT" {
+	if r.Method != http.MethodPut {
 		return false
 	}
 	if encETag == "" {
@@ -68,15 +69,15 @@ func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r
 		setCommonHeaders(w)
 
 		// set object-related metadata headers
-		w.Header().Set("Last-Modified", objInfo.ModTime.UTC().Format(http.TimeFormat))
+		w.Header().Set(xhttp.LastModified, objInfo.ModTime.UTC().Format(http.TimeFormat))
 
 		if objInfo.ETag != "" {
-			w.Header()["ETag"] = []string{"\"" + objInfo.ETag + "\""}
+			w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
 		}
 	}
 	// x-amz-copy-source-if-modified-since: Return the object only if it has been modified
 	// since the specified time otherwise return 412 (precondition failed).
-	ifModifiedSinceHeader := r.Header.Get("x-amz-copy-source-if-modified-since")
+	ifModifiedSinceHeader := r.Header.Get(xhttp.AmzCopySourceIfModifiedSince)
 	if ifModifiedSinceHeader != "" {
 		if givenTime, err := time.Parse(http.TimeFormat, ifModifiedSinceHeader); err == nil {
 			if !ifModifiedSince(objInfo.ModTime, givenTime) {
@@ -90,7 +91,7 @@ func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r
 
 	// x-amz-copy-source-if-unmodified-since : Return the object only if it has not been
 	// modified since the specified time, otherwise return a 412 (precondition failed).
-	ifUnmodifiedSinceHeader := r.Header.Get("x-amz-copy-source-if-unmodified-since")
+	ifUnmodifiedSinceHeader := r.Header.Get(xhttp.AmzCopySourceIfUnmodifiedSince)
 	if ifUnmodifiedSinceHeader != "" {
 		if givenTime, err := time.Parse(http.TimeFormat, ifUnmodifiedSinceHeader); err == nil {
 			if ifModifiedSince(objInfo.ModTime, givenTime) {
@@ -106,7 +107,7 @@ func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r
 
 	// x-amz-copy-source-if-match : Return the object only if its entity tag (ETag) is the
 	// same as the one specified; otherwise return a 412 (precondition failed).
-	ifMatchETagHeader := r.Header.Get("x-amz-copy-source-if-match")
+	ifMatchETagHeader := r.Header.Get(xhttp.AmzCopySourceIfMatch)
 	if ifMatchETagHeader != "" {
 		etag := objInfo.ETag
 		if shouldDecryptEtag {
@@ -122,7 +123,7 @@ func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r
 
 	// If-None-Match : Return the object only if its entity tag (ETag) is different from the
 	// one specified otherwise, return a 304 (not modified).
-	ifNoneMatchETagHeader := r.Header.Get("x-amz-copy-source-if-none-match")
+	ifNoneMatchETagHeader := r.Header.Get(xhttp.AmzCopySourceIfNoneMatch)
 	if ifNoneMatchETagHeader != "" {
 		etag := objInfo.ETag
 		if shouldDecryptEtag {
@@ -147,7 +148,7 @@ func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r
 //  If-None-Match
 func checkPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool {
 	// Return false for methods other than GET and HEAD.
-	if r.Method != "GET" && r.Method != "HEAD" {
+	if r.Method != http.MethodGet && r.Method != http.MethodHead {
 		return false
 	}
 	// If the object doesn't have a modtime (IsZero), or the modtime
@@ -163,15 +164,15 @@ func checkPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Requ
 		setCommonHeaders(w)
 
 		// set object-related metadata headers
-		w.Header().Set("Last-Modified", objInfo.ModTime.UTC().Format(http.TimeFormat))
+		w.Header().Set(xhttp.LastModified, objInfo.ModTime.UTC().Format(http.TimeFormat))
 
 		if objInfo.ETag != "" {
-			w.Header()["ETag"] = []string{"\"" + objInfo.ETag + "\""}
+			w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
 		}
 	}
 	// If-Modified-Since : Return the object only if it has been modified since the specified time,
 	// otherwise return a 304 (not modified).
-	ifModifiedSinceHeader := r.Header.Get("If-Modified-Since")
+	ifModifiedSinceHeader := r.Header.Get(xhttp.IfModifiedSince)
 	if ifModifiedSinceHeader != "" {
 		if givenTime, err := time.Parse(http.TimeFormat, ifModifiedSinceHeader); err == nil {
 			if !ifModifiedSince(objInfo.ModTime, givenTime) {
@@ -185,7 +186,7 @@ func checkPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Requ
 
 	// If-Unmodified-Since : Return the object only if it has not been modified since the specified
 	// time, otherwise return a 412 (precondition failed).
-	ifUnmodifiedSinceHeader := r.Header.Get("If-Unmodified-Since")
+	ifUnmodifiedSinceHeader := r.Header.Get(xhttp.IfUnmodifiedSince)
 	if ifUnmodifiedSinceHeader != "" {
 		if givenTime, err := time.Parse(http.TimeFormat, ifUnmodifiedSinceHeader); err == nil {
 			if ifModifiedSince(objInfo.ModTime, givenTime) {
@@ -199,7 +200,7 @@ func checkPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Requ
 
 	// If-Match : Return the object only if its entity tag (ETag) is the same as the one specified;
 	// otherwise return a 412 (precondition failed).
-	ifMatchETagHeader := r.Header.Get("If-Match")
+	ifMatchETagHeader := r.Header.Get(xhttp.IfMatch)
 	if ifMatchETagHeader != "" {
 		if !isETagEqual(objInfo.ETag, ifMatchETagHeader) {
 			// If the object ETag does not match with the specified ETag.
@@ -211,7 +212,7 @@ func checkPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Requ
 
 	// If-None-Match : Return the object only if its entity tag (ETag) is different from the
 	// one specified otherwise, return a 304 (not modified).
-	ifNoneMatchETagHeader := r.Header.Get("If-None-Match")
+	ifNoneMatchETagHeader := r.Header.Get(xhttp.IfNoneMatch)
 	if ifNoneMatchETagHeader != "" {
 		if isETagEqual(objInfo.ETag, ifNoneMatchETagHeader) {
 			// If the object ETag matches with the specified ETag.

cmd/object-handlers.go

@@ -36,6 +36,7 @@ import (
 	miniogo "github.com/minio/minio-go/v6"
 	"github.com/minio/minio-go/v6/pkg/encrypt"
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
@@ -50,12 +51,12 @@ import (
 
 // supportedHeadGetReqParams - supported request parameters for GET and HEAD presigned request.
 var supportedHeadGetReqParams = map[string]string{
-	"response-expires":             "Expires",
-	"response-content-type":        "Content-Type",
-	"response-cache-control":       "Cache-Control",
-	"response-content-encoding":    "Content-Encoding",
-	"response-content-language":    "Content-Language",
-	"response-content-disposition": "Content-Disposition",
+	"response-expires":             xhttp.Expires,
+	"response-content-type":        xhttp.ContentType,
+	"response-cache-control":       xhttp.CacheControl,
+	"response-content-encoding":    xhttp.ContentEncoding,
+	"response-content-language":    xhttp.ContentLanguage,
+	"response-content-disposition": xhttp.ContentDisposition,
 }
 
 const (
@@ -165,8 +166,8 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 				BucketName: bucket,
 				Key:        object,
 				Resource:   r.URL.Path,
-				RequestID:  w.Header().Get(responseRequestIDKey),
-				HostID:     w.Header().Get(responseDeploymentIDKey),
+				RequestID:  w.Header().Get(xhttp.AmzRequestID),
+				HostID:     globalDeploymentID,
 			})
 			writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML)
 		} else {
@@ -207,8 +208,8 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 				BucketName: bucket,
 				Key:        object,
 				Resource:   r.URL.Path,
-				RequestID:  w.Header().Get(responseRequestIDKey),
-				HostID:     w.Header().Get(responseDeploymentIDKey),
+				RequestID:  w.Header().Get(xhttp.AmzRequestID),
+				HostID:     globalDeploymentID,
 			})
 			writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML)
 		} else {
@@ -650,7 +651,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	// TODO: Reject requests where body/payload is present, for now we don't even read it.
 
 	// Read escaped copy source path to check for parameters.
-	cpSrcPath := r.Header.Get("X-Amz-Copy-Source")
+	cpSrcPath := r.Header.Get(xhttp.AmzCopySource)
 
 	// Check https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
 	// Regardless of whether you have enabled versioning, each object in your bucket
@@ -668,7 +669,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		// Note that url.Parse does the unescaping
 		cpSrcPath = u.Path
 	}
-	if vid := r.Header.Get("X-Amz-Copy-Source-Version-Id"); vid != "" {
+	if vid := r.Header.Get(xhttp.AmzCopySourceVersionID); vid != "" {
 		// Check if versionId header was added, if yes then check if
 		// its non "null" value, we should error out since we do not support
 		// any versions other than "null".
@@ -1052,7 +1053,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if crypto.S3KMS.IsRequested(r.Header) {
+	if crypto.S3KMS.IsRequested(r.Header) && !api.AllowSSEKMS() {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) // SSE-KMS is not supported
 		return
 	}
@@ -1064,6 +1065,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	bucket := vars["bucket"]
 	object := vars["object"]
 
+	// To detect if the client has disconnected.
+	r.Body = &detectDisconnect{r.Body, r.Context().Done()}
+
 	// X-Amz-Copy-Source shouldn't be set for this call.
 	if _, ok := r.Header["X-Amz-Copy-Source"]; ok {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopySource), r.URL, guessIsBrowserReq(r))
@@ -1178,7 +1182,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// This request header needs to be set prior to setting ObjectOptions
-	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
+	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) && !crypto.S3KMS.IsRequested(r.Header) {
 		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
 	}
 
@@ -1268,7 +1272,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	} else if hasServerSideEncryptionHeader(r.Header) {
 		etag = getDecryptedETag(r.Header, objInfo, false)
 	}
-	w.Header()["ETag"] = []string{"\"" + etag + "\""}
+	w.Header()[xhttp.ETag] = []string{"\"" + etag + "\""}
 
 	if objectAPI.IsEncryptionSupported() {
 		if crypto.IsEncrypted(objInfo.UserDefined) {
@@ -1315,7 +1319,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if crypto.S3KMS.IsRequested(r.Header) {
+	if crypto.S3KMS.IsRequested(r.Header) && !api.AllowSSEKMS() {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r)) // SSE-KMS is not supported
 		return
 	}
@@ -1333,7 +1337,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 	}
 
 	// This request header needs to be set prior to setting ObjectOptions
-	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
+	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) && !crypto.S3KMS.IsRequested(r.Header) {
 		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
 	}
 
@@ -1450,7 +1454,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	}
 
 	// Read escaped copy source path to check for parameters.
-	cpSrcPath := r.Header.Get("X-Amz-Copy-Source")
+	cpSrcPath := r.Header.Get(xhttp.AmzCopySource)
 
 	// Check https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
 	// Regardless of whether you have enabled versioning, each object in your bucket
@@ -1468,7 +1472,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		// Note that url.Parse does the unescaping
 		cpSrcPath = u.Path
 	}
-	if vid := r.Header.Get("X-Amz-Copy-Source-Version-Id"); vid != "" {
+	if vid := r.Header.Get(xhttp.AmzCopySourceVersionID); vid != "" {
 		// Check if X-Amz-Copy-Source-Version-Id header was added, if yes then check if
 		// its non "null" value, we should error out since we do not support
 		// any versions other than "null".
@@ -1537,7 +1541,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 
 	// Get request range.
 	var rs *HTTPRangeSpec
-	rangeHeader := r.Header.Get("x-amz-copy-source-range")
+	rangeHeader := r.Header.Get(xhttp.AmzCopySourceRange)
 	if rangeHeader != "" {
 		var parseRangeErr error
 		if rs, parseRangeErr = parseCopyPartRangeSpec(rangeHeader); parseRangeErr != nil {
@@ -1729,7 +1733,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	object := vars["object"]
 
 	// X-Amz-Copy-Source shouldn't be set for this call.
-	if _, ok := r.Header["X-Amz-Copy-Source"]; ok {
+	if _, ok := r.Header[xhttp.AmzCopySource]; ok {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopySource), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -1747,7 +1751,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	rAuthType := getRequestAuthType(r)
 	// For auth type streaming signature, we need to gather a different content length.
 	if rAuthType == authTypeStreamingSigned {
-		if sizeStr, ok := r.Header["X-Amz-Decoded-Content-Length"]; ok {
+		if sizeStr, ok := r.Header[xhttp.AmzDecodedContentLength]; ok {
 			if sizeStr[0] == "" {
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
 				return
@@ -1949,7 +1953,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	if isEncrypted {
 		etag = tryDecryptETag(objectEncryptionKey, partInfo.ETag, crypto.SSEC.IsRequested(r.Header))
 	}
-	w.Header()["ETag"] = []string{"\"" + etag + "\""}
+	w.Header()[xhttp.ETag] = []string{"\"" + etag + "\""}
 
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -2264,15 +2268,15 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
 			// Set retry-after header to indicate user-agents to retry request after 120secs.
 			// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
-			w.Header().Set("Retry-After", "120")
+			w.Header().Set(xhttp.RetryAfter, "120")
 		}
 
 		// Generate error response.
 		errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path,
-			w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
+			w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
 		encodedErrorResponse, _ := xml.Marshal(errorResponse)
 		setCommonHeaders(w)
-		w.Header().Set("Content-Type", string(mimeXML))
+		w.Header().Set(xhttp.ContentType, string(mimeXML))
 		w.Write(encodedErrorResponse)
 		w.(http.Flusher).Flush()
 	}
@@ -2307,7 +2311,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	}
 
 	// Set etag.
-	w.Header()["ETag"] = []string{"\"" + objInfo.ETag + "\""}
+	w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodedSuccessResponse)

cmd/peer-rest-client.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"bufio"
 	"bytes"
 	"context"
 	"crypto/tls"
@@ -31,8 +30,10 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/madmin"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/policy"
+	trace "github.com/minio/minio/pkg/trace"
 )
 
 // client to talk to peer Nodes.
@@ -212,7 +213,7 @@ func (client *peerRESTClient) ReloadFormat(dryRun bool) error {
 	return nil
 }
 
-// ListenBucketNotification - send listent bucket notification to peer nodes.
+// ListenBucketNotification - send listen bucket notification to peer nodes.
 func (client *peerRESTClient) ListenBucketNotification(bucket string, eventNames []event.Name,
 	pattern string, targetID event.TargetID, addr xnet.Host) error {
 	args := listenBucketNotificationReq{
@@ -422,52 +423,71 @@ func (client *peerRESTClient) SignalService(sig serviceSignal) error {
 	return nil
 }
 
-// Trace - send http trace request to peer nodes
-func (client *peerRESTClient) Trace(doneCh chan struct{}, trcAll bool) (chan []byte, error) {
-	ch := make(chan []byte)
+func (client *peerRESTClient) BackgroundHealStatus() (madmin.BgHealState, error) {
+	respBody, err := client.call(peerRESTMethodBackgroundHealStatus, nil, nil, -1)
+	if err != nil {
+		return madmin.BgHealState{}, err
+	}
+	defer http.DrainBody(respBody)
+
+	state := madmin.BgHealState{}
+	err = gob.NewDecoder(respBody).Decode(&state)
+	return state, err
+}
+
+func (client *peerRESTClient) doTrace(traceCh chan interface{}, doneCh chan struct{}, trcAll bool) {
+	values := make(url.Values)
+	values.Set(peerRESTTraceAll, strconv.FormatBool(trcAll))
+
+	// To cancel the REST request in case doneCh gets closed.
+	ctx, cancel := context.WithCancel(context.Background())
+
+	cancelCh := make(chan struct{})
+	defer close(cancelCh)
 	go func() {
-		cleanupFn := func(cancel context.CancelFunc, ch chan []byte, respBody io.ReadCloser) {
-			close(ch)
-			if cancel != nil {
-				cancel()
-			}
-			http.DrainBody(respBody)
+		select {
+		case <-doneCh:
+		case <-cancelCh:
+			// There was an error in the REST request.
 		}
+		cancel()
+	}()
+
+	respBody, err := client.callWithContext(ctx, peerRESTMethodTrace, values, nil, -1)
+	defer http.DrainBody(respBody)
+
+	if err != nil {
+		return
+	}
+
+	dec := gob.NewDecoder(respBody)
+	for {
+		var info trace.Info
+		if err = dec.Decode(&info); err != nil {
+			return
+		}
+		select {
+		case traceCh <- info:
+		default:
+			// Do not block on slow receivers.
+		}
+	}
+}
+
+// Trace - send http trace request to peer nodes
+func (client *peerRESTClient) Trace(traceCh chan interface{}, doneCh chan struct{}, trcAll bool) {
+	go func() {
 		for {
-			values := make(url.Values)
-			values.Set(peerRESTTraceAll, strconv.FormatBool(trcAll))
-			// get cancellation context to properly unsubscribe peers
-			ctx, cancel := context.WithCancel(context.Background())
-			respBody, err := client.callWithContext(ctx, peerRESTMethodTrace, values, nil, -1)
-			if err != nil {
-				//retry
-				time.Sleep(5 * time.Second)
-				select {
-				case <-doneCh:
-					cleanupFn(cancel, ch, respBody)
-					return
-				default:
-				}
-				continue
-			}
-			bio := bufio.NewScanner(respBody)
-			go func() {
-				<-doneCh
-				cancel()
-			}()
-			// Unmarshal each line, returns marshaled values.
-			for bio.Scan() {
-				ch <- bio.Bytes()
-			}
+			client.doTrace(traceCh, doneCh, trcAll)
 			select {
 			case <-doneCh:
-				cleanupFn(cancel, ch, respBody)
 				return
 			default:
+				// There was error in the REST request, retry after sometime as probably the peer is down.
+				time.Sleep(5 * time.Second)
 			}
 		}
 	}()
-	return ch, nil
 }
 
 func getRemoteHosts(endpoints EndpointList) []*xnet.Host {

cmd/peer-rest-common.go

@@ -26,6 +26,7 @@ const (
 	peerRESTMethodDrivePerfInfo            = "driveperfinfo"
 	peerRESTMethodDeleteBucket             = "deletebucket"
 	peerRESTMethodSignalService            = "signalservice"
+	peerRESTMethodBackgroundHealStatus     = "backgroundhealstatus"
 	peerRESTMethodGetLocks                 = "getlocks"
 	peerRESTMethodBucketPolicyRemove       = "removebucketpolicy"
 	peerRESTMethodLoadUser                 = "loaduser"

cmd/peer-rest-server.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"context"
 	"encoding/gob"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -29,6 +28,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
 	xnet "github.com/minio/minio/pkg/net"
@@ -676,39 +676,54 @@ func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	}
 	trcAll := r.URL.Query().Get(peerRESTTraceAll) == "true"
 
-	w.Header().Set("Connection", "close")
+	w.Header().Set(xhttp.Connection, "close")
 	w.WriteHeader(http.StatusOK)
 	w.(http.Flusher).Flush()
-	ch := globalTrace.pubsub.Subscribe()
-	defer globalTrace.pubsub.Unsubscribe(ch)
 
-	enc := json.NewEncoder(w)
-	enc.SetEscapeHTML(false)
+	filter := func(entry interface{}) bool {
+		if trcAll {
+			return true
+		}
+		trcInfo := entry.(trace.Info)
+		return !strings.HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath)
+	}
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	// Trace Publisher uses nonblocking publish and hence does not wait for slow subscribers.
+	// Use buffered channel to take care of burst sends or slow w.Write()
+	ch := make(chan interface{}, 2000)
+	globalHTTPTrace.Subscribe(ch, doneCh, filter)
+
+	enc := gob.NewEncoder(w)
 	for {
 		select {
 		case entry := <-ch:
-			trcInfo := entry.(trace.Info)
-			path := strings.TrimPrefix(trcInfo.ReqInfo.Path, "/")
-			// omit inter-node traffic if trcAll is false
-			if !trcAll && strings.HasPrefix(path, minioReservedBucket) {
-				continue
-			}
-
-			if err := enc.Encode(trcInfo); err != nil {
-				return
-			}
-
-			if _, err := w.Write([]byte("\n")); err != nil {
+			if err := enc.Encode(entry); err != nil {
 				return
 			}
 			w.(http.Flusher).Flush()
 		case <-r.Context().Done():
 			return
-
 		}
 	}
 }
 
+func (s *peerRESTServer) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "BackgroundHealStatus")
+
+	state := getLocalBackgroundHealStatus()
+
+	defer w.(http.Flusher).Flush()
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(state))
+}
+
 func (s *peerRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
 	w.WriteHeader(http.StatusForbidden)
 	w.Write([]byte(err.Error()))
@@ -755,6 +770,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodReloadFormat).HandlerFunc(httpTraceHdrs(server.ReloadFormatHandler)).Queries(restQueries(peerRESTDryRun)...)
 
 	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodTrace).HandlerFunc(server.TraceHandler)
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBackgroundHealStatus).HandlerFunc(server.BackgroundHealStatusHandler)
 
 	router.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
 }

cmd/policy.go

@@ -85,9 +85,6 @@ func (sys *PolicySys) Remove(bucketName string) {
 
 // IsAllowed - checks given policy args is allowed to continue the Rest API.
 func (sys *PolicySys) IsAllowed(args policy.Args) bool {
-	sys.RLock()
-	defer sys.RUnlock()
-
 	if globalIsGateway {
 		// When gateway is enabled, no cached value
 		// is used to validate bucket policies.
@@ -99,6 +96,9 @@ func (sys *PolicySys) IsAllowed(args policy.Args) bool {
 			}
 		}
 	} else {
+		sys.RLock()
+		defer sys.RUnlock()
+
 		// If policy is available for given bucket, check the policy.
 		if p, found := sys.bucketPolicyMap[args.BucketName]; found {
 			return p.IsAllowed(args)

cmd/routers.go

@@ -48,7 +48,7 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
 
 // List of some generic handlers which are applied for all incoming requests.
 var globalHandlers = []HandlerFunc{
-	// set x-amz-request-id, x-minio-deployment-id header.
+	// set x-amz-request-id header.
 	addCustomHeaders,
 	// set HTTP security headers such as Content-Security-Policy.
 	addSecurityHeaders,
@@ -119,8 +119,9 @@ func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
 		}
 	}
 
-	// Add API router, additionally all server mode support encryption.
-	registerAPIRouter(router, true)
+	// Add API router, additionally all server mode support encryption
+	// but don't allow SSE-KMS.
+	registerAPIRouter(router, true, false)
 
 	// Register rest of the handlers.
 	return registerHandlers(router, globalHandlers...), nil

cmd/server-main.go

@@ -27,7 +27,7 @@ import (
 	"syscall"
 
 	"github.com/minio/cli"
-	"github.com/minio/dsync"
+	"github.com/minio/dsync/v2"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/certs"
@@ -176,7 +176,7 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	// to IPv6 address ie minio will start listening on IPv6 address whereas another
 	// (non-)minio process is listening on IPv4 of given port.
 	// To avoid this error sutiation we check for port availability.
-	logger.FatalIf(checkPortAvailability(globalMinioPort), "Unable to start the server")
+	logger.FatalIf(checkPortAvailability(globalMinioHost, globalMinioPort), "Unable to start the server")
 
 	globalIsXL = (setupType == XLSetupType)
 	globalIsDistXL = (setupType == DistXLSetupType)
@@ -294,9 +294,6 @@ func serverMain(ctx *cli.Context) {
 		globalSweepHealState = initHealState()
 	}
 
-	// initialize globalTrace system
-	globalTrace = NewTraceSys(context.Background(), globalEndpoints)
-
 	// Configure server.
 	var handler http.Handler
 	handler, err = configureServerHandler(globalEndpoints)
@@ -317,6 +314,7 @@ func serverMain(ctx *cli.Context) {
 	}()
 
 	newObject, err := newObjectLayer(globalEndpoints)
+	logger.SetDeploymentID(globalDeploymentID)
 	if err != nil {
 		// Stop watching for any certificate changes.
 		globalTLSCerts.Stop()

cmd/server_test.go

@@ -34,6 +34,7 @@ import (
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -1202,7 +1203,7 @@ func (s *TestSuiteCommon) TestPutObject(c *check) {
 	c.Assert(err, nil)
 	c.Assert(response.StatusCode, http.StatusOK)
 	// The response Etag header should contain Md5sum of an empty string.
-	c.Assert(response.Header.Get("Etag"), "\""+emptyETag+"\"")
+	c.Assert(response.Header.Get(xhttp.ETag), "\""+emptyETag+"\"")
 }
 
 // TestListBuckets - Make request for listing of all buckets.
@@ -2728,5 +2729,5 @@ func (s *TestSuiteCommon) TestObjectMultipart(c *check) {
 		parts = append(parts, part)
 	}
 	etag := getCompleteMultipartMD5(parts)
-	c.Assert(canonicalizeETag(response.Header.Get("Etag")), etag)
+	c.Assert(canonicalizeETag(response.Header.Get(xhttp.ETag)), etag)
 }

cmd/signature-v2.go

@@ -28,6 +28,8 @@ import (
 	"strconv"
 	"strings"
 
+	xhttp "github.com/minio/minio/cmd/http"
+
 	"github.com/minio/minio/pkg/auth"
 )
 
@@ -68,13 +70,13 @@ var resourceList = []string{
 
 func doesPolicySignatureV2Match(formValues http.Header) APIErrorCode {
 	cred := globalServerConfig.GetCredential()
-	accessKey := formValues.Get("AWSAccessKeyId")
+	accessKey := formValues.Get(xhttp.AmzAccessKeyID)
 	cred, _, s3Err := checkKeyValid(accessKey)
 	if s3Err != ErrNone {
 		return s3Err
 	}
 	policy := formValues.Get("Policy")
-	signature := formValues.Get("Signature")
+	signature := formValues.Get(xhttp.AmzSignatureV2)
 	if !compareSignatureV2(signature, calculateSignatureV2(policy, cred.SecretKey)) {
 		return ErrSignatureDoesNotMatch
 	}
@@ -131,11 +133,11 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
 			return ErrInvalidQueryParams
 		}
 		switch keyval[0] {
-		case "AWSAccessKeyId":
+		case xhttp.AmzAccessKeyID:
 			accessKey = keyval[1]
-		case "Signature":
+		case xhttp.AmzSignatureV2:
 			gotSignature = keyval[1]
-		case "Expires":
+		case xhttp.Expires:
 			expires = keyval[1]
 		default:
 			filteredQueries = append(filteredQueries, query)
@@ -177,13 +179,13 @@ func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode {
 }
 
 func getReqAccessKeyV2(r *http.Request) (auth.Credentials, bool, APIErrorCode) {
-	if accessKey := r.URL.Query().Get("AWSAccessKeyId"); accessKey != "" {
+	if accessKey := r.URL.Query().Get(xhttp.AmzAccessKeyID); accessKey != "" {
 		return checkKeyValid(accessKey)
 	}
 
 	// below is V2 Signed Auth header format, splitting on `space` (after the `AWS` string).
 	// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature
-	authFields := strings.Split(r.Header.Get("Authorization"), " ")
+	authFields := strings.Split(r.Header.Get(xhttp.Authorization), " ")
 	if len(authFields) != 2 {
 		return auth.Credentials{}, false, ErrMissingFields
 	}
@@ -219,7 +221,7 @@ func getReqAccessKeyV2(r *http.Request) (auth.Credentials, bool, APIErrorCode) {
 
 func validateV2AuthHeader(r *http.Request) (auth.Credentials, APIErrorCode) {
 	var cred auth.Credentials
-	v2Auth := r.Header.Get("Authorization")
+	v2Auth := r.Header.Get(xhttp.Authorization)
 	if v2Auth == "" {
 		return cred, ErrAuthHeaderEmpty
 	}
@@ -238,7 +240,7 @@ func validateV2AuthHeader(r *http.Request) (auth.Credentials, APIErrorCode) {
 }
 
 func doesSignV2Match(r *http.Request) APIErrorCode {
-	v2Auth := r.Header.Get("Authorization")
+	v2Auth := r.Header.Get(xhttp.Authorization)
 	cred, apiError := validateV2AuthHeader(r)
 	if apiError != ErrNone {
 		return apiError
@@ -380,7 +382,7 @@ func getStringToSignV2(method string, encodedResource, encodedQuery string, head
 	date := expires // Date is set to expires date for presign operations.
 	if date == "" {
 		// If expires date is empty then request header Date is used.
-		date = headers.Get("Date")
+		date = headers.Get(xhttp.Date)
 	}
 
 	// From the Amazon docs:
@@ -393,8 +395,8 @@ func getStringToSignV2(method string, encodedResource, encodedQuery string, head
 	//	 CanonicalizedResource;
 	stringToSign := strings.Join([]string{
 		method,
-		headers.Get("Content-MD5"),
-		headers.Get("Content-Type"),
+		headers.Get(xhttp.ContentMD5),
+		headers.Get(xhttp.ContentType),
 		date,
 		canonicalHeaders,
 	}, "\n")

cmd/signature-v4-utils.go

@@ -26,6 +26,7 @@ import (
 	"strconv"
 	"strings"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/sha256-simd"
@@ -44,12 +45,12 @@ func skipContentSha256Cksum(r *http.Request) bool {
 	)
 
 	if isRequestPresignedSignatureV4(r) {
-		v, ok = r.URL.Query()["X-Amz-Content-Sha256"]
+		v, ok = r.URL.Query()[xhttp.AmzContentSha256]
 		if !ok {
-			v, ok = r.Header["X-Amz-Content-Sha256"]
+			v, ok = r.Header[xhttp.AmzContentSha256]
 		}
 	} else {
-		v, ok = r.Header["X-Amz-Content-Sha256"]
+		v, ok = r.Header[xhttp.AmzContentSha256]
 	}
 
 	// If x-amz-content-sha256 is set and the value is not
@@ -81,15 +82,15 @@ func getContentSha256Cksum(r *http.Request, stype serviceType) string {
 		// X-Amz-Content-Sha256, if not set in presigned requests, checksum
 		// will default to 'UNSIGNED-PAYLOAD'.
 		defaultSha256Cksum = unsignedPayload
-		v, ok = r.URL.Query()["X-Amz-Content-Sha256"]
+		v, ok = r.URL.Query()[xhttp.AmzContentSha256]
 		if !ok {
-			v, ok = r.Header["X-Amz-Content-Sha256"]
+			v, ok = r.Header[xhttp.AmzContentSha256]
 		}
 	} else {
 		// X-Amz-Content-Sha256, if not set in signed requests, checksum
 		// will default to sha256([]byte("")).
 		defaultSha256Cksum = emptySHA256
-		v, ok = r.Header["X-Amz-Content-Sha256"]
+		v, ok = r.Header[xhttp.AmzContentSha256]
 	}
 
 	// We found 'X-Amz-Content-Sha256' return the captured value.

cmd/signature-v4.go

@@ -36,6 +36,7 @@ import (
 	"time"
 
 	"github.com/minio/minio-go/v6/pkg/s3utils"
+	xhttp "github.com/minio/minio/cmd/http"
 	sha256 "github.com/minio/sha256-simd"
 )
 
@@ -172,7 +173,7 @@ func doesPolicySignatureV4Match(formValues http.Header) APIErrorCode {
 	region := globalServerConfig.GetRegion()
 
 	// Parse credential tag.
-	credHeader, err := parseCredentialHeader("Credential="+formValues.Get("X-Amz-Credential"), region, serviceS3)
+	credHeader, err := parseCredentialHeader("Credential="+formValues.Get(xhttp.AmzCredential), region, serviceS3)
 	if err != ErrNone {
 		return ErrMissingFields
 	}
@@ -189,7 +190,7 @@ func doesPolicySignatureV4Match(formValues http.Header) APIErrorCode {
 	newSignature := getSignature(signingKey, formValues.Get("Policy"))
 
 	// Verify signature.
-	if !compareSignatureV4(newSignature, formValues.Get("X-Amz-Signature")) {
+	if !compareSignatureV4(newSignature, formValues.Get(xhttp.AmzSignature)) {
 		return ErrSignatureDoesNotMatch
 	}
 
@@ -223,11 +224,11 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 
 	// Construct new query.
 	query := make(url.Values)
-	if req.URL.Query().Get("X-Amz-Content-Sha256") != "" {
-		query.Set("X-Amz-Content-Sha256", hashedPayload)
+	if req.URL.Query().Get(xhttp.AmzContentSha256) != "" {
+		query.Set(xhttp.AmzContentSha256, hashedPayload)
 	}
 
-	query.Set("X-Amz-Algorithm", signV4Algorithm)
+	query.Set(xhttp.AmzAlgorithm, signV4Algorithm)
 
 	// If the host which signed the request is slightly ahead in time (by less than globalMaxSkewTime) the
 	// request should still be allowed.
@@ -244,10 +245,10 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 	expireSeconds := int(pSignValues.Expires / time.Second)
 
 	// Construct the query.
-	query.Set("X-Amz-Date", t.Format(iso8601Format))
-	query.Set("X-Amz-Expires", strconv.Itoa(expireSeconds))
-	query.Set("X-Amz-SignedHeaders", getSignedHeaders(extractedSignedHeaders))
-	query.Set("X-Amz-Credential", cred.AccessKey+"/"+pSignValues.Credential.getScope())
+	query.Set(xhttp.AmzDate, t.Format(iso8601Format))
+	query.Set(xhttp.AmzExpires, strconv.Itoa(expireSeconds))
+	query.Set(xhttp.AmzSignedHeaders, getSignedHeaders(extractedSignedHeaders))
+	query.Set(xhttp.AmzCredential, cred.AccessKey+"/"+pSignValues.Credential.getScope())
 
 	// Save other headers available in the request parameters.
 	for k, v := range req.URL.Query() {
@@ -273,24 +274,24 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 	encodedQuery := query.Encode()
 
 	// Verify if date query is same.
-	if req.URL.Query().Get("X-Amz-Date") != query.Get("X-Amz-Date") {
+	if req.URL.Query().Get(xhttp.AmzDate) != query.Get(xhttp.AmzDate) {
 		return ErrSignatureDoesNotMatch
 	}
 	// Verify if expires query is same.
-	if req.URL.Query().Get("X-Amz-Expires") != query.Get("X-Amz-Expires") {
+	if req.URL.Query().Get(xhttp.AmzExpires) != query.Get(xhttp.AmzExpires) {
 		return ErrSignatureDoesNotMatch
 	}
 	// Verify if signed headers query is same.
-	if req.URL.Query().Get("X-Amz-SignedHeaders") != query.Get("X-Amz-SignedHeaders") {
+	if req.URL.Query().Get(xhttp.AmzSignedHeaders) != query.Get(xhttp.AmzSignedHeaders) {
 		return ErrSignatureDoesNotMatch
 	}
 	// Verify if credential query is same.
-	if req.URL.Query().Get("X-Amz-Credential") != query.Get("X-Amz-Credential") {
+	if req.URL.Query().Get(xhttp.AmzCredential) != query.Get(xhttp.AmzCredential) {
 		return ErrSignatureDoesNotMatch
 	}
 	// Verify if sha256 payload query is same.
-	if req.URL.Query().Get("X-Amz-Content-Sha256") != "" {
-		if req.URL.Query().Get("X-Amz-Content-Sha256") != query.Get("X-Amz-Content-Sha256") {
+	if req.URL.Query().Get(xhttp.AmzContentSha256) != "" {
+		if req.URL.Query().Get(xhttp.AmzContentSha256) != query.Get(xhttp.AmzContentSha256) {
 			return ErrContentSHA256Mismatch
 		}
 	}
@@ -311,7 +312,7 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 	newSignature := getSignature(presignedSigningKey, presignedStringToSign)
 
 	// Verify signature.
-	if !compareSignatureV4(req.URL.Query().Get("X-Amz-Signature"), newSignature) {
+	if !compareSignatureV4(req.URL.Query().Get(xhttp.AmzSignature), newSignature) {
 		return ErrSignatureDoesNotMatch
 	}
 	return ErrNone
@@ -325,7 +326,7 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string, st
 	req := *r
 
 	// Save authorization header.
-	v4Auth := req.Header.Get("Authorization")
+	v4Auth := req.Header.Get(xhttp.Authorization)
 
 	// Parse signature version '4' header.
 	signV4Values, err := parseSignV4(v4Auth, region, stype)
@@ -346,8 +347,8 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string, st
 
 	// Extract date, if not present throw error.
 	var date string
-	if date = req.Header.Get(http.CanonicalHeaderKey("x-amz-date")); date == "" {
-		if date = r.Header.Get("Date"); date == "" {
+	if date = req.Header.Get(xhttp.AmzDate); date == "" {
+		if date = r.Header.Get(xhttp.Date); date == "" {
 			return ErrMissingDateHeader
 		}
 	}

cmd/storage-rest-server.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 )
 
@@ -114,7 +115,7 @@ func (s *storageRESTServer) GetInstanceID(w http.ResponseWriter, r *http.Request
 		s.writeErrorResponse(w, err)
 		return
 	}
-	w.Header().Set("Content-Length", strconv.Itoa(len(s.instanceID)))
+	w.Header().Set(xhttp.ContentLength, strconv.Itoa(len(s.instanceID)))
 	w.Write([]byte(s.instanceID))
 	w.(http.Flusher).Flush()
 }
@@ -283,7 +284,7 @@ func (s *storageRESTServer) ReadAllHandler(w http.ResponseWriter, r *http.Reques
 		s.writeErrorResponse(w, err)
 		return
 	}
-	w.Header().Set("Content-Length", strconv.Itoa(len(buf)))
+	w.Header().Set(xhttp.ContentLength, strconv.Itoa(len(buf)))
 	w.Write(buf)
 	w.(http.Flusher).Flush()
 }
@@ -327,7 +328,7 @@ func (s *storageRESTServer) ReadFileHandler(w http.ResponseWriter, r *http.Reque
 		s.writeErrorResponse(w, err)
 		return
 	}
-	w.Header().Set("Content-Length", strconv.Itoa(len(buf)))
+	w.Header().Set(xhttp.ContentLength, strconv.Itoa(len(buf)))
 	w.Write(buf)
 	w.(http.Flusher).Flush()
 }
@@ -357,7 +358,7 @@ func (s *storageRESTServer) ReadFileStreamHandler(w http.ResponseWriter, r *http
 		return
 	}
 	defer rc.Close()
-	w.Header().Set("Content-Length", strconv.Itoa(length))
+	w.Header().Set(xhttp.ContentLength, strconv.Itoa(length))
 
 	io.Copy(w, rc)
 	w.(http.Flusher).Flush()

cmd/streaming-signature-v4.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/pkg/auth"
 	sha256 "github.com/minio/sha256-simd"
 )
@@ -69,7 +70,7 @@ func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature s
 	req := *r
 
 	// Save authorization header.
-	v4Auth := req.Header.Get("Authorization")
+	v4Auth := req.Header.Get(xhttp.Authorization)
 
 	// Parse signature version '4' header.
 	signV4Values, errCode := parseSignV4(v4Auth, globalServerConfig.GetRegion(), serviceS3)
@@ -81,7 +82,7 @@ func calculateSeedSignature(r *http.Request) (cred auth.Credentials, signature s
 	payload := streamingContentSHA256
 
 	// Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'
-	if payload != req.Header.Get("X-Amz-Content-Sha256") {
+	if payload != req.Header.Get(xhttp.AmzContentSha256) {
 		return cred, "", "", time.Time{}, ErrContentSHA256Mismatch
 	}
 

cmd/sts-errors.go

@@ -19,12 +19,14 @@ package cmd
 import (
 	"encoding/xml"
 	"net/http"
+
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
 // writeSTSErrorRespone writes error headers
 func writeSTSErrorResponse(w http.ResponseWriter, err STSError) {
 	// Generate error response.
-	stsErrorResponse := getSTSErrorResponse(err, w.Header().Get(responseRequestIDKey))
+	stsErrorResponse := getSTSErrorResponse(err, w.Header().Get(xhttp.AmzRequestID))
 	encodedErrorResponse := encodeResponse(stsErrorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }

cmd/sts-handlers.go

@@ -17,13 +17,17 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"encoding/base64"
 	"fmt"
 	"net/http"
 
 	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/iam/validator"
 	"github.com/minio/minio/pkg/wildcard"
 )
@@ -50,16 +54,16 @@ func registerSTSRouter(router *mux.Router) {
 	stsRouter := router.NewRoute().PathPrefix("/").Subrouter()
 
 	// Assume roles with no JWT, handles AssumeRole.
-	stsRouter.Methods("POST").MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
-		ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get("Content-Type"))
-		authOk := wildcard.MatchSimple("AWS4-HMAC-SHA256*", r.Header.Get("Authorization"))
+	stsRouter.Methods(http.MethodPost).MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
+		ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get(xhttp.ContentType))
+		authOk := wildcard.MatchSimple(signV4Algorithm+"*", r.Header.Get(xhttp.Authorization))
 		noQueries := len(r.URL.Query()) == 0
 		return ctypeOk && authOk && noQueries
 	}).HandlerFunc(httpTraceAll(sts.AssumeRole))
 
 	// Assume roles with JWT handler, handles both ClientGrants and WebIdentity.
 	stsRouter.Methods("POST").MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
-		ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get("Content-Type"))
+		ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get(xhttp.ContentType))
 		noQueries := len(r.URL.Query()) == 0
 		return ctypeOk && noQueries
 	}).HandlerFunc(httpTraceAll(sts.AssumeRoleWithJWT))
@@ -124,11 +128,6 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if r.Form.Get("Policy") != "" {
-		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
-		return
-	}
-
 	if r.Form.Get("Version") != stsAPIVersion {
 		logger.LogIf(ctx, fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get("Version"), stsAPIVersion))
 		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSMissingParameter))
@@ -147,6 +146,29 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	ctx = newContext(r, w, action)
 	defer logger.AuditLog(w, r, action, nil)
 
+	sessionPolicyStr := r.Form.Get("Policy")
+	// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html
+	// The plain text that you use for both inline and managed session
+	// policies shouldn't exceed 2048 characters.
+	if len(sessionPolicyStr) > 2048 {
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+		return
+	}
+
+	if len(sessionPolicyStr) > 0 {
+		sessionPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
+		if err != nil {
+			writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+			return
+		}
+
+		// Version in policy must not be empty
+		if sessionPolicy.Version == "" {
+			writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+			return
+		}
+	}
+
 	var err error
 	m := make(map[string]interface{})
 	m["exp"], err = validator.GetDefaultExpiration(r.Form.Get("DurationSeconds"))
@@ -171,7 +193,11 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 	// This policy is the policy associated with the user
 	// requesting for temporary credentials. The temporary
 	// credentials will inherit the same policy requirements.
-	m["policy"] = policyName
+	m[iampolicy.PolicyName] = policyName
+
+	if len(sessionPolicyStr) > 0 {
+		m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicyStr))
+	}
 
 	secret := globalServerConfig.GetCredential().SecretKey
 	cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
@@ -202,7 +228,7 @@ func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) {
 		},
 	}
 
-	assumeRoleResponse.ResponseMetadata.RequestID = w.Header().Get(responseRequestIDKey)
+	assumeRoleResponse.ResponseMetadata.RequestID = w.Header().Get(xhttp.AmzRequestID)
 	writeSuccessResponseXML(w, encodeResponse(assumeRoleResponse))
 }
 
@@ -216,11 +242,6 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	if r.Form.Get("Policy") != "" {
-		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
-		return
-	}
-
 	if r.Form.Get("Version") != stsAPIVersion {
 		logger.LogIf(ctx, fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get("Version"), stsAPIVersion))
 		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSMissingParameter))
@@ -276,6 +297,29 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	sessionPolicyStr := r.Form.Get("Policy")
+	// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html
+	// The plain text that you use for both inline and managed session
+	// policies shouldn't exceed 2048 characters.
+	if len(sessionPolicyStr) > 2048 {
+		writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+		return
+	}
+
+	if len(sessionPolicyStr) > 0 {
+		sessionPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
+		if err != nil {
+			writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+			return
+		}
+
+		// Version in policy must not be empty
+		if sessionPolicy.Version == "" {
+			writeSTSErrorResponse(w, stsErrCodes.ToSTSErr(ErrSTSInvalidParameterValue))
+			return
+		}
+	}
+
 	secret := globalServerConfig.GetCredential().SecretKey
 	cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
 	if err != nil {
@@ -289,7 +333,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
 	// be set and configured on your identity provider as part of
 	// JWT custom claims.
 	var policyName string
-	if v, ok := m["policy"]; ok {
+	if v, ok := m[iampolicy.PolicyName]; ok {
 		policyName, _ = v.(string)
 	}
 
@@ -298,6 +342,10 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
 		subFromToken, _ = v.(string)
 	}
 
+	if len(sessionPolicyStr) > 0 {
+		m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString([]byte(sessionPolicyStr))
+	}
+
 	// Set the newly generated credentials.
 	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred, policyName); err != nil {
 		logger.LogIf(ctx, err)
@@ -322,7 +370,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
 				SubjectFromToken: subFromToken,
 			},
 		}
-		clientGrantsResponse.ResponseMetadata.RequestID = w.Header().Get(responseRequestIDKey)
+		clientGrantsResponse.ResponseMetadata.RequestID = w.Header().Get(xhttp.AmzRequestID)
 		encodedSuccessResponse = encodeResponse(clientGrantsResponse)
 	case webIdentity:
 		webIdentityResponse := &AssumeRoleWithWebIdentityResponse{
@@ -331,7 +379,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Requ
 				SubjectFromWebIdentityToken: subFromToken,
 			},
 		}
-		webIdentityResponse.ResponseMetadata.RequestID = w.Header().Get(responseRequestIDKey)
+		webIdentityResponse.ResponseMetadata.RequestID = w.Header().Get(xhttp.AmzRequestID)
 		encodedSuccessResponse = encodeResponse(webIdentityResponse)
 	}
 

cmd/test-utils_test.go

@@ -2135,7 +2135,7 @@ func registerBucketLevelFunc(bucket *mux.Router, api objectAPIHandlers, apiFunct
 func registerAPIFunctions(muxRouter *mux.Router, objLayer ObjectLayer, apiFunctions ...string) {
 	if len(apiFunctions) == 0 {
 		// Register all api endpoints by default.
-		registerAPIRouter(muxRouter, true)
+		registerAPIRouter(muxRouter, true, false)
 		return
 	}
 	// API Router.
@@ -2176,7 +2176,7 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand
 		registerAPIFunctions(muxRouter, objLayer, apiFunctions...)
 		return muxRouter
 	}
-	registerAPIRouter(muxRouter, true)
+	registerAPIRouter(muxRouter, true, false)
 	return muxRouter
 }
 

cmd/utils.go

@@ -36,6 +36,7 @@ import (
 	"strings"
 	"time"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/handlers"
 
@@ -71,8 +72,20 @@ func cloneHeader(h http.Header) http.Header {
 	return h2
 }
 
+func request2BucketObjectName(r *http.Request) (bucketName, objectName string) {
+	path, err := getResource(r.URL.Path, r.Host, globalDomainNames)
+	if err != nil {
+		logger.CriticalIf(context.Background(), err)
+	}
+	return urlPath2BucketObjectName(path)
+}
+
 // Convert url path into bucket and object name.
 func urlPath2BucketObjectName(path string) (bucketName, objectName string) {
+	if path == "" || path == slashSeparator {
+		return "", ""
+	}
+
 	// Trim any preceding slash separator.
 	urlPath := strings.TrimPrefix(path, slashSeparator)
 
@@ -413,15 +426,15 @@ func newContext(r *http.Request, w http.ResponseWriter, api string) context.Cont
 		object = prefix
 	}
 	reqInfo := &logger.ReqInfo{
-		DeploymentID: w.Header().Get(responseDeploymentIDKey),
-		RequestID:    w.Header().Get(responseRequestIDKey),
+		DeploymentID: globalDeploymentID,
+		RequestID:    w.Header().Get(xhttp.AmzRequestID),
 		RemoteHost:   handlers.GetSourceIP(r),
 		UserAgent:    r.UserAgent(),
 		API:          api,
 		BucketName:   bucket,
 		ObjectName:   object,
 	}
-	return logger.SetReqInfo(context.Background(), reqInfo)
+	return logger.SetReqInfo(r.Context(), reqInfo)
 }
 
 // isNetworkOrHostDown - if there was a network error or if the host is down.

cmd/web-handlers.go

@@ -41,6 +41,7 @@ import (
 	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/browser"
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/dns"
@@ -401,11 +402,9 @@ type ListObjectsArgs struct {
 
 // ListObjectsRep - list objects response.
 type ListObjectsRep struct {
-	Objects     []WebObjectInfo `json:"objects"`
-	NextMarker  string          `json:"nextmarker"`
-	IsTruncated bool            `json:"istruncated"`
-	Writable    bool            `json:"writable"` // Used by client to show "upload file" button.
-	UIVersion   string          `json:"uiVersion"`
+	Objects   []WebObjectInfo `json:"objects"`
+	Writable  bool            `json:"writable"` // Used by client to show "upload file" button.
+	UIVersion string          `json:"uiVersion"`
 }
 
 // WebObjectInfo container for list objects metadata.
@@ -448,26 +447,36 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r
 		if err != nil {
 			return toJSONError(ctx, err, args.BucketName)
 		}
-		result, err := core.ListObjects(args.BucketName, args.Prefix, args.Marker, slashSeparator, 1000)
-		if err != nil {
-			return toJSONError(ctx, err, args.BucketName)
+
+		nextMarker := ""
+		// Fetch all the objects
+		for {
+			result, err := core.ListObjects(args.BucketName, args.Prefix, nextMarker, slashSeparator, 1000)
+			if err != nil {
+				return toJSONError(ctx, err, args.BucketName)
+			}
+
+			for _, obj := range result.Contents {
+				reply.Objects = append(reply.Objects, WebObjectInfo{
+					Key:          obj.Key,
+					LastModified: obj.LastModified,
+					Size:         obj.Size,
+					ContentType:  obj.ContentType,
+				})
+			}
+			for _, p := range result.CommonPrefixes {
+				reply.Objects = append(reply.Objects, WebObjectInfo{
+					Key: p.Prefix,
+				})
+			}
+
+			nextMarker = result.NextMarker
+
+			// Return when there are no more objects
+			if !result.IsTruncated {
+				return nil
+			}
 		}
-		reply.NextMarker = result.NextMarker
-		reply.IsTruncated = result.IsTruncated
-		for _, obj := range result.Contents {
-			reply.Objects = append(reply.Objects, WebObjectInfo{
-				Key:          obj.Key,
-				LastModified: obj.LastModified,
-				Size:         obj.Size,
-				ContentType:  obj.ContentType,
-			})
-		}
-		for _, p := range result.CommonPrefixes {
-			reply.Objects = append(reply.Objects, WebObjectInfo{
-				Key: p.Prefix,
-			})
-		}
-		return nil
 	}
 
 	claims, owner, authErr := webRequestAuthenticate(r)
@@ -551,35 +560,43 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r
 		return toJSONError(ctx, errInvalidBucketName)
 	}
 
-	lo, err := listObjects(ctx, args.BucketName, args.Prefix, args.Marker, slashSeparator, 1000)
-	if err != nil {
-		return &json2.Error{Message: err.Error()}
-	}
-	for i := range lo.Objects {
-		if crypto.IsEncrypted(lo.Objects[i].UserDefined) {
-			lo.Objects[i].Size, err = lo.Objects[i].DecryptedSize()
-			if err != nil {
-				return toJSONError(ctx, err)
+	nextMarker := ""
+	// Fetch all the objects
+	for {
+		lo, err := listObjects(ctx, args.BucketName, args.Prefix, nextMarker, slashSeparator, 1000)
+		if err != nil {
+			return &json2.Error{Message: err.Error()}
+		}
+		for i := range lo.Objects {
+			if crypto.IsEncrypted(lo.Objects[i].UserDefined) {
+				lo.Objects[i].Size, err = lo.Objects[i].DecryptedSize()
+				if err != nil {
+					return toJSONError(ctx, err)
+				}
 			}
 		}
-	}
-	reply.NextMarker = lo.NextMarker
-	reply.IsTruncated = lo.IsTruncated
-	for _, obj := range lo.Objects {
-		reply.Objects = append(reply.Objects, WebObjectInfo{
-			Key:          obj.Name,
-			LastModified: obj.ModTime,
-			Size:         obj.Size,
-			ContentType:  obj.ContentType,
-		})
-	}
-	for _, prefix := range lo.Prefixes {
-		reply.Objects = append(reply.Objects, WebObjectInfo{
-			Key: prefix,
-		})
-	}
 
-	return nil
+		for _, obj := range lo.Objects {
+			reply.Objects = append(reply.Objects, WebObjectInfo{
+				Key:          obj.Name,
+				LastModified: obj.ModTime,
+				Size:         obj.Size,
+				ContentType:  obj.ContentType,
+			})
+		}
+		for _, prefix := range lo.Prefixes {
+			reply.Objects = append(reply.Objects, WebObjectInfo{
+				Key: prefix,
+			})
+		}
+
+		nextMarker = lo.NextMarker
+
+		// Return when there are no more objects
+		if !lo.IsTruncated {
+			return nil
+		}
+	}
 }
 
 // RemoveObjectArgs - args to remove an object, JSON will look like.
@@ -1201,7 +1218,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Add content disposition.
-	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", path.Base(objInfo.Name)))
+	w.Header().Set(xhttp.ContentDisposition, fmt.Sprintf("attachment; filename=\"%s\"", path.Base(objInfo.Name)))
 
 	setHeadGetRespHeaders(w, r.URL.Query())
 
@@ -1863,11 +1880,11 @@ func presignedGet(host, bucket, object string, expiry int64, creds auth.Credenti
 	}
 
 	query := url.Values{}
-	query.Set("X-Amz-Algorithm", signV4Algorithm)
-	query.Set("X-Amz-Credential", credential)
-	query.Set("X-Amz-Date", dateStr)
-	query.Set("X-Amz-Expires", expiryStr)
-	query.Set("X-Amz-SignedHeaders", "host")
+	query.Set(xhttp.AmzAlgorithm, signV4Algorithm)
+	query.Set(xhttp.AmzCredential, credential)
+	query.Set(xhttp.AmzDate, dateStr)
+	query.Set(xhttp.AmzExpires, expiryStr)
+	query.Set(xhttp.AmzSignedHeaders, "host")
 	queryStr := s3utils.QueryEncode(query)
 
 	path := "/" + path.Join(bucket, object)
@@ -1881,7 +1898,7 @@ func presignedGet(host, bucket, object string, expiry int64, creds auth.Credenti
 	signature := getSignature(signingKey, stringToSign)
 
 	// Construct the final presigned URL.
-	return host + s3utils.EncodePath(path) + "?" + queryStr + "&" + "X-Amz-Signature=" + signature
+	return host + s3utils.EncodePath(path) + "?" + queryStr + "&" + xhttp.AmzSignature + "=" + signature
 }
 
 // toJSONError converts regular errors into more user friendly

cmd/xl-sets.go

@@ -23,6 +23,7 @@ import (
 	"io"
 	"net/http"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -686,7 +687,7 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke
 	}
 
 	if !cpSrcDstSame {
-		objectDWLock := destSet.nsMutex.NewNSLock(destBucket, destObject)
+		objectDWLock := destSet.nsMutex.NewNSLock(ctx, destBucket, destObject)
 		if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
 			return objInfo, err
 		}
@@ -907,10 +908,108 @@ func (s *xlSets) startMergeWalks(ctx context.Context, bucket, prefix, marker str
 	return entryChs
 }
 
+func (s *xlSets) listObjectsNonSlash(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
+	endWalkCh := make(chan struct{})
+	defer close(endWalkCh)
+	recursive := true
+	entryChs := s.startMergeWalks(context.Background(), bucket, prefix, "", recursive, endWalkCh)
+
+	readQuorum := s.drivesPerSet / 2
+	var objInfos []ObjectInfo
+	var eof bool
+	var prevPrefix string
+
+	for {
+		if len(objInfos) == maxKeys {
+			break
+		}
+		result, ok := leastEntry(entryChs, readQuorum)
+		if !ok {
+			eof = true
+			break
+		}
+
+		var objInfo ObjectInfo
+
+		index := strings.Index(strings.TrimPrefix(result.Name, prefix), delimiter)
+		if index == -1 {
+			objInfo = ObjectInfo{
+				IsDir:           false,
+				Bucket:          bucket,
+				Name:            result.Name,
+				ModTime:         result.ModTime,
+				Size:            result.Size,
+				ContentType:     result.Metadata["content-type"],
+				ContentEncoding: result.Metadata["content-encoding"],
+			}
+
+			// Extract etag from metadata.
+			objInfo.ETag = extractETag(result.Metadata)
+
+			// All the parts per object.
+			objInfo.Parts = result.Parts
+
+			// etag/md5Sum has already been extracted. We need to
+			// remove to avoid it from appearing as part of
+			// response headers. e.g, X-Minio-* or X-Amz-*.
+			objInfo.UserDefined = cleanMetadata(result.Metadata)
+
+			// Update storage class
+			if sc, ok := result.Metadata[amzStorageClass]; ok {
+				objInfo.StorageClass = sc
+			} else {
+				objInfo.StorageClass = globalMinioDefaultStorageClass
+			}
+		} else {
+			index = len(prefix) + index + len(delimiter)
+			currPrefix := result.Name[:index]
+			if currPrefix == prevPrefix {
+				continue
+			}
+			prevPrefix = currPrefix
+
+			objInfo = ObjectInfo{
+				Bucket: bucket,
+				Name:   currPrefix,
+				IsDir:  true,
+			}
+		}
+
+		if objInfo.Name <= marker {
+			continue
+		}
+
+		objInfos = append(objInfos, objInfo)
+	}
+
+	result := ListObjectsInfo{}
+	for _, objInfo := range objInfos {
+		if objInfo.IsDir {
+			result.Prefixes = append(result.Prefixes, objInfo.Name)
+			continue
+		}
+		result.Objects = append(result.Objects, objInfo)
+	}
+
+	if !eof {
+		result.IsTruncated = true
+		if len(objInfos) > 0 {
+			result.NextMarker = objInfos[len(objInfos)-1].Name
+		}
+	}
+
+	return result, nil
+}
+
 // ListObjects - implements listing of objects across disks, each disk is indepenently
 // walked and merged at this layer. Resulting value through the merge process sends
 // the data in lexically sorted order.
 func (s *xlSets) listObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int, heal bool) (loi ListObjectsInfo, err error) {
+	if delimiter != slashSeparator && delimiter != "" {
+		// "heal" option passed can be ignored as the heal-listing does not send non-standard delimiter.
+		return s.listObjectsNonSlash(ctx, bucket, prefix, marker, delimiter, maxKeys)
+	}
+
 	if err = checkListObjsArgs(ctx, bucket, prefix, marker, delimiter, s); err != nil {
 		return loi, err
 	}
@@ -1157,7 +1256,7 @@ func formatsToDrivesInfo(endpoints EndpointList, formats []*formatXLV3, sErrs []
 // healing in a distributed setup.
 func (s *xlSets) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
 	// Acquire lock on format.json
-	formatLock := s.getHashedSet(formatConfigFile).nsMutex.NewNSLock(minioMetaBucket, formatConfigFile)
+	formatLock := s.getHashedSet(formatConfigFile).nsMutex.NewNSLock(ctx, minioMetaBucket, formatConfigFile)
 	if err = formatLock.GetRLock(globalHealingTimeout); err != nil {
 		return err
 	}
@@ -1278,7 +1377,7 @@ func markRootDisksAsDown(storageDisks []StorageAPI) {
 // coded data in it.
 func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealResultItem, err error) {
 	// Acquire lock on format.json
-	formatLock := s.getHashedSet(formatConfigFile).nsMutex.NewNSLock(minioMetaBucket, formatConfigFile)
+	formatLock := s.getHashedSet(formatConfigFile).nsMutex.NewNSLock(ctx, minioMetaBucket, formatConfigFile)
 	if err = formatLock.GetLock(globalHealingTimeout); err != nil {
 		return madmin.HealResultItem{}, err
 	}
@@ -1431,7 +1530,7 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
 
 // HealBucket - heals inconsistent buckets and bucket metadata on all sets.
 func (s *xlSets) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (result madmin.HealResultItem, err error) {
-	bucketLock := globalNSMutex.NewNSLock(bucket, "")
+	bucketLock := globalNSMutex.NewNSLock(ctx, bucket, "")
 	if err := bucketLock.GetLock(globalHealingTimeout); err != nil {
 		return result, err
 	}

cmd/xl-v1-bucket.go

@@ -151,7 +151,7 @@ func (xl xlObjects) getBucketInfo(ctx context.Context, bucketName string) (bucke
 
 // GetBucketInfo - returns BucketInfo for a bucket.
 func (xl xlObjects) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
-	bucketLock := xl.nsMutex.NewNSLock(bucket, "")
+	bucketLock := xl.nsMutex.NewNSLock(ctx, bucket, "")
 	if e := bucketLock.GetRLock(globalObjectTimeout); e != nil {
 		return bi, e
 	}
@@ -237,7 +237,7 @@ func deleteDanglingBucket(ctx context.Context, storageDisks []StorageAPI, dErrs
 
 // DeleteBucket - deletes a bucket.
 func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string) error {
-	bucketLock := xl.nsMutex.NewNSLock(bucket, "")
+	bucketLock := xl.nsMutex.NewNSLock(ctx, bucket, "")
 	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
 		return err
 	}

cmd/xl-v1-healing.go

@@ -717,7 +717,7 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
 	}
 
 	// Lock the object before healing.
-	objectLock := xl.nsMutex.NewNSLock(bucket, object)
+	objectLock := xl.nsMutex.NewNSLock(ctx, bucket, object)
 	if lerr := objectLock.GetRLock(globalHealingTimeout); lerr != nil {
 		return defaultHealResult(latestXLMeta, storageDisks, errs, bucket, object), lerr
 	}

cmd/xl-v1-multipart.go

@@ -258,7 +258,7 @@ func (xl xlObjects) NewMultipartUpload(ctx context.Context, bucket, object strin
 func (xl xlObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error) {
 	// Hold read locks on source object only if we are
 	// going to read data from source object.
-	objectSRLock := xl.nsMutex.NewNSLock(srcBucket, srcObject)
+	objectSRLock := xl.nsMutex.NewNSLock(ctx, srcBucket, srcObject)
 	if err := objectSRLock.GetRLock(globalObjectTimeout); err != nil {
 		return pi, err
 	}
@@ -300,7 +300,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	uploadIDLockPath := xl.getUploadIDLockPath(bucket, object, uploadID)
 
 	// pre-check upload id lock.
-	preUploadIDLock := xl.nsMutex.NewNSLock(minioMetaMultipartBucket, uploadIDLockPath)
+	preUploadIDLock := xl.nsMutex.NewNSLock(ctx, minioMetaMultipartBucket, uploadIDLockPath)
 	if err := preUploadIDLock.GetRLock(globalOperationTimeout); err != nil {
 		return pi, err
 	}
@@ -398,7 +398,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	}
 
 	// post-upload check (write) lock
-	postUploadIDLock := xl.nsMutex.NewNSLock(minioMetaMultipartBucket, uploadIDLockPath)
+	postUploadIDLock := xl.nsMutex.NewNSLock(ctx, minioMetaMultipartBucket, uploadIDLockPath)
 	if err = postUploadIDLock.GetLock(globalOperationTimeout); err != nil {
 		return pi, err
 	}
@@ -492,7 +492,7 @@ func (xl xlObjects) ListObjectParts(ctx context.Context, bucket, object, uploadI
 	}
 	// Hold lock so that there is no competing
 	// abort-multipart-upload or complete-multipart-upload.
-	uploadIDLock := xl.nsMutex.NewNSLock(minioMetaMultipartBucket,
+	uploadIDLock := xl.nsMutex.NewNSLock(ctx, minioMetaMultipartBucket,
 		xl.getUploadIDLockPath(bucket, object, uploadID))
 	if err := uploadIDLock.GetLock(globalListingTimeout); err != nil {
 		return result, err
@@ -596,7 +596,7 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		return oi, err
 	}
 	// Hold write lock on the object.
-	destLock := xl.nsMutex.NewNSLock(bucket, object)
+	destLock := xl.nsMutex.NewNSLock(ctx, bucket, object)
 	if err := destLock.GetLock(globalObjectTimeout); err != nil {
 		return oi, err
 	}
@@ -611,7 +611,7 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	//
 	// 2) no one does a parallel complete-multipart-upload on this
 	// multipart upload
-	uploadIDLock := xl.nsMutex.NewNSLock(minioMetaMultipartBucket, uploadIDLockPath)
+	uploadIDLock := xl.nsMutex.NewNSLock(ctx, minioMetaMultipartBucket, uploadIDLockPath)
 	if err := uploadIDLock.GetLock(globalOperationTimeout); err != nil {
 		return oi, err
 	}
@@ -815,7 +815,7 @@ func (xl xlObjects) AbortMultipartUpload(ctx context.Context, bucket, object, up
 	uploadIDLockPath := xl.getUploadIDLockPath(bucket, object, uploadID)
 	// Hold lock so that there is no competing
 	// complete-multipart-upload or put-object-part.
-	uploadIDLock := xl.nsMutex.NewNSLock(minioMetaMultipartBucket, uploadIDLockPath)
+	uploadIDLock := xl.nsMutex.NewNSLock(ctx, minioMetaMultipartBucket, uploadIDLockPath)
 	if err := uploadIDLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}