fix: healing delete marker on versioned buckets (#10530)

Healing was not working correctly in the distributed mode because
errFileVersionNotFound was not properly converted in storage rest
client.

Besides, fixing the healing delete marker is not working as expected.

.github/stale.yml

@@ -14,6 +14,7 @@ onlyLabels: []
 exemptLabels:
   - "security"
   - "pending discussion"
+  - "do not close"
 
 # Set to true to ignore issues in a project (defaults to false)
 exemptProjects: false

.github/workflows/go.yml

@@ -40,7 +40,7 @@ jobs:
           sudo apt-get install devscripts shellcheck
           nancy_version=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/sonatype-nexus-community/nancy/releases/latest | sed "s/https:\/\/github.com\/sonatype-nexus-community\/nancy\/releases\/tag\///")
           curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-linux.amd64-${nancy_version} && chmod +x nancy
-          go list -m all | ./nancy
+          go list -m all | ./nancy sleuth
           make
           diff -au <(gofmt -s -d cmd) <(printf "")
           diff -au <(gofmt -s -d pkg) <(printf "")

.nancy-ignore

@@ -1,5 +1,6 @@
-CVE-2020-13223
-CVE-2020-7220
-CVE-2020-10661
-CVE-2020-10660
-CWE-190
+CVE-2018-17075
+CVE-2018-17848
+CVE-2018-17846
+CVE-2018-17847
+CVE-2018-17143
+CVE-2018-17142

README.md

@@ -88,7 +88,7 @@ service minio start
 ```
 
 ## Install from Source
-Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.13](https://golang.org/dl/#stable)
+Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.14](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go get github.com/minio/minio
@@ -177,7 +177,7 @@ mc admin update <minio alias, e.g., myminio>
 - `mc admin update` updates and restarts all servers simultaneously, applications would retry and continue their respective operations upon upgrade.
 - `mc admin update` is disabled in kubernetes/container environments, container environments provide their own mechanisms to rollout of updates.
 - In the case of federated setups `mc admin update` should be run against each cluster individually. Avoid updating `mc` to any new releases until all clusters have been successfully updated.
-- If using `kes` as KMS with MinIO, just replace the binary and restart `kes` more information about `kes` can be found [here](https://github.com/minio/kes/wiki)x
+- If using `kes` as KMS with MinIO, just replace the binary and restart `kes` more information about `kes` can be found [here](https://github.com/minio/kes/wiki)
 - If using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
 - If using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
 
@@ -193,4 +193,4 @@ mc admin update <minio alias, e.g., myminio>
 Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
 
 ## License
-Use of MinIO is governed by the Apache 2.0 License found at [LICENSE](./LICENSE).
+Use of MinIO is governed by the Apache 2.0 License found at [LICENSE](https://github.com/minio/minio/blob/master/LICENSE).

README_zh_CN.md

@@ -89,7 +89,7 @@ service minio start
 
 ## 使用源码安装
 
-采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://golang.org/doc/install)。最低需要Golang版本为 [go1.13](https://golang.org/dl/#stable)
+采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://golang.org/doc/install)。最低需要Golang版本为 [go1.14](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go get github.com/minio/minio

browser/app/js/browser/BrowserDropdown.js

@@ -57,22 +57,6 @@ export class BrowserDropdown extends React.Component {
     const { fetchServerInfo } = this.props
     fetchServerInfo()
   }
-  fullScreen(e) {
-    e.preventDefault()
-    let el = document.documentElement
-    if (el.requestFullscreen) {
-      el.requestFullscreen()
-    }
-    if (el.mozRequestFullScreen) {
-      el.mozRequestFullScreen()
-    }
-    if (el.webkitRequestFullscreen) {
-      el.webkitRequestFullscreen()
-    }
-    if (el.msRequestFullscreen) {
-      el.msRequestFullscreen()
-    }
-  }
   logout(e) {
     e.preventDefault()
     web.Logout()
@@ -87,24 +71,30 @@ export class BrowserDropdown extends React.Component {
             <i className="fas fa-bars" />
           </Dropdown.Toggle>
           <Dropdown.Menu className="dropdown-menu-right">
+            <li>
+              <a href="" onClick={this.showChangePassword.bind(this)}>
+                Change Password <i className="fas fa-cog" />
+              </a>
+              {this.state.showChangePasswordModal && (
+                <ChangePasswordModal
+                  serverInfo={serverInfo}
+                  hideChangePassword={this.hideChangePassword.bind(this)}
+                />
+              )}
+            </li>
+            <li>
+              <a target="_blank" href="https://docs.min.io/?ref=ob">
+                Documentation <i className="fas fa-book" />
+              </a>
+            </li>
             <li>
               <a target="_blank" href="https://github.com/minio/minio">
                 GitHub <i className="fab fa-github" />
               </a>
             </li>
             <li>
-              <a href="" onClick={this.fullScreen}>
-                Fullscreen <i className="fas fa-expand" />
-              </a>
-            </li>
-            <li>
-              <a target="_blank" href="https://docs.min.io/">
-                Documentation <i className="fas fa-book" />
-              </a>
-            </li>
-            <li>
-              <a target="_blank" href="https://slack.min.io">
-                Ask for help <i className="fas fa-question-circle" />
+              <a target="_blank" href="https://min.io/pricing?ref=ob">
+                Get Support <i className="fas fa-question-circle" />
               </a>
             </li>
             <li>
@@ -118,20 +108,9 @@ export class BrowserDropdown extends React.Component {
                 />
               )}
             </li>
-            <li>
-              <a href="" onClick={this.showChangePassword.bind(this)}>
-                Change Password <i className="fas fa-cog" />
-              </a>
-              {this.state.showChangePasswordModal && (
-                <ChangePasswordModal
-                  serverInfo={serverInfo}
-                  hideChangePassword={this.hideChangePassword.bind(this)}
-                />
-              )}
-            </li>
             <li>
               <a href="" id="logout" onClick={this.logout}>
-                Sign Out <i className="fas fa-sign-out-alt" />
+                Logout <i className="fas fa-sign-out-alt" />
               </a>
             </li>
           </Dropdown.Menu>

browser/app/js/browser/Header.js

@@ -15,6 +15,7 @@
  */
 
 import React from "react"
+import ObjectsSearch from "../objects/ObjectsSearch"
 import Path from "../objects/Path"
 import StorageInfo from "./StorageInfo"
 import BrowserDropdown from "./BrowserDropdown"
@@ -27,6 +28,7 @@ export const Header = () => {
     <header className="fe-header">
       <Path />
       {loggedIn && <StorageInfo />}
+      {loggedIn && <ObjectsSearch />}
       <ul className="feh-actions">
         {loggedIn ? (
           <BrowserDropdown />

browser/app/js/buckets/selectors.js

@@ -22,7 +22,8 @@ const bucketsFilterSelector = state => state.buckets.filter
 export const getFilteredBuckets = createSelector(
   bucketsSelector,
   bucketsFilterSelector,
-  (buckets, filter) => buckets.filter(bucket => bucket.indexOf(filter) > -1)
+  (buckets, filter) => buckets.filter(
+    bucket => bucket.toLowerCase().indexOf(filter.toLowerCase()) > -1)
 )
 
 export const getCurrentBucket = state => state.buckets.currentBucket

browser/app/js/objects/ObjectsListContainer.js

@@ -18,6 +18,7 @@ import React from "react"
 import { connect } from "react-redux"
 import InfiniteScroll from "react-infinite-scroller"
 import ObjectsList from "./ObjectsList"
+import { getFilteredObjects } from "./selectors"
 
 export class ObjectsListContainer extends React.Component {
   constructor(props) {
@@ -39,22 +40,29 @@ export class ObjectsListContainer extends React.Component {
       })
     }
   }
+  componentDidUpdate(prevProps) {
+    if (this.props.filter !== prevProps.filter) {
+      this.setState({
+        page: 1
+      })
+    }
+  }
   loadNextPage() {
     this.setState(state => {
       return { page: state.page + 1 }
     })
   }
   render() {
-    const { objects, listLoading } = this.props
+    const { filteredObjects, listLoading } = this.props
 
-    const visibleObjects = objects.slice(0, this.state.page * 100)
+    const visibleObjects = filteredObjects.slice(0, this.state.page * 100)
 
     return (
       <div style={{ position: "relative" }}>
         <InfiniteScroll
           pageStart={0}
           loadMore={this.loadNextPage}
-          hasMore={objects.length > visibleObjects.length}
+          hasMore={filteredObjects.length > visibleObjects.length}
           useWindow={true}
           initialLoad={false}
         >
@@ -70,7 +78,8 @@ const mapStateToProps = state => {
   return {
     currentBucket: state.buckets.currentBucket,
     currentPrefix: state.objects.currentPrefix,
-    objects: state.objects.list,
+    filteredObjects: getFilteredObjects(state),
+    filter: state.objects.filter,
     sortBy: state.objects.sortBy,
     sortOrder: state.objects.sortOrder,
     listLoading: state.objects.listLoading

browser/app/js/objects/ObjectsSearch.js

@@ -0,0 +1,43 @@
+/*
+ * MinIO Cloud Storage (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from "react"
+import { connect } from "react-redux"
+import * as actionsObjects from "./actions"
+
+export const ObjectsSearch = ({ onChange }) => (
+  <div
+    className="input-group ig-left ig-search-dark"
+    style={{ display: "block" }}
+  >
+    <input
+      className="ig-text"
+      type="input"
+      placeholder="Search Objects..."
+      onChange={e => onChange(e.target.value)}
+    />
+    <i className="ig-helpers" />
+  </div>
+)
+
+const mapDispatchToProps = dispatch => {
+  return {
+    onChange: filter =>
+      dispatch(actionsObjects.setFilter(filter))
+  }
+}
+
+export default connect(undefined, mapDispatchToProps)(ObjectsSearch)

browser/app/js/objects/__tests__/ObjectsListContainer.test.js

@@ -20,13 +20,13 @@ import { ObjectsListContainer } from "../ObjectsListContainer"
 
 describe("ObjectsList", () => {
   it("should render without crashing", () => {
-    shallow(<ObjectsListContainer objects={[]} />)
+    shallow(<ObjectsListContainer filteredObjects={[]} />)
   })
 
   it("should render ObjectsList with objects", () => {
     const wrapper = shallow(
       <ObjectsListContainer
-        objects={[{ name: "test1.jpg" }, { name: "test2.jpg" }]}
+        filteredObjects={[{ name: "test1.jpg" }, { name: "test2.jpg" }]}
       />
     )
     expect(wrapper.find("ObjectsList").length).toBe(1)
@@ -40,7 +40,7 @@ describe("ObjectsList", () => {
     const wrapper = shallow(
       <ObjectsListContainer
         currentBucket="test1"
-        objects={[]}
+        filteredObjects={[]}
         listLoading={true}
       />
     )

browser/app/js/objects/__tests__/ObjectsSearch.test.js

@@ -0,0 +1,32 @@
+/*
+ * MinIO Cloud Storage (C) 2018 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from "react"
+import { shallow } from "enzyme"
+import { ObjectsSearch } from "../ObjectsSearch"
+
+describe("ObjectsSearch", () => {
+  it("should render without crashing", () => {
+    shallow(<ObjectsSearch />)
+  })
+
+  it("should call onChange with search text", () => {
+    const onChange = jest.fn()
+    const wrapper = shallow(<ObjectsSearch onChange={onChange} />)
+    wrapper.find("input").simulate("change", { target: { value: "test" } })
+    expect(onChange).toHaveBeenCalledWith("test")
+  })
+})

browser/app/js/objects/__tests__/reducer.test.js

@@ -23,6 +23,7 @@ describe("objects reducer", () => {
     const initialState = reducer(undefined, {})
     expect(initialState).toEqual({
       list: [],
+      filter: "",
       listLoading: false,
       sortBy: "",
       sortOrder: SORT_ORDER_ASC,

browser/app/js/objects/actions.js

@@ -36,6 +36,7 @@ import { getServerInfo, hasServerPublicDomain } from '../browser/selectors'
 
 export const SET_LIST = "objects/SET_LIST"
 export const RESET_LIST = "objects/RESET_LIST"
+export const SET_FILTER = "objects/SET_FILTER"
 export const APPEND_LIST = "objects/APPEND_LIST"
 export const REMOVE = "objects/REMOVE"
 export const SET_SORT_BY = "objects/SET_SORT_BY"
@@ -57,6 +58,13 @@ export const resetList = () => ({
   type: RESET_LIST,
 })
 
+export const setFilter = filter => {
+  return {
+    type: SET_FILTER,
+    filter
+  }
+}
+
 export const setListLoading = (listLoading) => ({
   type: SET_LIST_LOADING,
   listLoading,

browser/app/js/objects/reducer.js

@@ -28,6 +28,7 @@ const removeObject = (list, objectToRemove, lookup) => {
 export default (
   state = {
     list: [],
+    filter: "",
     listLoading: false,
     sortBy: "",
     sortOrder: SORT_ORDER_ASC,
@@ -53,6 +54,11 @@ export default (
         ...state,
         list: []
       }
+    case actionsObjects.SET_FILTER:
+      return {
+        ...state,
+        filter: action.filter
+      }
     case actionsObjects.SET_LIST_LOADING:
       return {
         ...state,

browser/app/js/objects/selectors.js

@@ -21,3 +21,13 @@ export const getCurrentPrefix = state => state.objects.currentPrefix
 export const getCheckedList = state => state.objects.checkedList
 
 export const getPrefixWritable = state => state.objects.prefixWritable
+
+const objectsSelector = state => state.objects.list
+const objectsFilterSelector = state => state.objects.filter
+
+export const getFilteredObjects = createSelector(
+  objectsSelector,
+  objectsFilterSelector,
+  (objects, filter) => objects.filter(
+    object => object.name.toLowerCase().startsWith(filter.toLowerCase()))
+)

browser/app/less/inc/form.less

@@ -169,6 +169,24 @@ select.form-control {
     }
 }
 
+.ig-search-dark {
+    &:before {
+        font-family: @font-family-icon;
+        font-weight: 900;
+        content: '\f002';
+        font-size: 15px;
+        position: absolute;
+        left: 2px;
+        top: 8px;
+        color: rgba(0, 0, 0, 0.5);
+    }
+
+    .ig-text {
+        padding-left: 25px;
+        .placeholder(rgba(0, 0, 0, 0.5))
+    }
+}
+
 .ig-search {
     &:before {
         font-family: @font-family-icon;
@@ -270,4 +288,4 @@ select.form-control {
 .set-expire-decrease {
     bottom: -27px;
     .rotate(-180deg);
-}
+}

buildscripts/verify-build.sh

@@ -45,88 +45,63 @@ FUNCTIONAL_TESTS="$WORK_DIR/functional-tests.sh"
 function start_minio_fs()
 {
     "${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 &
-    minio_pid=$!
     sleep 10
-
-    echo "$minio_pid"
 }
 
 function start_minio_erasure()
 {
     "${MINIO[@]}" server "${WORK_DIR}/erasure-disk1" "${WORK_DIR}/erasure-disk2" "${WORK_DIR}/erasure-disk3" "${WORK_DIR}/erasure-disk4" >"$WORK_DIR/erasure-minio.log" 2>&1 &
-    minio_pid=$!
     sleep 15
-
-    echo "$minio_pid"
 }
 
 function start_minio_erasure_sets()
 {
     "${MINIO[@]}" server "${WORK_DIR}/erasure-disk-sets{1...32}" >"$WORK_DIR/erasure-minio-sets.log" 2>&1 &
-    minio_pid=$!
     sleep 15
-
-    echo "$minio_pid"
 }
 
 function start_minio_zone_erasure_sets()
 {
-    declare -a minio_pids
     export MINIO_ACCESS_KEY=$ACCESS_KEY
     export MINIO_SECRET_KEY=$SECRET_KEY
 
     "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://127.0.0.1:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9000.log" 2>&1 &
-    minio_pids[0]=$!
-
     "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://127.0.0.1:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9001.log" 2>&1 &
-    minio_pids[1]=$!
 
     sleep 40
-    echo "${minio_pids[@]}"
 }
 
 function start_minio_zone_erasure_sets_ipv6()
 {
-    declare -a minio_pids
     export MINIO_ACCESS_KEY=$ACCESS_KEY
     export MINIO_SECRET_KEY=$SECRET_KEY
 
     "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9000.log" 2>&1 &
-    minio_pids[0]=$!
-
     "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9001.log" 2>&1 &
-    minio_pids[1]=$!
 
     sleep 40
-    echo "${minio_pids[@]}"
 }
 
 function start_minio_dist_erasure()
 {
-    declare -a minio_pids
     export MINIO_ACCESS_KEY=$ACCESS_KEY
     export MINIO_SECRET_KEY=$SECRET_KEY
     "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
-    minio_pids[0]=$!
     "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &
-    minio_pids[1]=$!
     "${MINIO[@]}" server --address=:9002 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9002.log" 2>&1 &
-    minio_pids[2]=$!
     "${MINIO[@]}" server --address=:9003 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9003.log" 2>&1 &
-    minio_pids[3]=$!
 
     sleep 40
-    echo "${minio_pids[@]}"
 }
 
 function run_test_fs()
 {
-    minio_pid="$(start_minio_fs)"
+    start_minio_fs
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    kill "$minio_pid"
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
@@ -138,12 +113,12 @@ function run_test_fs()
 }
 
 function run_test_erasure_sets() {
-    minio_pid="$(start_minio_erasure_sets)"
+    start_minio_erasure_sets
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    kill "$minio_pid"
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
@@ -156,14 +131,12 @@ function run_test_erasure_sets() {
 
 function run_test_zone_erasure_sets()
 {
-    minio_pids=( $(start_minio_zone_erasure_sets) )
+    start_minio_zone_erasure_sets
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    for pid in "${minio_pids[@]}"; do
-        kill "$pid"
-    done
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
@@ -182,16 +155,14 @@ function run_test_zone_erasure_sets()
 
 function run_test_zone_erasure_sets_ipv6()
 {
-    minio_pids=( $(start_minio_zone_erasure_sets_ipv6) )
+    start_minio_zone_erasure_sets_ipv6
 
     export SERVER_ENDPOINT="[::1]:9000"
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    for pid in "${minio_pids[@]}"; do
-        kill "$pid"
-    done
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
@@ -210,12 +181,12 @@ function run_test_zone_erasure_sets_ipv6()
 
 function run_test_erasure()
 {
-    minio_pid="$(start_minio_erasure)"
+    start_minio_erasure
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    kill "$minio_pid"
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then
@@ -228,14 +199,12 @@ function run_test_erasure()
 
 function run_test_dist_erasure()
 {
-    minio_pids=( $(start_minio_dist_erasure) )
+    start_minio_dist_erasure
 
     (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
     rv=$?
 
-    for pid in "${minio_pids[@]}"; do
-        kill "$pid"
-    done
+    pkill minio
     sleep 3
 
     if [ "$rv" -ne 0 ]; then

cmd/admin-handlers.go

@@ -799,14 +799,12 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	keepConnLive(w, r, respCh)
 }
 
-func getAggregatedBackgroundHealState(ctx context.Context, failOnErr bool) (madmin.BgHealState, error) {
+func getAggregatedBackgroundHealState(ctx context.Context) (madmin.BgHealState, error) {
 	var bgHealStates []madmin.BgHealState
 
 	localHealState, ok := getLocalBackgroundHealStatus()
 	if !ok {
-		if failOnErr {
-			return madmin.BgHealState{}, errServerNotInitialized
-		}
+		return madmin.BgHealState{}, errServerNotInitialized
 	}
 
 	// Get local heal status first
@@ -815,14 +813,16 @@ func getAggregatedBackgroundHealState(ctx context.Context, failOnErr bool) (madm
 	if globalIsDistErasure {
 		// Get heal status from other peers
 		peersHealStates, nerrs := globalNotificationSys.BackgroundHealStatus()
+		var errCount int
 		for _, nerr := range nerrs {
 			if nerr.Err != nil {
-				if failOnErr {
-					return madmin.BgHealState{}, nerr.Err
-				}
 				logger.LogIf(ctx, nerr.Err)
+				errCount++
 			}
 		}
+		if errCount == len(nerrs) {
+			return madmin.BgHealState{}, fmt.Errorf("all remote servers failed to report heal status, cluster is unhealthy")
+		}
 		bgHealStates = append(bgHealStates, peersHealStates...)
 	}
 
@@ -868,7 +868,12 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	aggregateHealStateResult, _ := getAggregatedBackgroundHealState(r.Context(), false)
+	aggregateHealStateResult, err := getAggregatedBackgroundHealState(r.Context())
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
 	if err := json.NewEncoder(w).Encode(aggregateHealStateResult); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1085,7 +1090,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 	// Avoid reusing tcp connection if read timeout is hit
 	// This is needed to make r.Context().Done() work as
 	// expected in case of read timeout
-	w.Header().Add("Connection", "close")
+	w.Header().Set("Connection", "close")
 
 	setEventStreamHeaders(w)
 
@@ -1236,7 +1241,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	vars := mux.Vars(r)
+	query := r.URL.Query()
 	obdInfo := madmin.OBDInfo{}
 	obdInfoCh := make(chan madmin.OBDInfo)
 
@@ -1282,7 +1287,13 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	go func() {
 		defer close(obdInfoCh)
 
-		if cpu, ok := vars["syscpu"]; ok && cpu == "true" {
+		if log := query.Get("log"); log == "true" {
+			obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, getLocalLogOBD(deadlinedCtx, r))
+			obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, globalNotificationSys.LogOBDInfo(deadlinedCtx)...)
+			partialWrite(obdInfo)
+		}
+
+		if cpu := query.Get("syscpu"); cpu == "true" {
 			cpuInfo := getLocalCPUOBDInfo(deadlinedCtx, r)
 
 			obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo)
@@ -1290,7 +1301,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if diskHw, ok := vars["sysdiskhw"]; ok && diskHw == "true" {
+		if diskHw := query.Get("sysdiskhw"); diskHw == "true" {
 			diskHwInfo := getLocalDiskHwOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo)
@@ -1298,7 +1309,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if osInfo, ok := vars["sysosinfo"]; ok && osInfo == "true" {
+		if osInfo := query.Get("sysosinfo"); osInfo == "true" {
 			osInfo := getLocalOsInfoOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo)
@@ -1306,7 +1317,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if mem, ok := vars["sysmem"]; ok && mem == "true" {
+		if mem := query.Get("sysmem"); mem == "true" {
 			memInfo := getLocalMemOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo)
@@ -1314,7 +1325,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if proc, ok := vars["sysprocess"]; ok && proc == "true" {
+		if proc := query.Get("sysprocess"); proc == "true" {
 			procInfo := getLocalProcOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo)
@@ -1322,14 +1333,14 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if config, ok := vars["minioconfig"]; ok && config == "true" {
+		if config := query.Get("minioconfig"); config == "true" {
 			cfg, err := readServerConfig(ctx, objectAPI)
 			logger.LogIf(ctx, err)
 			obdInfo.Minio.Config = cfg
 			partialWrite(obdInfo)
 		}
 
-		if drive, ok := vars["perfdrive"]; ok && drive == "true" {
+		if drive := query.Get("perfdrive"); drive == "true" {
 			// Get drive obd details from local server's drive(s)
 			driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r)
 			driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r)
@@ -1360,7 +1371,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			partialWrite(obdInfo)
 		}
 
-		if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistErasure {
+		if net := query.Get("perfnet"); net == "true" && globalIsDistErasure {
 			obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx))
 			partialWrite(obdInfo)
 
@@ -1374,6 +1385,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 			obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx)
 			partialWrite(obdInfo)
 		}
+
 	}()
 
 	ticker := time.NewTicker(30 * time.Second)
@@ -1489,17 +1501,34 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		Notifications: notifyTarget,
 	}
 
+	// Collect any disk healing.
+	healing, _ := getAggregatedBackgroundHealState(ctx)
+	healDisks := make(map[string]struct{}, len(healing.HealDisks))
+	for _, disk := range healing.HealDisks {
+		healDisks[disk] = struct{}{}
+	}
+
 	// find all disks which belong to each respective endpoints
 	for i := range servers {
 		for _, disk := range storageInfo.Disks {
 			if strings.Contains(disk.Endpoint, servers[i].Endpoint) {
+				if _, ok := healDisks[disk.Endpoint]; ok {
+					disk.Healing = true
+				}
 				servers[i].Disks = append(servers[i].Disks, disk)
 			}
 		}
 	}
+
 	// add all the disks local to this server.
 	for _, disk := range storageInfo.Disks {
+		if disk.DrivePath == "" && disk.Endpoint == "" {
+			continue
+		}
 		if disk.Endpoint == disk.DrivePath {
+			if _, ok := healDisks[disk.Endpoint]; ok {
+				disk.Healing = true
+			}
 			servers[len(servers)-1].Disks = append(servers[len(servers)-1].Disks, disk)
 		}
 	}
@@ -1535,7 +1564,7 @@ func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
 	// Fetch the configured targets
 	tr := NewGatewayHTTPTransport()
 	defer tr.CloseIdleConnections()
-	targetList, err := notify.FetchRegisteredTargets(cfg, GlobalContext.Done(), tr, true, false)
+	targetList, err := notify.FetchRegisteredTargets(GlobalContext, cfg, tr, true, false)
 	if err != nil && err != notify.ErrTargetsOffline {
 		logger.LogIf(GlobalContext, err)
 		return nil
@@ -1579,12 +1608,12 @@ func fetchVaultStatus(cfg config.Config) madmin.Vault {
 	keyID := GlobalKMS.DefaultKeyID()
 	kmsInfo := GlobalKMS.Info()
 
-	if kmsInfo.Endpoint == "" {
+	if len(kmsInfo.Endpoints) == 0 {
 		vault.Status = "KMS configured using master key"
 		return vault
 	}
 
-	if err := checkConnection(kmsInfo.Endpoint, 15*time.Second); err != nil {
+	if err := checkConnection(kmsInfo.Endpoints[0], 15*time.Second); err != nil {
 		vault.Status = "offline"
 	} else {
 		vault.Status = "online"
@@ -1655,11 +1684,6 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
 	ctx, cancel := context.WithTimeout(GlobalContext, timeout)
 	defer cancel()
 
-	req, err := http.NewRequest(http.MethodHead, endpointStr, nil)
-	if err != nil {
-		return err
-	}
-
 	client := &http.Client{Transport: &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
 		DialContext:           xhttp.NewCustomDialContext(timeout),
@@ -1674,11 +1698,15 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
 	}}
 	defer client.CloseIdleConnections()
 
-	resp, err := client.Do(req.WithContext(ctx))
+	req, err := http.NewRequestWithContext(ctx, http.MethodHead, endpointStr, nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := client.Do(req)
 	if err != nil {
 		return err
 	}
 	defer xhttp.DrainBody(resp.Body)
-	resp.Body.Close()
 	return nil
 }

cmd/admin-heal-ops.go

@@ -89,13 +89,14 @@ type allHealState struct {
 
 	// map of heal path to heal sequence
 	healSeqMap     map[string]*healSequence
-	healLocalDisks []Endpoints
+	healLocalDisks map[Endpoint]struct{}
 }
 
 // newHealState - initialize global heal state management
 func newHealState() *allHealState {
 	healState := &allHealState{
-		healSeqMap: make(map[string]*healSequence),
+		healSeqMap:     make(map[string]*healSequence),
+		healLocalDisks: map[Endpoint]struct{}{},
 	}
 
 	go healState.periodicHealSeqsClean(GlobalContext)
@@ -103,20 +104,40 @@ func newHealState() *allHealState {
 	return healState
 }
 
-func (ahs *allHealState) getHealLocalDisks() []Endpoints {
+func (ahs *allHealState) healDriveCount() int {
 	ahs.Lock()
 	defer ahs.Unlock()
 
-	healLocalDisks := make([]Endpoints, len(ahs.healLocalDisks))
-	copy(healLocalDisks, ahs.healLocalDisks)
+	return len(ahs.healLocalDisks)
+}
+
+func (ahs *allHealState) getHealLocalDisks() Endpoints {
+	ahs.Lock()
+	defer ahs.Unlock()
+
+	var healLocalDisks Endpoints
+	for ep := range ahs.healLocalDisks {
+		healLocalDisks = append(healLocalDisks, ep)
+	}
 	return healLocalDisks
 }
 
-func (ahs *allHealState) updateHealLocalDisks(healLocalDisks []Endpoints) {
+func (ahs *allHealState) popHealLocalDisks(healLocalDisks ...Endpoint) {
 	ahs.Lock()
 	defer ahs.Unlock()
 
-	ahs.healLocalDisks = healLocalDisks
+	for _, ep := range healLocalDisks {
+		delete(ahs.healLocalDisks, ep)
+	}
+}
+
+func (ahs *allHealState) pushHealLocalDisks(healLocalDisks ...Endpoint) {
+	ahs.Lock()
+	defer ahs.Unlock()
+
+	for _, ep := range healLocalDisks {
+		ahs.healLocalDisks[ep] = struct{}{}
+	}
 }
 
 func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
@@ -647,6 +668,12 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	if source.opts != nil {
 		task.opts = *source.opts
 	}
+
+	h.mutex.Lock()
+	h.scannedItemsMap[healType]++
+	h.lastHealActivity = UTCNow()
+	h.mutex.Unlock()
+
 	globalBackgroundHealRoutine.queueHealTask(task)
 
 	select {
@@ -729,9 +756,6 @@ func (h *healSequence) healItemsFromSourceCh() error {
 						pathJoin(source.bucket, source.object), err))
 				}
 			}
-
-			h.scannedItemsMap[itemType]++
-			h.lastHealActivity = UTCNow()
 		case <-h.ctx.Done():
 			return nil
 		}
@@ -868,7 +892,9 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 	}
 
 	if err := h.queueHealTask(healSource{bucket: bucket}, madmin.HealItemBucket); err != nil {
-		return err
+		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			return err
+		}
 	}
 
 	if bucketsOnly {
@@ -882,6 +908,9 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 			oi, err := objectAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
 			if err == nil {
 				if err = h.healObject(bucket, h.object, oi.VersionID); err != nil {
+					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+						return nil
+					}
 					return err
 				}
 			}
@@ -891,7 +920,11 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 	}
 
 	if err := objectAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
-		return errFnHealFromAPIErr(h.ctx, err)
+		// Object might have been deleted, by the time heal
+		// was attempted we ignore this object an move on.
+		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			return errFnHealFromAPIErr(h.ctx, err)
+		}
 	}
 	return nil
 }
@@ -913,10 +946,5 @@ func (h *healSequence) healObject(bucket, object, versionID string) error {
 		object:    object,
 		versionID: versionID,
 	}, madmin.HealItemObject)
-	// Object might have been deleted, by the time heal
-	// was attempted we ignore this object an move on.
-	if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-		return nil
-	}
 	return err
 }

cmd/admin-router.go

@@ -212,18 +212,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 		if !globalIsGateway {
 			// -- OBD API --
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/obdinfo").
-				HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler)).
-				Queries("perfdrive", "{perfdrive:true|false}",
-					"perfnet", "{perfnet:true|false}",
-					"minioinfo", "{minioinfo:true|false}",
-					"minioconfig", "{minioconfig:true|false}",
-					"syscpu", "{syscpu:true|false}",
-					"sysdiskhw", "{sysdiskhw:true|false}",
-					"sysosinfo", "{sysosinfo:true|false}",
-					"sysmem", "{sysmem:true|false}",
-					"sysprocess", "{sysprocess:true|false}",
-				)
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
+				HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler))
 		}
 	}
 

cmd/api-errors.go

@@ -29,7 +29,7 @@ import (
 
 	minio "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/cmd/config/etcd/dns"
+	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
@@ -109,6 +109,7 @@ const (
 	ErrReplicationDestinationNotFoundError
 	ErrReplicationDestinationMissingLock
 	ErrReplicationTargetNotFoundError
+	ErrReplicationRemoteConnectionError
 	ErrBucketRemoteIdenticalToSource
 	ErrBucketRemoteAlreadyExists
 	ErrBucketRemoteArnTypeInvalid
@@ -823,6 +824,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The replication target does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrReplicationRemoteConnectionError: {
+		Code:           "XminioAdminReplicationRemoteConnectionError",
+		Description:    "Remote service endpoint or target bucket not available",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrBucketRemoteIdenticalToSource: {
 		Code:           "XminioAdminRemoteIdenticalToSource",
 		Description:    "The remote target cannot be identical to source",
@@ -965,7 +971,7 @@ var errorCodes = errorCodeMap{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMetadataTooLarge: {
-		Code:           "InvalidArgument",
+		Code:           "MetadataTooLarge",
 		Description:    "Your metadata headers exceed the maximum allowed metadata size.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
@@ -1906,6 +1912,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrReplicationDestinationMissingLock
 	case BucketRemoteTargetNotFound:
 		apiErr = ErrReplicationTargetNotFoundError
+	case BucketRemoteConnectionErr:
+		apiErr = ErrReplicationRemoteConnectionError
 	case BucketRemoteAlreadyExists:
 		apiErr = ErrBucketRemoteAlreadyExists
 	case BucketRemoteArnTypeInvalid:

cmd/api-response.go

@@ -36,7 +36,7 @@ import (
 const (
 	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
 	iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
-	maxObjectList     = 10000                      // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
+	maxObjectList     = 1000                       // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
 	maxDeleteList     = 10000                      // Limit number of objects deleted in a delete call.
 	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
 	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.
@@ -408,7 +408,7 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
 // generates ListBucketsResponse from array of BucketInfo which can be
 // serialized to match XML and JSON API spec output.
 func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
-	var listbuckets []Bucket
+	listbuckets := make([]Bucket, 0, len(buckets))
 	var data = ListBucketsResponse{}
 	var owner = Owner{}
 
@@ -428,8 +428,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
 func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
-	var versions []ObjectVersion
-	var prefixes []CommonPrefix
+	versions := make([]ObjectVersion, 0, len(resp.Objects))
 	var owner = Owner{}
 	var data = ListVersionsResponse{}
 
@@ -473,6 +472,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 	data.VersionIDMarker = versionIDMarker
 	data.IsTruncated = resp.IsTruncated
 
+	prefixes := make([]CommonPrefix, 0, len(resp.Prefixes))
 	for _, prefix := range resp.Prefixes {
 		var prefixItem = CommonPrefix{}
 		prefixItem.Prefix = s3EncodeName(prefix, encodingType)
@@ -484,8 +484,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 
 // generates an ListObjectsV1 response for the said bucket with other enumerated options.
 func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
-	var contents []Object
-	var prefixes []CommonPrefix
+	contents := make([]Object, 0, len(resp.Objects))
 	var owner = Owner{}
 	var data = ListObjectsResponse{}
 
@@ -517,9 +516,10 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 	data.Marker = s3EncodeName(marker, encodingType)
 	data.Delimiter = s3EncodeName(delimiter, encodingType)
 	data.MaxKeys = maxKeys
-
 	data.NextMarker = s3EncodeName(resp.NextMarker, encodingType)
 	data.IsTruncated = resp.IsTruncated
+
+	prefixes := make([]CommonPrefix, 0, len(resp.Prefixes))
 	for _, prefix := range resp.Prefixes {
 		var prefixItem = CommonPrefix{}
 		prefixItem.Prefix = s3EncodeName(prefix, encodingType)
@@ -531,8 +531,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
 func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata bool) ListObjectsV2Response {
-	var contents []Object
-	var commonPrefixes []CommonPrefix
+	contents := make([]Object, 0, len(objects))
 	var owner = Owner{}
 	var data = ListObjectsV2Response{}
 
@@ -585,6 +584,8 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 	data.ContinuationToken = base64.StdEncoding.EncodeToString([]byte(token))
 	data.NextContinuationToken = base64.StdEncoding.EncodeToString([]byte(nextToken))
 	data.IsTruncated = isTruncated
+
+	commonPrefixes := make([]CommonPrefix, 0, len(prefixes))
 	for _, prefix := range prefixes {
 		var prefixItem = CommonPrefix{}
 		prefixItem.Prefix = s3EncodeName(prefix, encodingType)

cmd/api-router.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"net"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -60,31 +61,50 @@ func newCachedObjectLayerFn() CacheObjectLayer {
 type objectAPIHandlers struct {
 	ObjectAPI func() ObjectLayer
 	CacheAPI  func() CacheObjectLayer
-	// Returns true of handlers should interpret encryption.
-	EncryptionEnabled func() bool
-	// Returns true if handlers allow SSE-KMS encryption headers.
-	AllowSSEKMS func() bool
+}
+
+// getHost tries its best to return the request host.
+// According to section 14.23 of RFC 2616 the Host header
+// can include the port number if the default value of 80 is not used.
+func getHost(r *http.Request) string {
+	if r.URL.IsAbs() {
+		return r.URL.Host
+	}
+	return r.Host
 }
 
 // registerAPIRouter - registers S3 compatible APIs.
-func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool) {
+func registerAPIRouter(router *mux.Router) {
 	// Initialize API.
 	api := objectAPIHandlers{
 		ObjectAPI: newObjectLayerFn,
 		CacheAPI:  newCachedObjectLayerFn,
-		EncryptionEnabled: func() bool {
-			return encryptionEnabled
-		},
-		AllowSSEKMS: func() bool {
-			return allowSSEKMS
-		},
 	}
 
 	// API Router
 	apiRouter := router.PathPrefix(SlashSeparator).Subrouter()
+
 	var routers []*mux.Router
 	for _, domainName := range globalDomainNames {
-		routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName).Subrouter())
+		if IsKubernetes() {
+			routers = append(routers, apiRouter.MatcherFunc(func(r *http.Request, match *mux.RouteMatch) bool {
+				host, _, _ := net.SplitHostPort(getHost(r))
+				// Make sure to skip matching minio.<domain>` this is
+				// specifically meant for operator/k8s deployment
+				// The reason we need to skip this is for a special
+				// usecase where we need to make sure that
+				// minio.<namespace>.svc.<cluster_domain> is ignored
+				// by the bucketDNS style to ensure that path style
+				// is available and honored at this domain.
+				//
+				// All other `<bucket>.<namespace>.svc.<cluster_domain>`
+				// makes sure that buckets are routed through this matcher
+				// to match for `<bucket>`
+				return host != minioReservedBucket+"."+domainName
+			}).Host("{bucket:.+}."+domainName).Subrouter())
+		} else {
+			routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName).Subrouter())
+		}
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
@@ -94,7 +114,10 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
 			maxClients(collectAPIStats("headobject", httpTraceAll(api.HeadObjectHandler))))
 		// CopyObjectPart
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(maxClients(collectAPIStats("copyobjectpart", httpTraceAll(api.CopyObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").
+			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
+			HandlerFunc(maxClients(collectAPIStats("copyobjectpart", httpTraceAll(api.CopyObjectPartHandler)))).
+			Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
 			maxClients(collectAPIStats("putobjectpart", httpTraceHdrs(api.PutObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
@@ -138,7 +161,8 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
 			maxClients(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler))))
 		// CopyObject
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(maxClients(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler))))
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
+			HandlerFunc(maxClients(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler))))
 		// PutObjectRetention
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
 			maxClients(collectAPIStats("putobjectretention", httpTraceAll(api.PutObjectRetentionHandler)))).Queries("retention", "")

cmd/auth-handler.go

@@ -346,8 +346,26 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 			// Request is allowed return the appropriate access key.
 			return cred.AccessKey, owner, ErrNone
 		}
+
+		if action == policy.ListBucketVersionsAction {
+			// In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission
+			// verify as a fallback.
+			if globalPolicySys.IsAllowed(policy.Args{
+				AccountName:     cred.AccessKey,
+				Action:          policy.ListBucketAction,
+				BucketName:      bucketName,
+				ConditionValues: getConditionValues(r, locationConstraint, "", nil),
+				IsOwner:         false,
+				ObjectName:      objectName,
+			}) {
+				// Request is allowed return the appropriate access key.
+				return cred.AccessKey, owner, ErrNone
+			}
+		}
+
 		return cred.AccessKey, owner, ErrAccessDenied
 	}
+
 	if globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
 		Action:          iampolicy.Action(action),
@@ -360,6 +378,22 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 		// Request is allowed return the appropriate access key.
 		return cred.AccessKey, owner, ErrNone
 	}
+	if action == policy.ListBucketVersionsAction {
+		// In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission
+		// verify as a fallback.
+		if globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     cred.AccessKey,
+			Action:          iampolicy.Action(policy.ListBucketAction),
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			ObjectName:      objectName,
+			IsOwner:         owner,
+			Claims:          claims,
+		}) {
+			// Request is allowed return the appropriate access key.
+			return cred.AccessKey, owner, ErrNone
+		}
+	}
 
 	return cred.AccessKey, owner, ErrAccessDenied
 }

cmd/background-heal-ops.go

@@ -21,7 +21,6 @@ import (
 	"path"
 	"time"
 
-	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -101,6 +100,7 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 				ObjectPathUpdated(path.Join(task.bucket, task.object))
 			}
 			task.responseCh <- healResult{result: res, err: err}
+
 		case <-h.doneCh:
 			return
 		case <-ctx.Done():
@@ -128,24 +128,5 @@ func healDiskFormat(ctx context.Context, objAPI ObjectLayer, opts madmin.HealOpt
 		return madmin.HealResultItem{}, err
 	}
 
-	// Healing succeeded notify the peers to reload format and re-initialize disks.
-	// We will not notify peers if healing is not required.
-	if err == nil {
-		// Notify servers in background and retry if needed.
-		go func() {
-		retry:
-			for _, nerr := range globalNotificationSys.ReloadFormat(opts.DryRun) {
-				if nerr.Err != nil {
-					if nerr.Err.Error() == errServerNotInitialized.Error() {
-						time.Sleep(time.Second)
-						goto retry
-					}
-					logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-					logger.LogIf(ctx, nerr.Err)
-				}
-			}
-		}()
-	}
-
 	return res, nil
 }

cmd/background-newdisks-heal-ops.go

@@ -26,7 +26,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 )
 
-const defaultMonitorNewDiskInterval = time.Minute * 3
+const defaultMonitorNewDiskInterval = time.Second * 10
 
 func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 	z, ok := objAPI.(*erasureZones)
@@ -36,15 +36,6 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 
 	initBackgroundHealing(ctx, objAPI) // start quick background healing
 
-	localDisksInZoneHeal := getLocalDisksToHeal(objAPI)
-	globalBackgroundHealState.updateHealLocalDisks(localDisksInZoneHeal)
-
-	drivesToHeal := getDrivesToHealCount(localDisksInZoneHeal)
-	if drivesToHeal != 0 {
-		logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content...",
-			drivesToHeal, defaultMonitorNewDiskInterval))
-	}
-
 	var bgSeq *healSequence
 	var found bool
 
@@ -56,7 +47,14 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 		time.Sleep(time.Second)
 	}
 
-	if drivesToHeal != 0 {
+	for _, ep := range getLocalDisksToHeal() {
+		globalBackgroundHealState.pushHealLocalDisks(ep)
+	}
+
+	if drivesToHeal := globalBackgroundHealState.healDriveCount(); drivesToHeal > 0 {
+		logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content...",
+			drivesToHeal, defaultMonitorNewDiskInterval))
+
 		// Heal any disk format and metadata early, if possible.
 		if err := bgSeq.healDiskMeta(); err != nil {
 			if newObjectLayerFn() != nil {
@@ -67,19 +65,11 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 		}
 	}
 
-	go monitorLocalDisksAndHeal(ctx, z, drivesToHeal, localDisksInZoneHeal, bgSeq)
+	go monitorLocalDisksAndHeal(ctx, z, bgSeq)
 }
 
-func getLocalDisksToHeal(objAPI ObjectLayer) []Endpoints {
-	z, ok := objAPI.(*erasureZones)
-	if !ok {
-		return nil
-	}
-
-	// Attempt a heal as the server starts-up first.
-	localDisksInZoneHeal := make([]Endpoints, len(z.zones))
-	for i, ep := range globalEndpoints {
-		localDisksToHeal := Endpoints{}
+func getLocalDisksToHeal() (disksToHeal Endpoints) {
+	for _, ep := range globalEndpoints {
 		for _, endpoint := range ep.Endpoints {
 			if !endpoint.IsLocal {
 				continue
@@ -88,28 +78,14 @@ func getLocalDisksToHeal(objAPI ObjectLayer) []Endpoints {
 			// and reformat if the current disk is not formatted
 			_, _, err := connectEndpoint(endpoint)
 			if errors.Is(err, errUnformattedDisk) {
-				localDisksToHeal = append(localDisksToHeal, endpoint)
+				disksToHeal = append(disksToHeal, endpoint)
 			}
 		}
-		if len(localDisksToHeal) == 0 {
-			continue
-		}
-		localDisksInZoneHeal[i] = localDisksToHeal
 	}
-	return localDisksInZoneHeal
+	return disksToHeal
 
 }
 
-func getDrivesToHealCount(localDisksInZoneHeal []Endpoints) int {
-	var drivesToHeal int
-	for _, eps := range localDisksInZoneHeal {
-		for range eps {
-			drivesToHeal++
-		}
-	}
-	return drivesToHeal
-}
-
 func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 	// Run the background healer
 	globalBackgroundHealRoutine = newHealRoutine()
@@ -121,76 +97,69 @@ func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 // monitorLocalDisksAndHeal - ensures that detected new disks are healed
 //  1. Only the concerned erasure set will be listed and healed
 //  2. Only the node hosting the disk is responsible to perform the heal
-func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, drivesToHeal int, localDisksInZoneHeal []Endpoints, bgSeq *healSequence) {
+func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healSequence) {
 	// Perform automatic disk healing when a disk is replaced locally.
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-time.After(defaultMonitorNewDiskInterval):
-			// heal only if new disks found.
-			if drivesToHeal == 0 {
-				localDisksInZoneHeal = getLocalDisksToHeal(z)
-				drivesToHeal = getDrivesToHealCount(localDisksInZoneHeal)
-				if drivesToHeal == 0 {
-					// No drives to heal.
-					globalBackgroundHealState.updateHealLocalDisks(nil)
-					continue
-				}
-				globalBackgroundHealState.updateHealLocalDisks(localDisksInZoneHeal)
-
-				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
-					drivesToHeal))
+			waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()), time.Second)
 
+			var erasureSetInZoneEndpointToHeal []map[int]Endpoints
+			healDisks := globalBackgroundHealState.getHealLocalDisks()
+			if len(healDisks) > 0 {
 				// Reformat disks
 				bgSeq.sourceCh <- healSource{bucket: SlashSeparator}
 
 				// Ensure that reformatting disks is finished
 				bgSeq.sourceCh <- healSource{bucket: nopHeal}
-			}
 
-			var erasureSetInZoneToHeal = make([][]int, len(localDisksInZoneHeal))
-			// Compute the list of erasure set to heal
-			for i, localDisksToHeal := range localDisksInZoneHeal {
-				var erasureSetToHeal []int
-				for _, endpoint := range localDisksToHeal {
-					// Load the new format of this passed endpoint
-					_, format, err := connectEndpoint(endpoint)
-					if err != nil {
-						printEndpointError(endpoint, err, true)
-						continue
-					}
+				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
+					len(healDisks)))
 
-					// Calculate the set index where the current endpoint belongs
-					setIndex, _, err := findDiskIndex(z.zones[i].format, format)
-					if err != nil {
-						printEndpointError(endpoint, err, false)
-						continue
-					}
-
-					erasureSetToHeal = append(erasureSetToHeal, setIndex)
-				}
-				erasureSetInZoneToHeal[i] = erasureSetToHeal
-			}
-
-			logger.Info("New unformatted drives detected attempting to heal the content...")
-			for i, disks := range localDisksInZoneHeal {
-				for _, disk := range disks {
-					logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
+				erasureSetInZoneEndpointToHeal = make([]map[int]Endpoints, len(z.zones))
+				for i := range z.zones {
+					erasureSetInZoneEndpointToHeal[i] = map[int]Endpoints{}
 				}
 			}
 
-			// Heal all erasure sets that need
-			for i, erasureSetToHeal := range erasureSetInZoneToHeal {
-				for _, setIndex := range erasureSetToHeal {
-					err := healErasureSet(ctx, setIndex, z.zones[i].sets[setIndex], z.zones[i].drivesPerSet)
-					if err != nil {
-						logger.LogIf(ctx, err)
-					}
+			// heal only if new disks found.
+			for _, endpoint := range healDisks {
+				// Load the new format of this passed endpoint
+				_, format, err := connectEndpoint(endpoint)
+				if err != nil {
+					printEndpointError(endpoint, err, true)
+					continue
+				}
 
-					// Only upon success reduce the counter
-					if err == nil {
-						drivesToHeal--
+				zoneIdx := globalEndpoints.GetLocalZoneIdx(endpoint)
+				if zoneIdx < 0 {
+					continue
+				}
+
+				// Calculate the set index where the current endpoint belongs
+				setIndex, _, err := findDiskIndex(z.zones[zoneIdx].format, format)
+				if err != nil {
+					printEndpointError(endpoint, err, false)
+					continue
+				}
+
+				erasureSetInZoneEndpointToHeal[zoneIdx][setIndex] = append(erasureSetInZoneEndpointToHeal[zoneIdx][setIndex], endpoint)
+			}
+
+			for i, setMap := range erasureSetInZoneEndpointToHeal {
+				for setIndex, endpoints := range setMap {
+					for _, ep := range endpoints {
+						logger.Info("Healing disk '%s' on %s zone", ep, humanize.Ordinal(i+1))
+
+						if err := healErasureSet(ctx, setIndex, z.zones[i].sets[setIndex], z.zones[i].setDriveCount); err != nil {
+							logger.LogIf(ctx, err)
+							continue
+						}
+
+						// Only upon success pop the healed disk.
+						globalBackgroundHealState.popHealLocalDisks(ep)
 					}
 				}
 			}

cmd/bitrot-streaming.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"encoding/hex"
 	"fmt"
 	"hash"
@@ -80,7 +81,7 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
 			bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
 			totalFileSize = bitrotSumsTotalSize + length
 		}
-		err := disk.CreateFile(volume, filePath, totalFileSize, r)
+		err := disk.CreateFile(context.TODO(), volume, filePath, totalFileSize, r)
 		r.CloseWithError(err)
 		close(bw.canClose)
 	}()
@@ -118,7 +119,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		// For the first ReadAt() call we need to open the stream for reading.
 		b.currOffset = offset
 		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
-		b.rc, err = b.disk.ReadFileStream(b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
+		b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
 		if err != nil {
 			return 0, err
 		}
@@ -139,8 +140,8 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	b.h.Write(buf)
 
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		err := &errHashMismatch{fmt.Sprintf("Disk: %s - content hash does not match - expected %s, got %s",
-			b.disk, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))}
+		err := &errHashMismatch{fmt.Sprintf("Disk: %s  -> %s/%s - content hash does not match - expected %s, got %s",
+			b.disk, b.volume, b.filePath, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))}
 		logger.LogIf(GlobalContext, err)
 		return 0, err
 	}

cmd/bitrot-whole.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"hash"
 	"io"
@@ -34,7 +35,7 @@ type wholeBitrotWriter struct {
 }
 
 func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
-	err := b.disk.AppendFile(b.volume, b.filePath, p)
+	err := b.disk.AppendFile(context.TODO(), b.volume, b.filePath, p)
 	if err != nil {
 		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
 		return 0, err
@@ -69,13 +70,13 @@ type wholeBitrotReader struct {
 func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error) {
 	if b.buf == nil {
 		b.buf = make([]byte, b.tillOffset-offset)
-		if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
-			logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
+		if _, err := b.disk.ReadFile(context.TODO(), b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
+			logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, err))
 			return 0, err
 		}
 	}
 	if len(b.buf) < len(buf) {
-		logger.LogIf(GlobalContext, errLessData)
+		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, errLessData))
 		return 0, errLessData
 	}
 	n = copy(buf, b.buf)

cmd/bitrot_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"io"
 	"io/ioutil"
 	"log"
@@ -34,12 +35,12 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	volume := "testvol"
 	filePath := "testfile"
 
-	disk, err := newXLStorage(tmpDir, "")
+	disk, err := newLocalXLStorage(tmpDir)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	disk.MakeVol(volume)
+	disk.MakeVol(context.Background(), volume)
 
 	writer := newBitrotWriter(disk, volume, filePath, 35, bitrotAlgo, 10)
 

cmd/bootstrap-peer-server.go

@@ -131,7 +131,7 @@ func (client *bootstrapRESTClient) callWithContext(ctx context.Context, method s
 		values = make(url.Values)
 	}
 
-	respBody, err = client.restClient.CallWithContext(ctx, method, values, body, length)
+	respBody, err = client.restClient.Call(ctx, method, values, body, length)
 	if err == nil {
 		return respBody, nil
 	}
@@ -178,17 +178,22 @@ func verifyServerSystemConfig(ctx context.Context, endpointZones EndpointZones)
 			}
 			onlineServers++
 		}
-		// Sleep for a while - so that we don't go into
-		// 100% CPU when half the endpoints are offline.
-		time.Sleep(500 * time.Millisecond)
-		retries++
-		// after 5 retries start logging that servers are not reachable yet
-		if retries >= 5 {
-			logger.Info(fmt.Sprintf("Waiting for atleast %d servers to be online for bootstrap check", len(clnts)/2))
-			logger.Info(fmt.Sprintf("Following servers are currently offline or unreachable %s", offlineEndpoints))
-			retries = 0 // reset to log again after 5 retries.
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+			// Sleep for a while - so that we don't go into
+			// 100% CPU when half the endpoints are offline.
+			time.Sleep(100 * time.Millisecond)
+			retries++
+			// after 5 retries start logging that servers are not reachable yet
+			if retries >= 5 {
+				logger.Info(fmt.Sprintf("Waiting for atleast %d servers to be online for bootstrap check", len(clnts)/2))
+				logger.Info(fmt.Sprintf("Following servers are currently offline or unreachable %s", offlineEndpoints))
+				retries = 0 // reset to log again after 5 retries.
+			}
+			offlineEndpoints = nil
 		}
-		offlineEndpoints = nil
 	}
 	return nil
 }
@@ -234,7 +239,7 @@ func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
 		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
 		// Instantiate a new rest client for healthcheck
 		// to avoid recursive healthCheckFn()
-		respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).CallWithContext(ctx, bootstrapRESTMethodHealth, nil, nil, -1)
+		respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, bootstrapRESTMethodHealth, nil, nil, -1)
 		xhttp.DrainBody(respBody)
 		cancel()
 		var ne *rest.NetworkError

cmd/bucket-handlers.go

@@ -33,7 +33,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/config/etcd/dns"
+	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -72,7 +72,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 
 	// Get buckets in the DNS
 	dnsBuckets, err := globalDNSConfig.List()
-	if err != nil && err != dns.ErrNoEntriesFound {
+	if err != nil && err != dns.ErrNoEntriesFound && err != dns.ErrNotImplemented {
 		logger.LogIf(GlobalContext, err)
 		return
 	}
@@ -80,33 +80,35 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	bucketsSet := set.NewStringSet()
 	bucketsToBeUpdated := set.NewStringSet()
 	bucketsInConflict := set.NewStringSet()
-	for _, bucket := range buckets {
-		bucketsSet.Add(bucket.Name)
-		r, ok := dnsBuckets[bucket.Name]
-		if !ok {
-			bucketsToBeUpdated.Add(bucket.Name)
-			continue
-		}
-		if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
-			if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
-				// No difference in terms of domainIPs and nothing
-				// has changed so we don't change anything on the etcd.
+	if dnsBuckets != nil {
+		for _, bucket := range buckets {
+			bucketsSet.Add(bucket.Name)
+			r, ok := dnsBuckets[bucket.Name]
+			if !ok {
+				bucketsToBeUpdated.Add(bucket.Name)
 				continue
 			}
-			// if domain IPs intersect then it won't be an empty set.
-			// such an intersection means that bucket exists on etcd.
-			// but if we do see a difference with local domain IPs with
-			// hostSlice from etcd then we should update with newer
-			// domainIPs, we proceed to do that here.
-			bucketsToBeUpdated.Add(bucket.Name)
-			continue
+			if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+				if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
+					// No difference in terms of domainIPs and nothing
+					// has changed so we don't change anything on the etcd.
+					continue
+				}
+				// if domain IPs intersect then it won't be an empty set.
+				// such an intersection means that bucket exists on etcd.
+				// but if we do see a difference with local domain IPs with
+				// hostSlice from etcd then we should update with newer
+				// domainIPs, we proceed to do that here.
+				bucketsToBeUpdated.Add(bucket.Name)
+				continue
+			}
+			// No IPs seem to intersect, this means that bucket exists but has
+			// different IP addresses perhaps from a different deployment.
+			// bucket names are globally unique in federation at a given
+			// path prefix, name collision is not allowed. We simply log
+			// an error and continue.
+			bucketsInConflict.Add(bucket.Name)
 		}
-		// No IPs seem to intersect, this means that bucket exists but has
-		// different IP addresses perhaps from a different deployment.
-		// bucket names are globally unique in federation at a given
-		// path prefix, name collision is not allowed. We simply log
-		// an error and continue.
-		bucketsInConflict.Add(bucket.Name)
 	}
 
 	// Add/update buckets that are not registered with the DNS
@@ -449,14 +451,15 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	deleteList := toNames(objectsToDelete)
 	dObjects, errs := deleteObjectsFn(ctx, bucket, deleteList, ObjectOptions{
-		Versioned: globalBucketVersioningSys.Enabled(bucket),
+		Versioned:        globalBucketVersioningSys.Enabled(bucket),
+		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
 	})
 
 	deletedObjects := make([]DeletedObject, len(deleteObjects.Objects))
 	for i := range errs {
 		dindex := objectsToDelete[deleteList[i]]
 		apiErr := toAPIError(ctx, errs[i])
-		if apiErr.Code == "" || apiErr.Code == "NoSuchKey" {
+		if apiErr.Code == "" || apiErr.Code == "NoSuchKey" || apiErr.Code == "InvalidArgument" {
 			deletedObjects[dindex] = dObjects[i]
 			continue
 		}
@@ -561,7 +564,9 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	if globalDNSConfig != nil {
 		sr, err := globalDNSConfig.Get(bucket)
 		if err != nil {
-			if err == dns.ErrNoEntriesFound {
+			// ErrNotImplemented indicates a DNS backend that doesn't need to check if bucket already
+			// exists elsewhere
+			if err == dns.ErrNoEntriesFound || err == dns.ErrNotImplemented {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts); err != nil {
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -654,16 +659,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if !api.EncryptionEnabled() && crypto.IsRequested(r.Header) {
+
+	if !objectAPI.IsEncryptionSupported() && crypto.IsRequested(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	bucket := mux.Vars(r)["bucket"]
 
-	// To detect if the client has disconnected.
-	r.Body = &contextReader{r.Body, r.Context()}
-
 	// Require Content-Length to be set in the request
 	size := r.ContentLength
 	if size < 0 {
@@ -804,7 +807,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if _, err = globalBucketSSEConfigSys.Get(bucket); err == nil || globalAutoEncryption {
 		// This request header needs to be set prior to setting ObjectOptions
 		if !crypto.SSEC.IsRequested(r.Header) {
-			r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
+			r.Header.Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
 		}
 	}
 
@@ -999,7 +1002,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	if globalDNSConfig != nil {
 		if err := globalDNSConfig.Delete(bucket); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually using etcdctl", err))
+			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually", err))
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}

cmd/bucket-listobjects-handlers.go

@@ -93,7 +93,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketVersionsAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}

cmd/bucket-metadata-sys.go

@@ -81,7 +81,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		// This code is needed only for gateway implementations.
 		switch configFile {
 		case bucketSSEConfig:
-			if globalGatewayName == "nas" {
+			if globalGatewayName == NASBackendGateway {
 				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
 				if err != nil {
 					return err
@@ -90,7 +90,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 				return meta.Save(GlobalContext, objAPI)
 			}
 		case bucketLifecycleConfig:
-			if globalGatewayName == "nas" {
+			if globalGatewayName == NASBackendGateway {
 				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
 				if err != nil {
 					return err
@@ -99,7 +99,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 				return meta.Save(GlobalContext, objAPI)
 			}
 		case bucketTaggingConfig:
-			if globalGatewayName == "nas" {
+			if globalGatewayName == NASBackendGateway {
 				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
 				if err != nil {
 					return err
@@ -108,7 +108,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 				return meta.Save(GlobalContext, objAPI)
 			}
 		case bucketNotificationConfig:
-			if globalGatewayName == "nas" {
+			if globalGatewayName == NASBackendGateway {
 				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
 				if err != nil {
 					return err
@@ -273,7 +273,7 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
 // GetNotificationConfig returns configured notification config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
-	if globalIsGateway && globalGatewayName == "nas" {
+	if globalIsGateway && globalGatewayName == NASBackendGateway {
 		// Only needed in case of NAS gateway.
 		objAPI := newObjectLayerWithoutSafeModeFn()
 		if objAPI == nil {

cmd/bucket-metadata.go

@@ -354,7 +354,7 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
 
 // deleteBucketMetadata deletes bucket metadata
 // If config does not exist no error is returned.
-func deleteBucketMetadata(ctx context.Context, obj ObjectLayer, bucket string) error {
+func deleteBucketMetadata(ctx context.Context, obj objectDeleter, bucket string) error {
 	metadataFiles := []string{
 		dataUsageCacheName,
 		bucketMetadataFile,

cmd/bucket-policy.go

@@ -151,9 +151,10 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 		if ok {
 			// Special case for AD/LDAP STS users
 			if k == ldapUser {
-				args[ldapUserPolicyVariable] = []string{vStr}
+				args["user"] = []string{vStr}
+			} else {
+				args[k] = []string{vStr}
 			}
-			args[k] = []string{vStr}
 		}
 	}
 

cmd/bucket-replication.go

@@ -18,7 +18,9 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"net/http"
+	"runtime"
 	"strings"
 	"time"
 
@@ -169,38 +171,53 @@ func putReplicationOpts(dest replication.Destination, objInfo ObjectInfo) (putOp
 
 // replicateObject replicates the specified version of the object to destination bucket
 // The source object is then updated to reflect the replication status.
-func replicateObject(ctx context.Context, bucket, object, versionID string, objectAPI ObjectLayer, eventArg *eventArgs, healPending bool) {
+func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLayer) {
+	bucket := objInfo.Bucket
+	object := objInfo.Name
+
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return
 	}
+
 	tgt := globalBucketTargetSys.GetReplicationTargetClient(ctx, cfg.RoleArn)
 	if tgt == nil {
 		return
 	}
-	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})
+
+	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{
+		VersionID: objInfo.VersionID,
+	})
 	if err != nil {
 		return
 	}
-	defer gr.Close()
-	objInfo := gr.ObjInfo
+
+	objInfo = gr.ObjInfo
 	size, err := objInfo.GetActualSize()
 	if err != nil {
 		logger.LogIf(ctx, err)
+		gr.Close()
 		return
 	}
 
 	dest := cfg.GetDestination()
 	if dest.Bucket == "" {
+		gr.Close()
 		return
 	}
+
+	// if heal encounters a pending replication status, either replication
+	// has failed due to server shutdown or crawler and PutObject replication are in contention.
+	healPending := objInfo.ReplicationStatus == replication.Pending
+
 	// In the rare event that replication is in pending state either due to
 	// server shut down/crash before replication completed or healing and PutObject
 	// race - do an additional stat to see if the version ID exists
 	if healPending {
 		_, err := tgt.StatObject(ctx, dest.Bucket, object, miniogo.StatObjectOptions{VersionID: objInfo.VersionID})
 		if err == nil {
+			gr.Close()
 			// object with same VersionID already exists, replication kicked off by
 			// PutObject might have completed.
 			return
@@ -210,29 +227,35 @@ func replicateObject(ctx context.Context, bucket, object, versionID string, obje
 
 	replicationStatus := replication.Complete
 	_, err = tgt.PutObject(ctx, dest.Bucket, object, gr, size, "", "", putOpts)
+	gr.Close()
 	if err != nil {
 		replicationStatus = replication.Failed
-		// Notify replication failure  event.
-		if eventArg == nil {
-			eventArg = &eventArgs{
-				BucketName: bucket,
-				Object:     objInfo,
-				Host:       "Internal: [Replication]",
-			}
-		}
-		eventArg.EventName = event.OperationReplicationFailed
-		eventArg.Object.UserDefined[xhttp.AmzBucketReplicationStatus] = replicationStatus.String()
-		sendEvent(*eventArg)
 	}
 	objInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replicationStatus.String()
 	if objInfo.UserTags != "" {
 		objInfo.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags
 	}
+
+	// FIXME: add support for missing replication events
+	// - event.ObjectReplicationNotTracked
+	// - event.ObjectReplicationMissedThreshold
+	// - event.ObjectReplicationReplicatedAfterThreshold
+	if replicationStatus == replication.Failed {
+		sendEvent(eventArgs{
+			EventName:  event.ObjectReplicationFailed,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [Replication]",
+		})
+	}
+
 	objInfo.metadataOnly = true // Perform only metadata updates.
 	if _, err = objectAPI.CopyObject(ctx, bucket, object, bucket, object, objInfo, ObjectOptions{
 		VersionID: objInfo.VersionID,
-	}, ObjectOptions{VersionID: objInfo.VersionID}); err != nil {
-		logger.LogIf(ctx, err)
+	}, ObjectOptions{
+		VersionID: objInfo.VersionID,
+	}); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s: %s", objInfo.VersionID, err))
 	}
 }
 
@@ -258,3 +281,60 @@ func filterReplicationStatusMetadata(metadata map[string]string) map[string]stri
 	delKey(xhttp.AmzBucketReplicationStatus)
 	return dst
 }
+
+type replicationState struct {
+	// add future metrics here
+	replicaCh chan ObjectInfo
+}
+
+func (r *replicationState) queueReplicaTask(oi ObjectInfo) {
+	select {
+	case r.replicaCh <- oi:
+	default:
+	}
+}
+
+var (
+	globalReplicationState *replicationState
+	// TODO: currently keeping it conservative
+	// but eventually can be tuned in future,
+	// take only half the CPUs for replication
+	// conservatively.
+	globalReplicationConcurrent = runtime.GOMAXPROCS(0) / 2
+)
+
+func newReplicationState() *replicationState {
+	return &replicationState{
+		replicaCh: make(chan ObjectInfo, globalReplicationConcurrent*2),
+	}
+}
+
+// addWorker creates a new worker to process tasks
+func (r *replicationState) addWorker(ctx context.Context, objectAPI ObjectLayer) {
+	// Add a new worker.
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				close(r.replicaCh)
+				return
+			case oi, ok := <-r.replicaCh:
+				if !ok {
+					return
+				}
+				replicateObject(ctx, oi, objectAPI)
+			}
+		}
+	}()
+}
+
+func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
+	if globalReplicationState == nil {
+		return
+	}
+
+	// Start with globalReplicationConcurrent.
+	for i := 0; i < globalReplicationConcurrent; i++ {
+		globalReplicationState.addWorker(ctx, objectAPI)
+	}
+}

cmd/bucket-targets.go

@@ -21,6 +21,7 @@ import (
 	"net/http"
 	"sync"
 
+	minio "github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio/pkg/bucket/versioning"
@@ -91,13 +92,13 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 		}
 		vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
 		if err != nil {
-			if isErrBucketNotFound(err) {
+			if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
 				return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
 			}
-			if vcfg.Status != string(versioning.Enabled) {
-				return BucketReplicationTargetNotVersioned{Bucket: tgt.TargetBucket}
-			}
-			return err
+			return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
+		}
+		if vcfg.Status != string(versioning.Enabled) {
+			return BucketReplicationTargetNotVersioned{Bucket: tgt.TargetBucket}
 		}
 	}
 

cmd/common-main.go

@@ -20,6 +20,7 @@ import (
 	"crypto/x509"
 	"encoding/gob"
 	"errors"
+	"fmt"
 	"net"
 	"net/url"
 	"os"
@@ -42,15 +43,11 @@ func init() {
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
-	// Initialize globalConsoleSys system
-	globalConsoleSys = NewConsoleLogger(GlobalContext)
-	logger.AddTarget(globalConsoleSys)
-
 	gob.Register(StorageErr(""))
 }
 
 func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
-	if (globalAutoEncryption || GlobalKMS != nil) && !objAPI.IsEncryptionSupported() {
+	if (GlobalKMS != nil) && !objAPI.IsEncryptionSupported() {
 		logger.Fatal(errInvalidArgument,
 			"Encryption support is requested but '%s' does not support encryption", name)
 	}
@@ -293,7 +290,7 @@ func logStartupMessage(msg string) {
 	logger.StartupMessage(msg)
 }
 
-func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
+func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secureConn bool, err error) {
 	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
 		return nil, nil, false, nil
 	}
@@ -302,11 +299,62 @@ func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn b
 		return nil, nil, false, err
 	}
 
-	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), config.LoadX509KeyPair)
+	manager, err = certs.NewManager(GlobalContext, getPublicCertFile(), getPrivateKeyFile(), config.LoadX509KeyPair)
 	if err != nil {
 		return nil, nil, false, err
 	}
 
+	// MinIO has support for multiple certificates. It expects the following structure:
+	//  certs/
+	//   │
+	//   ├─ public.crt
+	//   ├─ private.key
+	//   │
+	//   ├─ example.com/
+	//   │   │
+	//   │   ├─ public.crt
+	//   │   └─ private.key
+	//   └─ foobar.org/
+	//      │
+	//      ├─ public.crt
+	//      └─ private.key
+	//   ...
+	//
+	// Therefore, we read all filenames in the cert directory and check
+	// for each directory whether it contains a public.crt and private.key.
+	// If so, we try to add it to certificate manager.
+	root, err := os.Open(globalCertsDir.Get())
+	if err != nil {
+		return nil, nil, false, err
+	}
+	defer root.Close()
+
+	files, err := root.Readdir(-1)
+	if err != nil {
+		return nil, nil, false, err
+	}
+	for _, file := range files {
+		// We exclude any regular file and the "CAs/" directory.
+		// The "CAs/" directory contains (root) CA certificates
+		// that MinIO adds to its list of trusted roots (tls.Config.RootCAs).
+		// Therefore, "CAs/" does not contain X.509 certificates that
+		// are meant to be served by MinIO.
+		if !file.IsDir() || file.Name() == "CAs" {
+			continue
+		}
+
+		var (
+			certFile = filepath.Join(root.Name(), file.Name(), publicCertFile)
+			keyFile  = filepath.Join(root.Name(), file.Name(), privateKeyFile)
+		)
+		if !isFile(certFile) || !isFile(keyFile) {
+			continue
+		}
+		if err := manager.AddCertificate(certFile, keyFile); err != nil {
+			err = fmt.Errorf("Failed to load TLS certificate '%s': %v", certFile, err)
+			logger.LogIf(GlobalContext, err, logger.Minio)
+		}
+	}
 	secureConn = true
-	return x509Certs, c, secureConn, nil
+	return x509Certs, manager, secureConn, nil
 }

cmd/config-common.go

@@ -46,7 +46,11 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
 	return buffer.Bytes(), nil
 }
 
-func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
+type objectDeleter interface {
+	DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error)
+}
+
+func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string) error {
 	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile, ObjectOptions{})
 	if err != nil && isErrObjectNotFound(err) {
 		return errConfigNotFound

cmd/config-current.go

@@ -25,8 +25,9 @@ import (
 	"github.com/minio/minio/cmd/config/api"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
+	"github.com/minio/minio/cmd/config/crawler"
+	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/config/etcd"
-	"github.com/minio/minio/cmd/config/etcd/dns"
 	xldap "github.com/minio/minio/cmd/config/identity/ldap"
 	"github.com/minio/minio/cmd/config/identity/openid"
 	"github.com/minio/minio/cmd/config/notify"
@@ -55,6 +56,7 @@ func initHelp() {
 		config.KmsKesSubSys:         crypto.DefaultKesKVS,
 		config.LoggerWebhookSubSys:  logger.DefaultKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditKVS,
+		config.CrawlerSubSys:        crawler.DefaultKVS,
 	}
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
@@ -106,6 +108,10 @@ func initHelp() {
 			Key:         config.APISubSys,
 			Description: "manage global HTTP API call specific features, such as throttling, authentication types, etc.",
 		},
+		config.HelpKV{
+			Key:         config.CrawlerSubSys,
+			Description: "manage continuous disk crawling for bucket disk usage, lifecycle, quota and data integrity checks",
+		},
 		config.HelpKV{
 			Key:             config.LoggerWebhookSubSys,
 			Description:     "send server logs to webhook endpoints",
@@ -185,6 +191,7 @@ func initHelp() {
 		config.EtcdSubSys:           etcd.Help,
 		config.CacheSubSys:          cache.Help,
 		config.CompressionSubSys:    compress.Help,
+		config.CrawlerSubSys:        crawler.Help,
 		config.IdentityOpenIDSubSys: openid.Help,
 		config.IdentityLDAPSubSys:   xldap.Help,
 		config.PolicyOPASubSys:      opa.Help,
@@ -246,6 +253,10 @@ func validateConfig(s config.Config, setDriveCount int) error {
 		return err
 	}
 
+	if _, err := crawler.LookupConfig(s[config.CrawlerSubSys][config.Default]); err != nil {
+		return err
+	}
+
 	{
 		etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 		if err != nil {
@@ -306,8 +317,7 @@ func validateConfig(s config.Config, setDriveCount int) error {
 		return err
 	}
 
-	return notify.TestNotificationTargets(s, GlobalContext.Done(), NewGatewayHTTPTransport(),
-		globalNotificationSys.ConfiguredTargetIDs())
+	return notify.TestNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalNotificationSys.ConfiguredTargetIDs())
 }
 
 func lookupConfigs(s config.Config, setDriveCount int) {
@@ -322,6 +332,19 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 		}
 	}
 
+	if dnsURL, dnsUser, dnsPass, ok := env.LookupEnv(config.EnvDNSWebhook); ok {
+		globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
+			dns.Authentication(dnsUser, dnsPass),
+			dns.RootCAs(globalRootCAs))
+		if err != nil {
+			if globalIsGateway {
+				logger.FatalIf(err, "Unable to initialize remote webhook DNS config")
+			} else {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
+			}
+		}
+	}
+
 	etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 	if err != nil {
 		if globalIsGateway {
@@ -343,19 +366,25 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 			}
 		}
 
-		if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil && globalDNSConfig == nil {
-			globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
-				dns.DomainNames(globalDomainNames),
-				dns.DomainIPs(globalDomainIPs),
-				dns.DomainPort(globalMinioPort),
-				dns.CoreDNSPath(etcdCfg.CoreDNSPath),
-			)
-			if err != nil {
-				if globalIsGateway {
-					logger.FatalIf(err, "Unable to initialize DNS config")
-				} else {
-					logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
-						globalDomainNames, err))
+		if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
+			if globalDNSConfig != nil {
+				// if global DNS is already configured, indicate with a warning, incase
+				// users are confused.
+				logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, not using etcd for DNS store", globalDNSConfig))
+			} else {
+				globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
+					dns.DomainNames(globalDomainNames),
+					dns.DomainIPs(globalDomainIPs),
+					dns.DomainPort(globalMinioPort),
+					dns.CoreDNSPath(etcdCfg.CoreDNSPath),
+				)
+				if err != nil {
+					if globalIsGateway {
+						logger.FatalIf(err, "Unable to initialize DNS config")
+					} else {
+						logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
+							globalDomainNames, err))
+					}
 				}
 			}
 		}
@@ -378,7 +407,12 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
 	}
 
-	globalAPIConfig.init(apiConfig)
+	globalAPIConfig.init(apiConfig, setDriveCount)
+
+	// Initialize remote instance transport once.
+	getRemoteInstanceTransportOnce.Do(func() {
+		getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
+	})
 
 	if globalIsErasure {
 		globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
@@ -404,6 +438,10 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 			}
 		}
 	}
+	globalCrawlerConfig, err = crawler.LookupConfig(s[config.CrawlerSubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to read crawler config: %w", err))
+	}
 
 	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())
 	if err != nil {
@@ -417,6 +455,9 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 
 	// Enable auto-encryption if enabled
 	globalAutoEncryption = kmsCfg.AutoEncryption
+	if globalAutoEncryption && !globalIsGateway {
+		logger.LogIf(ctx, fmt.Errorf("%s env is deprecated please migrate to using `mc encrypt` at bucket level", crypto.EnvKMSAutoEncryption))
+	}
 
 	globalCompressConfig, err = compress.LookupConfig(s[config.CompressionSubSys][config.Default])
 	if err != nil {
@@ -484,12 +525,12 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 		}
 	}
 
-	globalConfigTargetList, err = notify.GetNotificationTargets(s, GlobalContext.Done(), NewGatewayHTTPTransport(), false)
+	globalConfigTargetList, err = notify.GetNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), false)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
 
-	globalEnvTargetList, err = notify.GetNotificationTargets(newServerConfig(), GlobalContext.Done(), NewGatewayHTTPTransport(), true)
+	globalEnvTargetList, err = notify.GetNotificationTargets(GlobalContext, newServerConfig(), NewGatewayHTTPTransport(), true)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}

cmd/config/api/api.go

@@ -29,15 +29,17 @@ import (
 
 // API sub-system constants
 const (
-	apiRequestsMax      = "requests_max"
-	apiRequestsDeadline = "requests_deadline"
-	apiReadyDeadline    = "ready_deadline"
-	apiCorsAllowOrigin  = "cors_allow_origin"
+	apiRequestsMax             = "requests_max"
+	apiRequestsDeadline        = "requests_deadline"
+	apiReadyDeadline           = "ready_deadline"
+	apiCorsAllowOrigin         = "cors_allow_origin"
+	apiRemoteTransportDeadline = "remote_transport_deadline"
 
-	EnvAPIRequestsMax      = "MINIO_API_REQUESTS_MAX"
-	EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE"
-	EnvAPIReadyDeadline    = "MINIO_API_READY_DEADLINE"
-	EnvAPICorsAllowOrigin  = "MINIO_API_CORS_ALLOW_ORIGIN"
+	EnvAPIRequestsMax             = "MINIO_API_REQUESTS_MAX"
+	EnvAPIRequestsDeadline        = "MINIO_API_REQUESTS_DEADLINE"
+	EnvAPIReadyDeadline           = "MINIO_API_READY_DEADLINE"
+	EnvAPICorsAllowOrigin         = "MINIO_API_CORS_ALLOW_ORIGIN"
+	EnvAPIRemoteTransportDeadline = "MINIO_API_REMOTE_TRANSPORT_DEADLINE"
 )
 
 // DefaultKVS - default storage class config
@@ -59,15 +61,20 @@ var (
 			Key:   apiCorsAllowOrigin,
 			Value: "*",
 		},
+		config.KV{
+			Key:   apiRemoteTransportDeadline,
+			Value: "2h",
+		},
 	}
 )
 
 // Config storage class configuration
 type Config struct {
-	APIRequestsMax      int           `json:"requests_max"`
-	APIRequestsDeadline time.Duration `json:"requests_deadline"`
-	APIReadyDeadline    time.Duration `json:"ready_deadline"`
-	APICorsAllowOrigin  []string      `json:"cors_allow_origin"`
+	RequestsMax             int           `json:"requests_max"`
+	RequestsDeadline        time.Duration `json:"requests_deadline"`
+	ReadyDeadline           time.Duration `json:"ready_deadline"`
+	CorsAllowOrigin         []string      `json:"cors_allow_origin"`
+	RemoteTransportDeadline time.Duration `json:"remote_transport_deadline"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -108,10 +115,17 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	}
 
 	corsAllowOrigin := strings.Split(env.Get(EnvAPICorsAllowOrigin, kvs.Get(apiCorsAllowOrigin)), ",")
+
+	remoteTransportDeadline, err := time.ParseDuration(env.Get(EnvAPIRemoteTransportDeadline, kvs.Get(apiRemoteTransportDeadline)))
+	if err != nil {
+		return cfg, err
+	}
+
 	return Config{
-		APIRequestsMax:      requestsMax,
-		APIRequestsDeadline: requestsDeadline,
-		APIReadyDeadline:    readyDeadline,
-		APICorsAllowOrigin:  corsAllowOrigin,
+		RequestsMax:             requestsMax,
+		RequestsDeadline:        requestsDeadline,
+		ReadyDeadline:           readyDeadline,
+		CorsAllowOrigin:         corsAllowOrigin,
+		RemoteTransportDeadline: remoteTransportDeadline,
 	}, nil
 }

cmd/config/api/help.go

@@ -39,5 +39,11 @@ var (
 			Optional:    true,
 			Type:        "csv",
 		},
+		config.HelpKV{
+			Key:         apiRemoteTransportDeadline,
+			Description: `set the deadline for API requests on remote transports while proxying between federated instances e.g. "2h"`,
+			Optional:    true,
+			Type:        "duration",
+		},
 	}
 )

cmd/config/config.go

@@ -76,6 +76,7 @@ const (
 	KmsKesSubSys         = "kms_kes"
 	LoggerWebhookSubSys  = "logger_webhook"
 	AuditWebhookSubSys   = "audit_webhook"
+	CrawlerSubSys        = "crawler"
 
 	// Add new constants here if you add new fields to config.
 )
@@ -112,6 +113,7 @@ var SubSystems = set.CreateStringSet([]string{
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
+	CrawlerSubSys,
 	NotifyAMQPSubSys,
 	NotifyESSubSys,
 	NotifyKafkaSubSys,
@@ -138,6 +140,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
+	CrawlerSubSys,
 }...)
 
 // Constant separators
@@ -448,6 +451,11 @@ func (c Config) Merge() Config {
 					ckvs.Set(kv.Key, kv.Value)
 				}
 			}
+			if _, ok := cp[subSys]; !ok {
+				// A config subsystem was removed or server was downgraded.
+				Logger.Info("config: ignoring unknown subsystem config %q\n", subSys)
+				continue
+			}
 			cp[subSys][tgt] = ckvs
 		}
 	}

cmd/config/constants.go

@@ -33,6 +33,7 @@ const (
 	EnvPublicIPs    = "MINIO_PUBLIC_IPS"
 	EnvFSOSync      = "MINIO_FS_OSYNC"
 	EnvArgs         = "MINIO_ARGS"
+	EnvDNSWebhook   = "MINIO_DNS_WEBHOOK_ENDPOINT"
 
 	EnvUpdate = "MINIO_UPDATE"
 

cmd/config/crawler/crawler.go

@@ -0,0 +1,67 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package crawler
+
+import (
+	"errors"
+
+	"github.com/minio/minio/cmd/config"
+)
+
+// Compression environment variables
+const (
+	BitrotScan = "bitrotscan"
+)
+
+// Config represents the crawler settings.
+type Config struct {
+	// Bitrot will perform bitrot scan on local disk when checking objects.
+	Bitrot bool `json:"bitrotscan"`
+}
+
+var (
+	// DefaultKVS - default KV config for crawler settings
+	DefaultKVS = config.KVS{
+		config.KV{
+			Key:   BitrotScan,
+			Value: config.EnableOff,
+		},
+	}
+
+	// Help provides help for config values
+	Help = config.HelpKVS{
+		config.HelpKV{
+			Key:         BitrotScan,
+			Description: `perform bitrot scan on disks when checking objects during crawl`,
+			Optional:    true,
+			Type:        "on|off",
+		},
+	}
+)
+
+// LookupConfig - lookup config and override with valid environment settings if any.
+func LookupConfig(kvs config.KVS) (cfg Config, err error) {
+	if err = config.CheckValidKeys(config.CrawlerSubSys, kvs, DefaultKVS); err != nil {
+		return cfg, err
+	}
+	bitrot := kvs.Get(BitrotScan)
+	if bitrot != config.EnableOn && bitrot != config.EnableOff {
+		return cfg, errors.New(BitrotScan + ": must be 'on' or 'off'")
+	}
+	cfg.Bitrot = bitrot == config.EnableOn
+	return cfg, nil
+}

cmd/config/dns/etcd_dns.go

@@ -26,9 +26,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/minio-go/v7/pkg/set"
-
 	"github.com/coredns/coredns/plugin/etcd/msg"
+	"github.com/minio/minio-go/v7/pkg/set"
 	"go.etcd.io/etcd/v3/clientv3"
 )
 
@@ -214,6 +213,11 @@ func (c *CoreDNS) DeleteRecord(record SrvRecord) error {
 	return nil
 }
 
+// String stringer name for this implementation of dns.Store
+func (c *CoreDNS) String() string {
+	return "etcdDNS"
+}
+
 // CoreDNS - represents dns config for coredns server.
 type CoreDNS struct {
 	domainNames []string
@@ -223,13 +227,13 @@ type CoreDNS struct {
 	etcdClient  *clientv3.Client
 }
 
-// Option - functional options pattern style
-type Option func(*CoreDNS)
+// EtcdOption - functional options pattern style
+type EtcdOption func(*CoreDNS)
 
 // DomainNames set a list of domain names used by this CoreDNS
 // client setting, note this will fail if set to empty when
 // constructor initializes.
-func DomainNames(domainNames []string) Option {
+func DomainNames(domainNames []string) EtcdOption {
 	return func(args *CoreDNS) {
 		args.domainNames = domainNames
 	}
@@ -237,14 +241,14 @@ func DomainNames(domainNames []string) Option {
 
 // DomainIPs set a list of custom domain IPs, note this will
 // fail if set to empty when constructor initializes.
-func DomainIPs(domainIPs set.StringSet) Option {
+func DomainIPs(domainIPs set.StringSet) EtcdOption {
 	return func(args *CoreDNS) {
 		args.domainIPs = domainIPs
 	}
 }
 
 // DomainPort - is a string version of server port
-func DomainPort(domainPort string) Option {
+func DomainPort(domainPort string) EtcdOption {
 	return func(args *CoreDNS) {
 		args.domainPort = domainPort
 	}
@@ -253,14 +257,14 @@ func DomainPort(domainPort string) Option {
 // CoreDNSPath - custom prefix on etcd to populate DNS
 // service records, optional and can be empty.
 // if empty then c.prefixPath is used i.e "/skydns"
-func CoreDNSPath(prefix string) Option {
+func CoreDNSPath(prefix string) EtcdOption {
 	return func(args *CoreDNS) {
 		args.prefixPath = prefix
 	}
 }
 
 // NewCoreDNS - initialize a new coreDNS set/unset values.
-func NewCoreDNS(cfg clientv3.Config, setters ...Option) (Store, error) {
+func NewCoreDNS(cfg clientv3.Config, setters ...EtcdOption) (Store, error) {
 	etcdClient, err := clientv3.New(cfg)
 	if err != nil {
 		return nil, err

cmd/config/dns/operator_dns.go

@@ -0,0 +1,221 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package dns
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/dgrijalva/jwt-go"
+	"github.com/minio/minio/cmd/config"
+	xhttp "github.com/minio/minio/cmd/http"
+)
+
+var (
+	defaultOperatorContextTimeout = 10 * time.Second
+	// ErrNotImplemented - Indicates the functionality which is not implemented
+	ErrNotImplemented = errors.New("The method is not implemented")
+)
+
+func (c *OperatorDNS) addAuthHeader(r *http.Request) error {
+	if c.username == "" || c.password == "" {
+		return nil
+	}
+
+	claims := &jwt.StandardClaims{
+		ExpiresAt: int64(15 * time.Minute),
+		Issuer:    c.username,
+		Subject:   config.EnvDNSWebhook,
+	}
+
+	token := jwt.NewWithClaims(jwt.SigningMethodHS512, claims)
+	ss, err := token.SignedString([]byte(c.password))
+	if err != nil {
+		return err
+	}
+
+	r.Header.Set("Authorization", "Bearer "+ss)
+	return nil
+}
+
+func (c *OperatorDNS) endpoint(bucket string, delete bool) (string, error) {
+	u, err := url.Parse(c.Endpoint)
+	if err != nil {
+		return "", err
+	}
+	q := u.Query()
+	q.Add("bucket", bucket)
+	q.Add("delete", strconv.FormatBool(delete))
+	u.RawQuery = q.Encode()
+	return u.String(), nil
+}
+
+// Put - Adds DNS entries into operator webhook server
+func (c *OperatorDNS) Put(bucket string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultOperatorContextTimeout)
+	defer cancel()
+	e, err := c.endpoint(bucket, false)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, e, nil)
+	if err != nil {
+		return err
+	}
+	if err = c.addAuthHeader(req); err != nil {
+		return err
+	}
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		if derr := c.Delete(bucket); derr != nil {
+			return derr
+		}
+	}
+	xhttp.DrainBody(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("request to create the service for bucket %s, failed with status %s", bucket, resp.Status)
+	}
+	return nil
+}
+
+// Delete - Removes DNS entries added in Put().
+func (c *OperatorDNS) Delete(bucket string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultOperatorContextTimeout)
+	defer cancel()
+	e, err := c.endpoint(bucket, true)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, e, nil)
+	if err != nil {
+		return err
+	}
+	if err = c.addAuthHeader(req); err != nil {
+		return err
+	}
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+	xhttp.DrainBody(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("request to delete the service for bucket %s, failed with status %s", bucket, resp.Status)
+	}
+	return nil
+}
+
+// DeleteRecord - Removes a specific DNS entry
+// No Op for Operator because operator deals on with bucket entries
+func (c *OperatorDNS) DeleteRecord(record SrvRecord) error {
+	return ErrNotImplemented
+}
+
+// Close closes the internal http client
+func (c *OperatorDNS) Close() error {
+	c.httpClient.CloseIdleConnections()
+	return nil
+}
+
+// List - Retrieves list of DNS entries for the domain.
+// This is a No Op for Operator because, there is no intent to enforce global
+// namespace at MinIO level with this DNS entry. The global namespace in
+// enforced by the Kubernetes Operator
+func (c *OperatorDNS) List() (srvRecords map[string][]SrvRecord, err error) {
+	return nil, ErrNotImplemented
+}
+
+// Get - Retrieves DNS records for a bucket.
+// This is a No Op for Operator because, there is no intent to enforce global
+// namespace at MinIO level with this DNS entry. The global namespace in
+// enforced by the Kubernetes Operator
+func (c *OperatorDNS) Get(bucket string) (srvRecords []SrvRecord, err error) {
+	return nil, ErrNotImplemented
+}
+
+// String stringer name for this implementation of dns.Store
+func (c *OperatorDNS) String() string {
+	return "webhookDNS"
+}
+
+// OperatorDNS - represents dns config for MinIO k8s operator.
+type OperatorDNS struct {
+	httpClient *http.Client
+	Endpoint   string
+	rootCAs    *x509.CertPool
+	username   string
+	password   string
+}
+
+// OperatorOption - functional options pattern style for OperatorDNS
+type OperatorOption func(*OperatorDNS)
+
+// Authentication - custom username and password for authenticating at the endpoint
+func Authentication(username, password string) OperatorOption {
+	return func(args *OperatorDNS) {
+		args.username = username
+		args.password = password
+	}
+}
+
+// RootCAs - add custom trust certs pool
+func RootCAs(CAs *x509.CertPool) OperatorOption {
+	return func(args *OperatorDNS) {
+		args.rootCAs = CAs
+	}
+}
+
+// NewOperatorDNS - initialize a new K8S Operator DNS set/unset values.
+func NewOperatorDNS(endpoint string, setters ...OperatorOption) (Store, error) {
+	if endpoint == "" {
+		return nil, errors.New("invalid argument")
+	}
+
+	args := &OperatorDNS{
+		Endpoint: endpoint,
+	}
+	for _, setter := range setters {
+		setter(args)
+	}
+	args.httpClient = &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   3 * time.Second,
+				KeepAlive: 5 * time.Second,
+			}).DialContext,
+			ResponseHeaderTimeout: 3 * time.Second,
+			TLSHandshakeTimeout:   3 * time.Second,
+			ExpectContinueTimeout: 3 * time.Second,
+			TLSClientConfig: &tls.Config{
+				RootCAs: args.rootCAs,
+			},
+			// Go net/http automatically unzip if content-type is
+			// gzip disable this feature, as we are always interested
+			// in raw stream.
+			DisableCompression: true,
+		},
+	}
+	return args, nil
+}

cmd/config/dns/store.go

@@ -24,4 +24,5 @@ type Store interface {
 	List() (map[string][]SrvRecord, error)
 	DeleteRecord(record SrvRecord) error
 	Close() error
+	String() string
 }

cmd/config/identity/openid/jwt.go

@@ -25,6 +25,7 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	jwtgo "github.com/dgrijalva/jwt-go"
@@ -49,10 +50,14 @@ type Config struct {
 	publicKeys   map[string]crypto.PublicKey
 	transport    *http.Transport
 	closeRespFn  func(io.ReadCloser)
+	mutex        *sync.Mutex
 }
 
 // PopulatePublicKey - populates a new publickey from the JWKS URL.
 func (r *Config) PopulatePublicKey() error {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+
 	if r.JWKS.URL == nil || r.JWKS.URL.String() == "" {
 		return nil
 	}
@@ -185,7 +190,15 @@ func (p *JWT) Validate(token, dsecs string) (map[string]interface{}, error) {
 	var claims jwtgo.MapClaims
 	jwtToken, err := jp.ParseWithClaims(token, &claims, keyFuncCallback)
 	if err != nil {
-		return nil, err
+		// Re-populate the public key in-case the JWKS
+		// pubkeys are refreshed
+		if err = p.PopulatePublicKey(); err != nil {
+			return nil, err
+		}
+		jwtToken, err = jwtgo.ParseWithClaims(token, &claims, keyFuncCallback)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	if !jwtToken.Valid {
@@ -317,6 +330,7 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 		ClientID:    env.Get(EnvIdentityOpenIDClientID, kvs.Get(ClientID)),
 		transport:   transport,
 		closeRespFn: closeRespFn,
+		mutex:       &sync.Mutex{}, // allocate for copying
 	}
 
 	configURL := env.Get(EnvIdentityOpenIDURL, kvs.Get(ConfigURL))

cmd/config/identity/openid/jwt_test.go

@@ -20,6 +20,7 @@ import (
 	"crypto"
 	"encoding/json"
 	"net/url"
+	"sync"
 	"testing"
 	"time"
 
@@ -89,6 +90,7 @@ func TestJWTAzureFail(t *testing.T) {
 	}
 
 	cfg := Config{}
+	cfg.mutex = &sync.Mutex{}
 	cfg.JWKS.URL = u1
 	cfg.publicKeys = keys
 	jwt := NewJWT(cfg)
@@ -136,6 +138,7 @@ func TestJWT(t *testing.T) {
 	}
 
 	cfg := Config{}
+	cfg.mutex = &sync.Mutex{}
 	cfg.JWKS.URL = u1
 	cfg.publicKeys = keys
 	jwt := NewJWT(cfg)

cmd/config/logger.go

@@ -0,0 +1,27 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package config
+
+import "context"
+
+// Logger contains injected logger methods.
+var Logger = struct {
+	Info  func(msg string, data ...interface{})
+	LogIf func(ctx context.Context, err error, errKind ...interface{})
+}{
+	// Initialized via injection.
+}

cmd/config/notify/parse.go

@@ -43,11 +43,10 @@ var ErrTargetsOffline = errors.New("one or more targets are offline. Please use
 
 // TestNotificationTargets is similar to GetNotificationTargets()
 // avoids explicit registration.
-func TestNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport,
-	targetIDs []event.TargetID) error {
+func TestNotificationTargets(ctx context.Context, cfg config.Config, transport *http.Transport, targetIDs []event.TargetID) error {
 	test := true
 	returnOnTargetError := true
-	targets, err := RegisterNotificationTargets(cfg, doneCh, transport, targetIDs, test, returnOnTargetError)
+	targets, err := RegisterNotificationTargets(ctx, cfg, transport, targetIDs, test, returnOnTargetError)
 	if err == nil {
 		// Close all targets since we are only testing connections.
 		for _, t := range targets.TargetMap() {
@@ -60,9 +59,9 @@ func TestNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transpor
 
 // GetNotificationTargets registers and initializes all notification
 // targets, returns error if any.
-func GetNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, test bool) (*event.TargetList, error) {
+func GetNotificationTargets(ctx context.Context, cfg config.Config, transport *http.Transport, test bool) (*event.TargetList, error) {
 	returnOnTargetError := false
-	return RegisterNotificationTargets(cfg, doneCh, transport, nil, test, returnOnTargetError)
+	return RegisterNotificationTargets(ctx, cfg, transport, nil, test, returnOnTargetError)
 }
 
 // RegisterNotificationTargets - returns TargetList which contains enabled targets in serverConfig.
@@ -70,8 +69,8 @@ func GetNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport
 // * Add a new target in pkg/event/target package.
 // * Add newly added target configuration to serverConfig.Notify.<TARGET_NAME>.
 // * Handle the configuration in this function to create/add into TargetList.
-func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, targetIDs []event.TargetID, test bool, returnOnTargetError bool) (*event.TargetList, error) {
-	targetList, err := FetchRegisteredTargets(cfg, doneCh, transport, test, returnOnTargetError)
+func RegisterNotificationTargets(ctx context.Context, cfg config.Config, transport *http.Transport, targetIDs []event.TargetID, test bool, returnOnTargetError bool) (*event.TargetList, error) {
+	targetList, err := FetchRegisteredTargets(ctx, cfg, transport, test, returnOnTargetError)
 	if err != nil {
 		return targetList, err
 	}
@@ -94,7 +93,7 @@ func RegisterNotificationTargets(cfg config.Config, doneCh <-chan struct{}, tran
 // FetchRegisteredTargets - Returns a set of configured TargetList
 // If `returnOnTargetError` is set to true, The function returns when a target initialization fails
 // Else, the function will return a complete TargetList irrespective of errors
-func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport *http.Transport, test bool, returnOnTargetError bool) (_ *event.TargetList, err error) {
+func FetchRegisteredTargets(ctx context.Context, cfg config.Config, transport *http.Transport, test bool, returnOnTargetError bool) (_ *event.TargetList, err error) {
 	targetList := event.NewTargetList()
 	var targetsOffline bool
 
@@ -167,7 +166,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewAMQPTarget(id, args, doneCh, logger.LogOnceIf, test)
+		newTarget, err := target.NewAMQPTarget(id, args, ctx.Done(), logger.LogOnceIf, test)
 		if err != nil {
 			targetsOffline = true
 			if returnOnTargetError {
@@ -188,7 +187,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewElasticsearchTarget(id, args, doneCh, logger.LogOnceIf, test)
+		newTarget, err := target.NewElasticsearchTarget(id, args, ctx.Done(), logger.LogOnceIf, test)
 		if err != nil {
 			targetsOffline = true
 			if returnOnTargetError {
@@ -209,7 +208,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 			continue
 		}
 		args.TLS.RootCAs = transport.TLSClientConfig.RootCAs
-		newTarget, err := target.NewKafkaTarget(id, args, doneCh, logger.LogOnceIf, test)
+		newTarget, err := target.NewKafkaTarget(id, args, ctx.Done(), logger.LogOnceIf, test)
 		if err != nil {
 			targetsOffline = true
 			if returnOnTargetError {
@@ -230,7 +229,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 			continue
 		}
 		args.RootCAs = transport.TLSClientConfig.RootCAs
-		newTarget, err := target.NewMQTTTarget(id, args, doneCh, logger.LogOnceIf, test)
+		newTarget, err := target.NewMQTTTarget(id, args, ctx.Done(), logger.LogOnceIf, test)
 		if err != nil {
 			targetsOffline = true
 			if returnOnTargetError {
@@ -250,7 +249,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewMySQLTarget(id, args, doneCh, logger.LogOnceIf, test)
+		newTarget, err := target.NewMySQLTarget(id, args, ctx.Done(), logger.LogOnceIf, test)
 		if err != nil {
 			targetsOffline = true
 			if returnOnTargetError {
@@ -270,7 +269,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewNATSTarget(id, args, doneCh, logger.LogOnceIf, test)
+		newTarget, err := target.NewNATSTarget(id, args, ctx.Done(), logger.LogOnceIf, test)
 		if err != nil {
 			targetsOffline = true
 			if returnOnTargetError {
@@ -290,7 +289,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewNSQTarget(id, args, doneCh, logger.LogOnceIf, test)
+		newTarget, err := target.NewNSQTarget(id, args, ctx.Done(), logger.LogOnceIf, test)
 		if err != nil {
 			targetsOffline = true
 			if returnOnTargetError {
@@ -310,7 +309,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewPostgreSQLTarget(id, args, doneCh, logger.LogOnceIf, test)
+		newTarget, err := target.NewPostgreSQLTarget(id, args, ctx.Done(), logger.LogOnceIf, test)
 		if err != nil {
 			targetsOffline = true
 			if returnOnTargetError {
@@ -330,7 +329,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewRedisTarget(id, args, doneCh, logger.LogOnceIf, test)
+		newTarget, err := target.NewRedisTarget(id, args, ctx.Done(), logger.LogOnceIf, test)
 		if err != nil {
 			targetsOffline = true
 			if returnOnTargetError {
@@ -350,7 +349,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 		if !args.Enable {
 			continue
 		}
-		newTarget, err := target.NewWebhookTarget(id, args, doneCh, logger.LogOnceIf, transport, test)
+		newTarget, err := target.NewWebhookTarget(ctx, id, args, logger.LogOnceIf, transport, test)
 		if err != nil {
 			targetsOffline = true
 			if returnOnTargetError {

cmd/config/storageclass/storage-class.go

@@ -156,7 +156,7 @@ func parseStorageClass(storageClassEnv string) (sc StorageClass, err error) {
 }
 
 // Validates the parity disks.
-func validateParity(ssParity, rrsParity, drivesPerSet int) (err error) {
+func validateParity(ssParity, rrsParity, setDriveCount int) (err error) {
 	if ssParity == 0 && rrsParity == 0 {
 		return nil
 	}
@@ -174,12 +174,12 @@ func validateParity(ssParity, rrsParity, drivesPerSet int) (err error) {
 		return fmt.Errorf("Reduced redundancy storage class parity %d should be greater than or equal to %d", rrsParity, minParityDisks)
 	}
 
-	if ssParity > drivesPerSet/2 {
-		return fmt.Errorf("Standard storage class parity %d should be less than or equal to %d", ssParity, drivesPerSet/2)
+	if ssParity > setDriveCount/2 {
+		return fmt.Errorf("Standard storage class parity %d should be less than or equal to %d", ssParity, setDriveCount/2)
 	}
 
-	if rrsParity > drivesPerSet/2 {
-		return fmt.Errorf("Reduced redundancy storage class parity %d should be less than  or equal to %d", rrsParity, drivesPerSet/2)
+	if rrsParity > setDriveCount/2 {
+		return fmt.Errorf("Reduced redundancy storage class parity %d should be less than  or equal to %d", rrsParity, setDriveCount/2)
 	}
 
 	if ssParity > 0 && rrsParity > 0 {
@@ -220,9 +220,9 @@ func Enabled(kvs config.KVS) bool {
 }
 
 // LookupConfig - lookup storage class config and override with valid environment settings if any.
-func LookupConfig(kvs config.KVS, drivesPerSet int) (cfg Config, err error) {
+func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 	cfg = Config{}
-	cfg.Standard.Parity = drivesPerSet / 2
+	cfg.Standard.Parity = setDriveCount / 2
 	cfg.RRS.Parity = defaultRRSParity
 
 	if err = config.CheckValidKeys(config.StorageClassSubSys, kvs, DefaultKVS); err != nil {
@@ -239,7 +239,7 @@ func LookupConfig(kvs config.KVS, drivesPerSet int) (cfg Config, err error) {
 		}
 	}
 	if cfg.Standard.Parity == 0 {
-		cfg.Standard.Parity = drivesPerSet / 2
+		cfg.Standard.Parity = setDriveCount / 2
 	}
 
 	if rrsc != "" {
@@ -254,7 +254,7 @@ func LookupConfig(kvs config.KVS, drivesPerSet int) (cfg Config, err error) {
 
 	// Validation is done after parsing both the storage classes. This is needed because we need one
 	// storage class value to deduce the correct value of the other storage class.
-	if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, drivesPerSet); err != nil {
+	if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, setDriveCount); err != nil {
 		return Config{}, err
 	}
 

cmd/config/storageclass/storage-class_test.go

@@ -69,10 +69,10 @@ func TestParseStorageClass(t *testing.T) {
 
 func TestValidateParity(t *testing.T) {
 	tests := []struct {
-		rrsParity    int
-		ssParity     int
-		success      bool
-		drivesPerSet int
+		rrsParity     int
+		ssParity      int
+		success       bool
+		setDriveCount int
 	}{
 		{2, 4, true, 16},
 		{3, 3, true, 16},
@@ -85,7 +85,7 @@ func TestValidateParity(t *testing.T) {
 		{9, 2, false, 16},
 	}
 	for i, tt := range tests {
-		err := validateParity(tt.ssParity, tt.rrsParity, tt.drivesPerSet)
+		err := validateParity(tt.ssParity, tt.rrsParity, tt.setDriveCount)
 		if err != nil && tt.success {
 			t.Errorf("Test %d, Expected success, got %s", i+1, err)
 		}

cmd/consolelogger.go

@@ -122,6 +122,24 @@ func (sys *HTTPConsoleLoggerSys) Validate() error {
 	return nil
 }
 
+// Content returns the console stdout log
+func (sys *HTTPConsoleLoggerSys) Content() (logs []log.Entry) {
+	sys.RLock()
+	sys.logBuf.Do(func(p interface{}) {
+		if p != nil {
+			lg, ok := p.(log.Info)
+			if ok {
+				if (lg.Entry != log.Entry{}) {
+					logs = append(logs, lg.Entry)
+				}
+			}
+		}
+	})
+	sys.RUnlock()
+
+	return
+}
+
 // Send log message 'e' to console and publish to console
 // log pubsub system
 func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error {

cmd/crypto/config.go

@@ -16,11 +16,14 @@ package crypto
 
 import (
 	"errors"
+	"math/rand"
 	"net/http"
 	"reflect"
 	"strconv"
+	"strings"
 
 	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/ellipses"
 	"github.com/minio/minio/pkg/env"
 	xnet "github.com/minio/minio/pkg/net"
 )
@@ -167,7 +170,8 @@ const (
 
 const (
 	// EnvKMSKesEndpoint is the environment variable used to specify
-	// the kes server HTTPS endpoint.
+	// one or multiple KES server HTTPS endpoints. The individual
+	// endpoints should be separated by ','.
 	EnvKMSKesEndpoint = "MINIO_KMS_KES_ENDPOINT"
 
 	// EnvKMSKesKeyFile is the environment variable used to specify
@@ -216,16 +220,36 @@ func LookupKesConfig(kvs config.KVS) (KesConfig, error) {
 	kesCfg := KesConfig{}
 
 	endpointStr := env.Get(EnvKMSKesEndpoint, kvs.Get(KMSKesEndpoint))
-	if endpointStr != "" {
-		// Lookup kes configuration & overwrite config entry if ENV var is present
-		endpoint, err := xnet.ParseHTTPURL(endpointStr)
+	var endpoints []string
+	for _, endpoint := range strings.Split(endpointStr, ",") {
+		if strings.TrimSpace(endpoint) == "" {
+			continue
+		}
+		if !ellipses.HasEllipses(endpoint) {
+			endpoints = append(endpoints, endpoint)
+			continue
+		}
+		pattern, err := ellipses.FindEllipsesPatterns(endpoint)
 		if err != nil {
 			return kesCfg, err
 		}
-		endpointStr = endpoint.String()
+		for _, p := range pattern {
+			endpoints = append(endpoints, p.Expand()...)
+		}
+	}
+	if len(endpoints) == 0 {
+		return kesCfg, nil
 	}
 
-	kesCfg.Endpoint = endpointStr
+	randNum := rand.Intn(len(endpoints) + 1) // We add 1 b/c len(endpoints) may be 0: See: rand.Intn docs
+	kesCfg.Endpoint = make([]string, len(endpoints))
+	for i, endpoint := range endpoints {
+		endpoint, err := xnet.ParseHTTPURL(endpoint)
+		if err != nil {
+			return kesCfg, err
+		}
+		kesCfg.Endpoint[(randNum+i)%len(endpoints)] = endpoint.String()
+	}
 	kesCfg.KeyFile = env.Get(EnvKMSKesKeyFile, kvs.Get(KMSKesKeyFile))
 	kesCfg.CertFile = env.Get(EnvKMSKesCertFile, kvs.Get(KMSKesCertFile))
 	kesCfg.CAPath = env.Get(EnvKMSKesCAPath, kvs.Get(KMSKesCAPath))

cmd/crypto/kes.go

@@ -46,8 +46,8 @@ var ErrKESKeyExists = NewKESError(http.StatusBadRequest, "key does already exist
 type KesConfig struct {
 	Enabled bool
 
-	// The kes server endpoint.
-	Endpoint string
+	// The KES server endpoints.
+	Endpoint []string
 
 	// The path to the TLS private key used
 	// by MinIO to authenticate to the kes
@@ -86,7 +86,7 @@ type KesConfig struct {
 // Verify verifies if the kes configuration is correct
 func (k KesConfig) Verify() (err error) {
 	switch {
-	case k.Endpoint == "":
+	case len(k.Endpoint) == 0:
 		err = Errorf("crypto: missing kes endpoint")
 	case k.CertFile == "":
 		err = Errorf("crypto: missing cert file")
@@ -101,7 +101,7 @@ func (k KesConfig) Verify() (err error) {
 type kesService struct {
 	client *kesClient
 
-	endpoint     string
+	endpoints    []string
 	defaultKeyID string
 }
 
@@ -141,12 +141,12 @@ func NewKes(cfg KesConfig) (KMS, error) {
 
 	return &kesService{
 		client: &kesClient{
-			addr: cfg.Endpoint,
+			endpoints: cfg.Endpoint,
 			httpClient: http.Client{
 				Transport: cfg.Transport,
 			},
 		},
-		endpoint:     cfg.Endpoint,
+		endpoints:    cfg.Endpoint,
 		defaultKeyID: cfg.DefaultKeyID,
 	}, nil
 }
@@ -163,9 +163,9 @@ func (kes *kesService) DefaultKeyID() string {
 // method.
 func (kes *kesService) Info() KMSInfo {
 	return KMSInfo{
-		Endpoint: kes.endpoint,
-		Name:     kes.DefaultKeyID(),
-		AuthType: "TLS",
+		Endpoints: kes.endpoints,
+		Name:      kes.DefaultKeyID(),
+		AuthType:  "TLS",
 	}
 }
 
@@ -221,7 +221,7 @@ func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 //   • GenerateDataKey (API: /v1/key/generate/)
 //   • DecryptDataKey  (API: /v1/key/decrypt/)
 type kesClient struct {
-	addr       string
+	endpoints  []string
 	httpClient http.Client
 }
 
@@ -232,8 +232,8 @@ type kesClient struct {
 // application does not have the cryptographic key at
 // any point in time.
 func (c *kesClient) CreateKey(name string) error {
-	url := fmt.Sprintf("%s/v1/key/create/%s", c.addr, url.PathEscape(name))
-	_, err := c.postRetry(url, nil, 0) // No request body and no response expected
+	path := fmt.Sprintf("/v1/key/create/%s", url.PathEscape(name))
+	_, err := c.postRetry(path, nil, 0) // No request body and no response expected
 	if err != nil {
 		return err
 	}
@@ -265,8 +265,8 @@ func (c *kesClient) GenerateDataKey(name string, context []byte) ([]byte, []byte
 	}
 
 	const limit = 1 << 20 // A plaintext/ciphertext key pair will never be larger than 1 MB
-	url := fmt.Sprintf("%s/v1/key/generate/%s", c.addr, url.PathEscape(name))
-	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
+	path := fmt.Sprintf("/v1/key/generate/%s", url.PathEscape(name))
+	resp, err := c.postRetry(path, bytes.NewReader(body), limit)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -302,8 +302,8 @@ func (c *kesClient) DecryptDataKey(name string, ciphertext, context []byte) ([]b
 	}
 
 	const limit = 1 << 20 // A data key will never be larger than 1 MiB
-	url := fmt.Sprintf("%s/v1/key/decrypt/%s", c.addr, url.PathEscape(name))
-	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
+	path := fmt.Sprintf("/v1/key/decrypt/%s", url.PathEscape(name))
+	resp, err := c.postRetry(path, bytes.NewReader(body), limit)
 	if err != nil {
 		return nil, err
 	}
@@ -402,29 +402,34 @@ func (c *kesClient) post(url string, body io.Reader, limit int64) (io.Reader, er
 	return &respBody, nil
 }
 
-func (c *kesClient) postRetry(url string, body io.ReadSeeker, limit int64) (io.Reader, error) {
+func (c *kesClient) postRetry(path string, body io.ReadSeeker, limit int64) (io.Reader, error) {
+	retryMax := 1 + len(c.endpoints)
 	for i := 0; ; i++ {
 		if body != nil {
 			body.Seek(0, io.SeekStart) // seek to the beginning of the body.
 		}
-		response, err := c.post(url, body, limit)
+
+		response, err := c.post(c.endpoints[i%len(c.endpoints)]+path, body, limit)
 		if err == nil {
 			return response, nil
 		}
 
+		// If the error is not temp. / retryable => fail the request immediately.
 		if !xnet.IsNetworkOrHostDown(err) &&
 			!errors.Is(err, io.EOF) &&
 			!errors.Is(err, io.ErrUnexpectedEOF) &&
 			!errors.Is(err, context.DeadlineExceeded) {
 			return nil, err
 		}
-
-		// retriable network errors.
-		remain := retryMax - i
-		if remain <= 0 {
+		if remain := retryMax - i; remain <= 0 { // Fail if we exceeded our retry limit.
 			return response, err
 		}
 
+		// If there are more KES instances then skip waiting and
+		// try the next endpoint directly.
+		if i < len(c.endpoints) {
+			continue
+		}
 		<-time.After(LinearJitterBackoff(retryWaitMin, retryWaitMax, i))
 	}
 }

cmd/crypto/kms.go

@@ -109,9 +109,9 @@ type masterKeyKMS struct {
 // KMSInfo contains some describing information about
 // the KMS.
 type KMSInfo struct {
-	Endpoint string
-	Name     string
-	AuthType string
+	Endpoints []string
+	Name      string
+	AuthType  string
 }
 
 // NewMasterKey returns a basic KMS implementation from a single 256 bit master key.
@@ -147,9 +147,9 @@ func (kms *masterKeyKMS) GenerateKey(keyID string, ctx Context) (key [32]byte, s
 // KMS is configured directly using master key
 func (kms *masterKeyKMS) Info() (info KMSInfo) {
 	return KMSInfo{
-		Endpoint: "",
-		Name:     "",
-		AuthType: "master-key",
+		Endpoints: []string{},
+		Name:      "",
+		AuthType:  "master-key",
 	}
 }
 

cmd/crypto/metadata.go

@@ -129,7 +129,7 @@ func (ssec) IsEncrypted(metadata map[string]string) bool {
 // metadata is nil.
 func CreateMultipartMetadata(metadata map[string]string) map[string]string {
 	if metadata == nil {
-		metadata = map[string]string{}
+		return map[string]string{SSEMultipart: ""}
 	}
 	metadata[SSEMultipart] = ""
 	return metadata
@@ -156,7 +156,7 @@ func (s3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte
 	}
 
 	if metadata == nil {
-		metadata = map[string]string{}
+		metadata = make(map[string]string, 5)
 	}
 
 	metadata[SSESealAlgorithm] = sealedKey.Algorithm
@@ -236,7 +236,7 @@ func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[
 	}
 
 	if metadata == nil {
-		metadata = map[string]string{}
+		metadata = make(map[string]string, 3)
 	}
 	metadata[SSESealAlgorithm] = SealAlgorithm
 	metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])

cmd/crypto/retry.go

@@ -21,9 +21,8 @@ import (
 
 // default retry configuration
 const (
-	retryWaitMin = 500 * time.Millisecond // minimum retry limit.
-	retryWaitMax = 3 * time.Second        // 3 secs worth of max retry.
-	retryMax     = 2
+	retryWaitMin = 100 * time.Millisecond  // minimum retry limit.
+	retryWaitMax = 1500 * time.Millisecond // 1.5 secs worth of max retry.
 )
 
 // LinearJitterBackoff provides the time.Duration for a caller to

cmd/crypto/vault.go

@@ -199,13 +199,13 @@ func (v *vaultService) DefaultKeyID() string {
 }
 
 // Info returns some information about the Vault,
-// configuration - like the endpoint or authentication
+// configuration - like the endpoints or authentication
 // method.
 func (v *vaultService) Info() KMSInfo {
 	return KMSInfo{
-		Endpoint: v.config.Endpoint,
-		Name:     v.DefaultKeyID(),
-		AuthType: v.config.Auth.Type,
+		Endpoints: []string{v.config.Endpoint},
+		Name:      v.DefaultKeyID(),
+		AuthType:  v.config.Auth.Type,
 	}
 }
 

cmd/data-crawler.go

@@ -27,9 +27,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/minio/pkg/madmin"
-
 	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/crawler"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/bucket/replication"
@@ -37,6 +36,7 @@ import (
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/madmin"
 	"github.com/willf/bloom"
 )
 
@@ -46,7 +46,18 @@ const (
 	dataCrawlStartDelay      = 5 * time.Minute  // Time to wait on startup and between cycles.
 	dataUsageUpdateDirCycles = 16               // Visit all folders every n cycles.
 
-	healDeleteDangling = true
+	healDeleteDangling    = true
+	healFolderIncludeProb = 32  // Include a clean folder one in n cycles.
+	healObjectSelectProb  = 512 // Overall probability of a file being scanned; one in n.
+
+	// sleep for an hour after a lock timeout
+	// before retrying to acquire lock again.
+	dataCrawlerLeaderLockTimeoutSleepInterval = time.Hour
+)
+
+var (
+	globalCrawlerConfig          crawler.Config
+	dataCrawlerLeaderLockTimeout = newDynamicTimeout(1*time.Minute, 30*time.Second)
 )
 
 // initDataCrawler will start the crawler unless disabled.
@@ -60,6 +71,18 @@ func initDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 // The function will block until the context is canceled.
 // There should only ever be one crawler running per cluster.
 func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
+	// Make sure only 1 crawler is running on the cluster.
+	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "runDataCrawler.lock")
+	for {
+		err := locker.GetLock(dataCrawlerLeaderLockTimeout)
+		if err != nil {
+			time.Sleep(dataCrawlerLeaderLockTimeoutSleepInterval)
+			continue
+		}
+		break
+		// No unlock for "leader" lock.
+	}
+
 	// Load current bloom cycle
 	nextBloomCycle := intDataUpdateTracker.current() + 1
 	var buf bytes.Buffer
@@ -174,9 +197,9 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 	// Enable healing in XL mode.
 	if globalIsErasure {
 		// Include a clean folder one in n cycles.
-		s.healFolderInclude = 32
+		s.healFolderInclude = healFolderIncludeProb
 		// Do a heal check on an object once every n cycles. Must divide into healFolderInclude
-		s.healObjectSelect = 512
+		s.healObjectSelect = healObjectSelectProb
 	}
 	if len(cache.Info.BloomFilter) > 0 {
 		s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
@@ -327,7 +350,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 		// If there are lifecycle rules for the prefix, remove the filter.
 		filter := f.withFilter
 		var activeLifeCycle *lifecycle.Lifecycle
-		if f.oldCache.Info.lifeCycle != nil && filter != nil {
+		if f.oldCache.Info.lifeCycle != nil {
 			_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
 			if f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
 				if f.dataUsageCrawlDebug {
@@ -602,8 +625,9 @@ func (i *crawlItem) transformMetaDir() {
 
 // actionMeta contains information used to apply actions.
 type actionMeta struct {
-	oi          ObjectInfo
-	numVersions int // The number of versions of this object
+	oi               ObjectInfo
+	successorModTime time.Time // The modtime of the successor version
+	numVersions      int       // The number of versions of this object
 }
 
 // applyActions will apply lifecycle checks on to a scanned item.
@@ -623,7 +647,7 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return 0
 		}
-		if !errors.Is(err, NotImplemented{}) {
+		if err != nil && !errors.Is(err, NotImplemented{}) {
 			logger.LogIf(ctx, err)
 			return 0
 		}
@@ -636,13 +660,14 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	versionID := meta.oi.VersionID
 	action := i.lifeCycle.ComputeAction(
 		lifecycle.ObjectOpts{
-			Name:         i.objectPath(),
-			UserTags:     meta.oi.UserTags,
-			ModTime:      meta.oi.ModTime,
-			VersionID:    meta.oi.VersionID,
-			DeleteMarker: meta.oi.DeleteMarker,
-			IsLatest:     meta.oi.IsLatest,
-			NumVersions:  meta.numVersions,
+			Name:             i.objectPath(),
+			UserTags:         meta.oi.UserTags,
+			ModTime:          meta.oi.ModTime,
+			VersionID:        meta.oi.VersionID,
+			DeleteMarker:     meta.oi.DeleteMarker,
+			IsLatest:         meta.oi.IsLatest,
+			NumVersions:      meta.numVersions,
+			SuccessorModTime: meta.successorModTime,
 		})
 	if i.debug {
 		logger.Info(color.Green("applyActions:")+" lifecycle: %q (version-id=%s), Initial scan: %v", i.objectPath(), versionID, action)
@@ -679,13 +704,14 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	// Recalculate action.
 	action = i.lifeCycle.ComputeAction(
 		lifecycle.ObjectOpts{
-			Name:         i.objectPath(),
-			UserTags:     obj.UserTags,
-			ModTime:      obj.ModTime,
-			VersionID:    obj.VersionID,
-			DeleteMarker: obj.DeleteMarker,
-			IsLatest:     obj.IsLatest,
-			NumVersions:  meta.numVersions,
+			Name:             i.objectPath(),
+			UserTags:         obj.UserTags,
+			ModTime:          obj.ModTime,
+			VersionID:        obj.VersionID,
+			DeleteMarker:     obj.DeleteMarker,
+			IsLatest:         obj.IsLatest,
+			NumVersions:      meta.numVersions,
+			SuccessorModTime: meta.successorModTime,
 		})
 	if i.debug {
 		logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action)
@@ -756,9 +782,6 @@ func sleepDuration(d time.Duration, x float64) {
 func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, meta actionMeta) {
 	if meta.oi.ReplicationStatus == replication.Pending ||
 		meta.oi.ReplicationStatus == replication.Failed {
-		// if heal encounters a pending replication status, either replication
-		// has failed due to server shutdown or crawler and PutObject replication are in contention.
-		healPending := meta.oi.ReplicationStatus == replication.Pending
-		replicateObject(ctx, meta.oi.Bucket, meta.oi.Name, meta.oi.VersionID, o, nil, healPending)
+		globalReplicationState.queueReplicaTask(meta.oi)
 	}
 }

cmd/data-update-tracker.go

@@ -426,6 +426,8 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 	}
 	// Ignore what remains on the stream.
 	// Update d:
+	d.mu.Lock()
+	defer d.mu.Unlock()
 	d.Current = dst.Current
 	d.History = dst.History
 	d.Saved = dst.Saved

cmd/data-usage-cache.go

@@ -428,10 +428,15 @@ func (d *dataUsageCache) merge(other dataUsageCache) {
 	}
 }
 
+type objectIO interface {
+	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
+	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
+}
+
 // load the cache content with name from minioMetaBackgroundOpsBucket.
 // Only backend errors are returned as errors.
 // If the object is not found or unable to deserialize d is cleared and nil error is returned.
-func (d *dataUsageCache) load(ctx context.Context, store ObjectLayer, name string) error {
+func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error {
 	var buf bytes.Buffer
 	err := store.GetObject(ctx, dataUsageBucket, name, 0, -1, &buf, "", ObjectOptions{})
 	if err != nil {
@@ -450,7 +455,7 @@ func (d *dataUsageCache) load(ctx context.Context, store ObjectLayer, name strin
 }
 
 // save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
-func (d *dataUsageCache) save(ctx context.Context, store ObjectLayer, name string) error {
+func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) error {
 	b := d.serialize()
 	size := int64(len(b))
 	r, err := hash.NewReader(bytes.NewReader(b), size, "", "", size, false)

cmd/data-usage_test.go

@@ -240,7 +240,6 @@ func TestDataUsageUpdate(t *testing.T) {
 				t.Fatal("got nil result")
 			}
 			if w.flatten {
-				t.Log(e.Children)
 				*e = got.flatten(*e)
 			}
 			if e.Size != int64(w.size) {

cmd/disk-cache-backend.go

@@ -157,7 +157,7 @@ func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCa
 	}
 	cache := diskCache{
 		dir:           dir,
-		triggerGC:     make(chan struct{}),
+		triggerGC:     make(chan struct{}, 1),
 		stats:         CacheDiskStats{Dir: dir},
 		quotaPct:      quotaPct,
 		after:         config.After,
@@ -174,7 +174,7 @@ func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCa
 		nsMutex: newNSLock(false),
 	}
 	go cache.purgeWait(ctx)
-	cache.diskUsageHigh() // update if cache usage is already high.
+	cache.diskSpaceAvailable(0) // update if cache usage is already high.
 	cache.NewNSLockFn = func(ctx context.Context, cachePath string) RWLocker {
 		return cache.nsMutex.NewNSLock(ctx, nil, cachePath, "")
 	}
@@ -203,9 +203,9 @@ func (c *diskCache) diskUsageLow() bool {
 	return low
 }
 
-// Returns if the disk usage reaches high water mark w.r.t the configured cache quota.
-// gc starts if high water mark reached.
-func (c *diskCache) diskUsageHigh() bool {
+// Returns if the disk usage reaches  or exceeds configured cache quota when size is added.
+// If current usage without size exceeds high watermark a GC is automatically queued.
+func (c *diskCache) diskSpaceAvailable(size int64) bool {
 	gcTriggerPct := c.quotaPct * c.highWatermark / 100
 	di, err := disk.GetInfo(c.dir)
 	if err != nil {
@@ -214,27 +214,30 @@ func (c *diskCache) diskUsageHigh() bool {
 		logger.LogIf(ctx, err)
 		return false
 	}
-	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	high := int(usedPercent) >= gcTriggerPct
-	atomic.StoreUint64(&c.stats.UsagePercent, usedPercent)
-	if high {
-		atomic.StoreInt32(&c.stats.UsageState, 1)
-	}
-	return high
-}
-
-// Returns if size space can be allocated without exceeding
-// max disk usable for caching
-func (c *diskCache) diskAvailable(size int64) bool {
-	di, err := disk.GetInfo(c.dir)
-	if err != nil {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
-		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-		logger.LogIf(ctx, err)
+	if di.Total == 0 {
+		logger.Info("diskCache: Received 0 total disk size")
 		return false
 	}
-	usedPercent := (di.Total - (di.Free - uint64(size))) * 100 / di.Total
-	return int(usedPercent) < c.quotaPct
+	usedPercent := float64(di.Total-di.Free) * 100 / float64(di.Total)
+	if usedPercent >= float64(gcTriggerPct) {
+		atomic.StoreInt32(&c.stats.UsageState, 1)
+		c.queueGC()
+	}
+	atomic.StoreUint64(&c.stats.UsagePercent, uint64(usedPercent))
+
+	// Recalculate percentage with provided size added.
+	usedPercent = float64(di.Total-di.Free+uint64(size)) * 100 / float64(di.Total)
+
+	return usedPercent < float64(c.quotaPct)
+}
+
+// queueGC will queue a GC.
+// Calling this function is always non-blocking.
+func (c *diskCache) queueGC() {
+	select {
+	case c.triggerGC <- struct{}{}:
+	default:
+	}
 }
 
 // toClear returns how many bytes should be cleared to reach the low watermark quota.
@@ -247,7 +250,7 @@ func (c *diskCache) toClear() uint64 {
 		logger.LogIf(ctx, err)
 		return 0
 	}
-	return bytesToClear(int64(di.Total), int64(di.Free), uint64(c.quotaPct), uint64(c.lowWatermark))
+	return bytesToClear(int64(di.Total), int64(di.Free), uint64(c.quotaPct), uint64(c.lowWatermark), uint64(c.highWatermark))
 }
 
 var (
@@ -658,8 +661,7 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
 
 // Caches the object to disk
 func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) error {
-	if c.diskUsageHigh() {
-		c.triggerGC <- struct{}{}
+	if !c.diskSpaceAvailable(size) {
 		io.Copy(ioutil.Discard, data)
 		return errDiskFull
 	}
@@ -688,16 +690,13 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 	if rs != nil {
 		return c.putRange(ctx, bucket, object, data, size, rs, opts)
 	}
-	if !c.diskAvailable(size) {
+	if !c.diskSpaceAvailable(size) {
 		return errDiskFull
 	}
 	if err := os.MkdirAll(cachePath, 0777); err != nil {
 		return err
 	}
-	var metadata = make(map[string]string)
-	for k, v := range opts.UserDefined {
-		metadata[k] = v
-	}
+	var metadata = cloneMSS(opts.UserDefined)
 	var reader = data
 	var actualSize = uint64(size)
 	if globalCacheKMS != nil {
@@ -719,7 +718,7 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 
 	if actualSize != uint64(n) {
 		removeAll(cachePath)
-		return IncompleteBody{}
+		return IncompleteBody{Bucket: bucket, Object: object}
 	}
 	return c.saveMetadata(ctx, bucket, object, metadata, n, nil, "", incHitsOnly)
 }
@@ -730,17 +729,14 @@ func (c *diskCache) putRange(ctx context.Context, bucket, object string, data io
 	if err != nil {
 		return err
 	}
-	if !c.diskAvailable(rlen) {
+	if !c.diskSpaceAvailable(rlen) {
 		return errDiskFull
 	}
 	cachePath := getCacheSHADir(c.dir, bucket, object)
 	if err := os.MkdirAll(cachePath, 0777); err != nil {
 		return err
 	}
-	var metadata = make(map[string]string)
-	for k, v := range opts.UserDefined {
-		metadata[k] = v
-	}
+	var metadata = cloneMSS(opts.UserDefined)
 	var reader = data
 	var actualSize = uint64(rlen)
 	// objSize is the actual size of object (with encryption overhead if any)
@@ -766,7 +762,7 @@ func (c *diskCache) putRange(ctx context.Context, bucket, object string, data io
 	}
 	if actualSize != uint64(n) {
 		removeAll(cachePath)
-		return IncompleteBody{}
+		return IncompleteBody{Bucket: bucket, Object: object}
 	}
 	return c.saveMetadata(ctx, bucket, object, metadata, int64(objSize), rs, cacheFile, false)
 }

cmd/disk-cache-utils.go

@@ -489,9 +489,15 @@ func (f *fileScorer) queueString() string {
 // bytesToClear() returns the number of bytes to clear to reach low watermark
 // w.r.t quota given disk total and free space, quota in % allocated to cache
 // and low watermark % w.r.t allowed quota.
-func bytesToClear(total, free int64, quotaPct, lowWatermark uint64) uint64 {
-	used := (total - free)
+// If the high watermark hasn't been reached 0 will be returned.
+func bytesToClear(total, free int64, quotaPct, lowWatermark, highWatermark uint64) uint64 {
+	used := total - free
 	quotaAllowed := total * (int64)(quotaPct) / 100
-	lowWMUsage := (total * (int64)(lowWatermark*quotaPct) / (100 * 100))
+	highWMUsage := total * (int64)(highWatermark*quotaPct) / (100 * 100)
+	if used < highWMUsage {
+		return 0
+	}
+	// Return bytes needed to reach low watermark.
+	lowWMUsage := total * (int64)(lowWatermark*quotaPct) / (100 * 100)
 	return (uint64)(math.Min(float64(quotaAllowed), math.Max(0.0, float64(used-lowWMUsage))))
 }

cmd/disk-cache-utils_test.go

@@ -149,22 +149,26 @@ func TestNewFileScorer(t *testing.T) {
 }
 func TestBytesToClear(t *testing.T) {
 	testCases := []struct {
-		total        int64
-		free         int64
-		quotaPct     uint64
-		watermarkLow uint64
-		expected     uint64
+		total         int64
+		free          int64
+		quotaPct      uint64
+		watermarkLow  uint64
+		watermarkHigh uint64
+		expected      uint64
 	}{
-		{1000, 800, 40, 90, 0},
-		{1000, 200, 40, 90, 400},
-		{1000, 400, 40, 90, 240},
-		{1000, 600, 40, 90, 40},
-		{1000, 600, 40, 70, 120},
-		{1000, 1000, 90, 70, 0},
-		{1000, 0, 90, 70, 370},
+		{total: 1000, free: 800, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 0},
+		{total: 1000, free: 200, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 400},
+		{total: 1000, free: 400, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 240},
+		{total: 1000, free: 600, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 40},
+		{total: 1000, free: 600, quotaPct: 40, watermarkLow: 70, watermarkHigh: 70, expected: 120},
+		{total: 1000, free: 1000, quotaPct: 90, watermarkLow: 70, watermarkHigh: 70, expected: 0},
+
+		// High not yet reached..
+		{total: 1000, free: 250, quotaPct: 100, watermarkLow: 50, watermarkHigh: 90, expected: 0},
+		{total: 1000, free: 250, quotaPct: 100, watermarkLow: 50, watermarkHigh: 90, expected: 0},
 	}
 	for i, tc := range testCases {
-		toClear := bytesToClear(tc.total, tc.free, tc.quotaPct, tc.watermarkLow)
+		toClear := bytesToClear(tc.total, tc.free, tc.quotaPct, tc.watermarkLow, tc.watermarkHigh)
 		if tc.expected != toClear {
 			t.Errorf("test %d expected %v, got %v", i, tc.expected, toClear)
 		}

cmd/disk-cache.go

@@ -85,16 +85,15 @@ type cacheObjects struct {
 }
 
 func (c *cacheObjects) incHitsToMeta(ctx context.Context, dcache *diskCache, bucket, object string, size int64, eTag string, rs *HTTPRangeSpec) error {
-	metadata := make(map[string]string)
-	metadata["etag"] = eTag
+	metadata := map[string]string{"etag": eTag}
 	return dcache.SaveMetadata(ctx, bucket, object, metadata, size, rs, "", true)
 }
 
 // Backend metadata could have changed through server side copy - reset cache metadata if that is the case
 func (c *cacheObjects) updateMetadataIfChanged(ctx context.Context, dcache *diskCache, bucket, object string, bkObjectInfo, cacheObjInfo ObjectInfo, rs *HTTPRangeSpec) error {
 
-	bkMeta := make(map[string]string)
-	cacheMeta := make(map[string]string)
+	bkMeta := make(map[string]string, len(bkObjectInfo.UserDefined))
+	cacheMeta := make(map[string]string, len(cacheObjInfo.UserDefined))
 	for k, v := range bkObjectInfo.UserDefined {
 		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 			// Do not need to send any internal metadata
@@ -166,7 +165,7 @@ func (c *cacheObjects) DeleteObjects(ctx context.Context, bucket string, objects
 
 // construct a metadata k-v map
 func getMetadata(objInfo ObjectInfo) map[string]string {
-	metadata := make(map[string]string)
+	metadata := make(map[string]string, len(objInfo.UserDefined)+4)
 	metadata["etag"] = objInfo.ETag
 	metadata["content-type"] = objInfo.ContentType
 	if objInfo.ContentEncoding != "" {
@@ -284,12 +283,6 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	// Reaching here implies cache miss
 	c.cacheStats.incMiss()
 
-	// Since we got here, we are serving the request from backend,
-	// and also adding the object to the cache.
-	if dcache.diskUsageHigh() {
-		dcache.triggerGC <- struct{}{} // this is non-blocking
-	}
-
 	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 
 	if bkErr != nil {
@@ -306,7 +299,9 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	if cacheErr == nil {
 		bkReader.ObjInfo.CacheLookupStatus = CacheHit
 	}
-	if !dcache.diskAvailable(objInfo.Size) {
+
+	// Check if we can add it without exceeding total cache size.
+	if !dcache.diskSpaceAvailable(objInfo.Size) {
 		return bkReader, bkErr
 	}
 
@@ -338,11 +333,12 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	// Initialize pipe.
 	pipeReader, pipeWriter := io.Pipe()
 	teeReader := io.TeeReader(bkReader, pipeWriter)
+	userDefined := getMetadata(bkReader.ObjInfo)
 	go func() {
 		putErr := dcache.Put(ctx, bucket, object,
 			io.LimitReader(pipeReader, bkReader.ObjInfo.Size),
 			bkReader.ObjInfo.Size, nil, ObjectOptions{
-				UserDefined: getMetadata(bkReader.ObjInfo),
+				UserDefined: userDefined,
 			}, false)
 		// close the write end of the pipe, so the error gets
 		// propagated to getObjReader
@@ -612,9 +608,10 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 	}
 
 	// fetch from backend if there is no space on cache drive
-	if !dcache.diskAvailable(size) {
+	if !dcache.diskSpaceAvailable(size) {
 		return putObjectFn(ctx, bucket, object, r, opts)
 	}
+
 	if opts.ServerSideEncryption != nil {
 		dcache.Delete(ctx, bucket, object)
 		return putObjectFn(ctx, bucket, object, r, opts)
@@ -721,7 +718,9 @@ func (c *cacheObjects) gc(ctx context.Context) {
 			}
 			for _, dcache := range c.cache {
 				if dcache != nil {
-					dcache.triggerGC <- struct{}{}
+					// Check if there is disk.
+					// Will queue a GC scan if at high watermark.
+					dcache.diskSpaceAvailable(0)
 				}
 			}
 		}

cmd/dynamic-timeouts.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"math"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -26,7 +27,8 @@ const (
 	dynamicTimeoutIncreaseThresholdPct = 0.33 // Upper threshold for failures in order to increase timeout
 	dynamicTimeoutDecreaseThresholdPct = 0.10 // Lower threshold for failures in order to decrease timeout
 	dynamicTimeoutLogSize              = 16
-	maxDuration                        = 1<<63 - 1
+	maxDuration                        = math.MaxInt64
+	maxDynamicTimeout                  = 24 * time.Hour // Never set timeout bigger than this.
 )
 
 // timeouts that are dynamically adapted based on actual usage results
@@ -40,6 +42,12 @@ type dynamicTimeout struct {
 
 // newDynamicTimeout returns a new dynamic timeout initialized with timeout value
 func newDynamicTimeout(timeout, minimum time.Duration) *dynamicTimeout {
+	if timeout <= 0 || minimum <= 0 {
+		panic("newDynamicTimeout: negative or zero timeout")
+	}
+	if minimum > timeout {
+		minimum = timeout
+	}
 	return &dynamicTimeout{timeout: int64(timeout), minimum: int64(minimum)}
 }
 
@@ -61,60 +69,73 @@ func (dt *dynamicTimeout) LogFailure() {
 
 // logEntry stores a log entry
 func (dt *dynamicTimeout) logEntry(duration time.Duration) {
+	if duration < 0 {
+		return
+	}
 	entries := int(atomic.AddInt64(&dt.entries, 1))
 	index := entries - 1
 	if index < dynamicTimeoutLogSize {
 		dt.mutex.Lock()
 		dt.log[index] = duration
+
+		// We leak entries while we copy
+		if entries == dynamicTimeoutLogSize {
+
+			// Make copy on stack in order to call adjust()
+			logCopy := [dynamicTimeoutLogSize]time.Duration{}
+			copy(logCopy[:], dt.log[:])
+
+			// reset log entries
+			atomic.StoreInt64(&dt.entries, 0)
+			dt.mutex.Unlock()
+
+			dt.adjust(logCopy)
+			return
+		}
 		dt.mutex.Unlock()
 	}
-	if entries == dynamicTimeoutLogSize {
-		dt.mutex.Lock()
-
-		// Make copy on stack in order to call adjust()
-		logCopy := [dynamicTimeoutLogSize]time.Duration{}
-		copy(logCopy[:], dt.log[:])
-
-		// reset log entries
-		atomic.StoreInt64(&dt.entries, 0)
-
-		dt.mutex.Unlock()
-
-		dt.adjust(logCopy)
-	}
 }
 
 // adjust changes the value of the dynamic timeout based on the
 // previous results
 func (dt *dynamicTimeout) adjust(entries [dynamicTimeoutLogSize]time.Duration) {
-
-	failures, average := 0, int64(0)
-	for i := 0; i < len(entries); i++ {
-		if entries[i] == maxDuration {
+	failures, max := 0, time.Duration(0)
+	for _, dur := range entries[:] {
+		if dur == maxDuration {
 			failures++
-		} else {
-			average += int64(entries[i])
+		} else if dur > max {
+			max = dur
 		}
 	}
-	if failures < len(entries) {
-		average /= int64(len(entries) - failures)
-	}
 
-	timeOutHitPct := float64(failures) / float64(len(entries))
+	failPct := float64(failures) / float64(len(entries))
 
-	if timeOutHitPct > dynamicTimeoutIncreaseThresholdPct {
+	if failPct > dynamicTimeoutIncreaseThresholdPct {
 		// We are hitting the timeout too often, so increase the timeout by 25%
 		timeout := atomic.LoadInt64(&dt.timeout) * 125 / 100
-		atomic.StoreInt64(&dt.timeout, timeout)
-	} else if timeOutHitPct < dynamicTimeoutDecreaseThresholdPct {
-		// We are hitting the timeout relatively few times, so decrease the timeout
-		average = average * 125 / 100 // Add buffer of 25% on top of average
 
-		timeout := (atomic.LoadInt64(&dt.timeout) + average) / 2 // Middle between current timeout and average success
+		// Set upper cap.
+		if timeout > int64(maxDynamicTimeout) {
+			timeout = int64(maxDynamicTimeout)
+		}
+		// Safety, shouldn't happen
+		if timeout < dt.minimum {
+			timeout = dt.minimum
+		}
+		atomic.StoreInt64(&dt.timeout, timeout)
+	} else if failPct < dynamicTimeoutDecreaseThresholdPct {
+		// We are hitting the timeout relatively few times,
+		// so decrease the timeout towards 25 % of maximum time spent.
+		max = max * 125 / 100
+
+		timeout := atomic.LoadInt64(&dt.timeout)
+		if max < time.Duration(timeout) {
+			// Move 50% toward the max.
+			timeout = (int64(max) + timeout) / 2
+		}
 		if timeout < dt.minimum {
 			timeout = dt.minimum
 		}
 		atomic.StoreInt64(&dt.timeout, timeout)
 	}
-
 }

cmd/dynamic-timeouts_test.go

@@ -18,6 +18,8 @@ package cmd
 
 import (
 	"math/rand"
+	"runtime"
+	"sync"
 	"testing"
 	"time"
 )
@@ -98,7 +100,7 @@ func TestDynamicTimeoutDualDecrease(t *testing.T) {
 	adjustedAgain := timeout.Timeout()
 
 	if initial <= adjusted || adjusted <= adjustedAgain {
-		t.Errorf("Failure to decrease timeout multiple times")
+		t.Errorf("Failure to decrease timeout multiple times, initial: %v, adjusted: %v, again: %v", initial, adjusted, adjustedAgain)
 	}
 }
 
@@ -123,6 +125,30 @@ func TestDynamicTimeoutManyDecreases(t *testing.T) {
 	}
 }
 
+func TestDynamicTimeoutConcurrent(t *testing.T) {
+	// Race test.
+	timeout := newDynamicTimeout(time.Second, time.Millisecond)
+	var wg sync.WaitGroup
+	for i := 0; i < runtime.GOMAXPROCS(0); i++ {
+		wg.Add(1)
+		rng := rand.New(rand.NewSource(int64(i)))
+		go func() {
+			defer wg.Done()
+			for i := 0; i < 100; i++ {
+				timeout.LogFailure()
+				for j := 0; j < 100; j++ {
+					timeout.LogSuccess(time.Duration(float64(time.Second) * rng.Float64()))
+				}
+				to := timeout.Timeout()
+				if to < time.Millisecond || to > time.Second {
+					panic(to)
+				}
+			}
+		}()
+	}
+	wg.Wait()
+}
+
 func TestDynamicTimeoutHitMinimum(t *testing.T) {
 
 	const minimum = 30 * time.Second
@@ -168,7 +194,7 @@ func TestDynamicTimeoutAdjustExponential(t *testing.T) {
 
 	timeout := newDynamicTimeout(time.Minute, time.Second)
 
-	rand.Seed(time.Now().UTC().UnixNano())
+	rand.Seed(0)
 
 	initial := timeout.Timeout()
 
@@ -188,7 +214,7 @@ func TestDynamicTimeoutAdjustNormalized(t *testing.T) {
 
 	timeout := newDynamicTimeout(time.Minute, time.Second)
 
-	rand.Seed(time.Now().UTC().UnixNano())
+	rand.Seed(0)
 
 	initial := timeout.Timeout()
 

cmd/encryption-v1.go

@@ -388,12 +388,7 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
 		object:            object,
 		customerKeyHeader: h.Get(crypto.SSECKey),
 		copySource:        copySource,
-	}
-
-	w.metadata = map[string]string{}
-	// Copy encryption metadata for internal use.
-	for k, v := range oi.UserDefined {
-		w.metadata[k] = v
+		metadata:          cloneMSS(oi.UserDefined),
 	}
 
 	if w.copySource {
@@ -432,10 +427,7 @@ type DecryptBlocksReader struct {
 }
 
 func (d *DecryptBlocksReader) buildDecrypter(partID int) error {
-	m := make(map[string]string)
-	for k, v := range d.metadata {
-		m[k] = v
-	}
+	m := cloneMSS(d.metadata)
 	// Initialize the first decrypter; new decrypters will be
 	// initialized in Read() operation as needed.
 	var key []byte

cmd/endpoint.go

@@ -24,6 +24,7 @@ import (
 	"net/url"
 	"path"
 	"path/filepath"
+	"reflect"
 	"runtime"
 	"strconv"
 	"strings"
@@ -193,7 +194,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 }
 
 // ZoneEndpoints represent endpoints in a given zone
-// along with its setCount and drivesPerSet.
+// along with its setCount and setDriveCount.
 type ZoneEndpoints struct {
 	SetCount     int
 	DrivesPerSet int
@@ -203,6 +204,21 @@ type ZoneEndpoints struct {
 // EndpointZones - list of list of endpoints
 type EndpointZones []ZoneEndpoints
 
+// GetLocalZoneIdx returns the zone which endpoint belongs to locally.
+// if ep is remote this code will return -1 zoneIndex
+func (l EndpointZones) GetLocalZoneIdx(ep Endpoint) int {
+	for i, zep := range l {
+		for _, cep := range zep.Endpoints {
+			if cep.IsLocal && ep.IsLocal {
+				if reflect.DeepEqual(cep, ep) {
+					return i
+				}
+			}
+		}
+	}
+	return -1
+}
+
 // Add add zone endpoints
 func (l *EndpointZones) Add(zeps ZoneEndpoints) error {
 	existSet := set.NewStringSet()
@@ -753,9 +769,16 @@ func GetProxyEndpoints(endpointZones EndpointZones) ([]ProxyEndpoint, error) {
 					RootCAs:    globalRootCAs,
 				}
 			}
+
+			tr := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)()
+			// Allow more requests to be in flight with higher response header timeout.
+			tr.ResponseHeaderTimeout = 30 * time.Minute
+			tr.MaxIdleConns = 64
+			tr.MaxIdleConnsPerHost = 64
+
 			proxyEps = append(proxyEps, ProxyEndpoint{
 				Endpoint:  endpoint,
-				Transport: newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)(),
+				Transport: tr,
 			})
 		}
 	}

cmd/erasure-bucket.go

@@ -49,7 +49,7 @@ func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket stri
 		index := index
 		g.Go(func() error {
 			if storageDisks[index] != nil {
-				if err := storageDisks[index].MakeVol(bucket); err != nil {
+				if err := storageDisks[index].MakeVol(ctx, bucket); err != nil {
 					if err != errVolumeExists {
 						logger.LogIf(ctx, err)
 					}
@@ -75,7 +75,7 @@ func undoDeleteBucket(storageDisks []StorageAPI, bucket string) {
 		}
 		index := index
 		g.Go(func() error {
-			_ = storageDisks[index].MakeVol(bucket)
+			_ = storageDisks[index].MakeVol(context.Background(), bucket)
 			return nil
 		}, index)
 	}
@@ -92,7 +92,7 @@ func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (
 			bucketErrs = append(bucketErrs, errDiskNotFound)
 			continue
 		}
-		volInfo, serr := disk.StatVol(bucketName)
+		volInfo, serr := disk.StatVol(ctx, bucketName)
 		if serr == nil {
 			return BucketInfo(volInfo), nil
 		}
@@ -129,12 +129,11 @@ func (er erasureObjects) listBuckets(ctx context.Context) (bucketsInfo []BucketI
 			continue
 		}
 		var volsInfo []VolInfo
-		volsInfo, err = disk.ListVols()
+		volsInfo, err = disk.ListVols(ctx)
 		if err == nil {
 			// NOTE: The assumption here is that volumes across all disks in
 			// readQuorum have consistent view i.e they all have same number
-			// of buckets. This is essentially not verified since healing
-			// should take care of this.
+			// of buckets.
 			var bucketsInfo []BucketInfo
 			for _, volInfo := range volsInfo {
 				if isReservedOrInvalidBucket(volInfo.Name, true) {
@@ -182,10 +181,10 @@ func deleteDanglingBucket(ctx context.Context, storageDisks []StorageAPI, dErrs
 	for index, err := range dErrs {
 		if err == errVolumeNotEmpty {
 			// Attempt to delete bucket again.
-			if derr := storageDisks[index].DeleteVol(bucket, false); derr == errVolumeNotEmpty {
+			if derr := storageDisks[index].DeleteVol(ctx, bucket, false); derr == errVolumeNotEmpty {
 				_ = cleanupDir(ctx, storageDisks[index], bucket, "")
 
-				_ = storageDisks[index].DeleteVol(bucket, false)
+				_ = storageDisks[index].DeleteVol(ctx, bucket, false)
 
 				// Cleanup all the previously incomplete multiparts.
 				_ = cleanupDir(ctx, storageDisks[index], minioMetaMultipartBucket, bucket)
@@ -205,7 +204,7 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceD
 		index := index
 		g.Go(func() error {
 			if storageDisks[index] != nil {
-				if err := storageDisks[index].DeleteVol(bucket, forceDelete); err != nil {
+				if err := storageDisks[index].DeleteVol(ctx, bucket, forceDelete); err != nil {
 					return err
 				}
 				err := cleanupDir(ctx, storageDisks[index], minioMetaMultipartBucket, bucket)
@@ -244,7 +243,10 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceD
 	// If we reduce quorum to nil, means we have deleted buckets properly
 	// on some servers in quorum, we should look for volumeNotEmpty errors
 	// and delete those buckets as well.
-	deleteDanglingBucket(ctx, storageDisks, dErrs, bucket)
+	//
+	// let this call succeed, even if client cancels the context
+	// this is to ensure that we don't leave any stale content
+	deleteDanglingBucket(context.Background(), storageDisks, dErrs, bucket)
 
 	return nil
 }

cmd/erasure-common.go

@@ -23,6 +23,35 @@ import (
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
+func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
+	disks := er.getDisks()
+	// Based on the random shuffling return back randomized disks.
+	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
+		if disks[i-1] != nil && disks[i-1].IsLocal() {
+			newDisks = append(newDisks, disks[i-1])
+		}
+	}
+	return newDisks
+}
+
+// getLoadBalancedNDisks - fetches load balanced (sufficiently randomized) disk slice
+// with N disks online. If ndisks is zero or negative, then it will returns all disks,
+// same if ndisks is greater than the number of all disks.
+func (er erasureObjects) getLoadBalancedNDisks(ndisks int) (newDisks []StorageAPI) {
+	disks := er.getLoadBalancedDisks()
+	for _, disk := range disks {
+		if disk == nil {
+			continue
+		}
+		newDisks = append(newDisks, disk)
+		ndisks--
+		if ndisks == 0 {
+			break
+		}
+	}
+	return
+}
+
 // getLoadBalancedDisks - fetches load balanced (sufficiently randomized) disk slice.
 func (er erasureObjects) getLoadBalancedDisks() (newDisks []StorageAPI) {
 	disks := er.getDisks()
@@ -66,7 +95,7 @@ func (er erasureObjects) isObject(ctx context.Context, bucket, prefix string) (o
 				return errDiskNotFound
 			}
 			// Check if 'prefix' is an object on this 'disk', else continue the check the next disk
-			return storageDisks[index].CheckFile(bucket, prefix)
+			return storageDisks[index].CheckFile(ctx, bucket, prefix)
 		}, index)
 	}
 

cmd/erasure-common_test.go

@@ -32,6 +32,7 @@ func TestErasureParentDirIsObject(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to initialize 'Erasure' object layer.")
 	}
+	defer obj.Shutdown(context.Background())
 
 	// Remove all disks.
 	for _, disk := range fsDisks {

cmd/erasure-decode_test.go

@@ -28,7 +28,7 @@ import (
 	humanize "github.com/dustin/go-humanize"
 )
 
-func (a badDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
+func (a badDisk) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
 	return 0, errFaultyDisk
 }
 

cmd/erasure-encode_test.go

@@ -32,11 +32,11 @@ func (a badDisk) String() string {
 	return "bad-disk"
 }
 
-func (a badDisk) AppendFile(volume string, path string, buf []byte) error {
+func (a badDisk) AppendFile(ctx context.Context, volume string, path string, buf []byte) error {
 	return errFaultyDisk
 }
 
-func (a badDisk) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) {
+func (a badDisk) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
 	return nil, errFaultyDisk
 }
 
@@ -44,7 +44,7 @@ func (a badDisk) UpdateBloomFilter(ctx context.Context, oldest, current uint64)
 	return nil, errFaultyDisk
 }
 
-func (a badDisk) CreateFile(volume, path string, size int64, reader io.Reader) error {
+func (a badDisk) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
 	return errFaultyDisk
 }
 
@@ -195,7 +195,7 @@ func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64,
 			if disk == OfflineDisk {
 				continue
 			}
-			disk.DeleteFile("testbucket", "object")
+			disk.DeleteFile(context.Background(), "testbucket", "object")
 			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		_, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)

cmd/erasure-healing-common.go

@@ -175,9 +175,9 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			// disk has a valid xl.meta but may not have all the
 			// parts. This is considered an outdated disk, since
 			// it needs healing too.
-			dataErrs[i] = onlineDisk.VerifyFile(bucket, object, partsMetadata[i])
+			dataErrs[i] = onlineDisk.VerifyFile(ctx, bucket, object, partsMetadata[i])
 		case madmin.HealNormalScan:
-			dataErrs[i] = onlineDisk.CheckParts(bucket, object, partsMetadata[i])
+			dataErrs[i] = onlineDisk.CheckParts(ctx, bucket, object, partsMetadata[i])
 		}
 
 		if dataErrs[i] == nil {

cmd/erasure-healing-common_test.go

@@ -99,6 +99,7 @@ func TestListOnlineDisks(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Prepare Erasure backend failed - %v", err)
 	}
+	defer obj.Shutdown(context.Background())
 	defer removeRoots(disks)
 
 	type tamperKind int
@@ -210,7 +211,7 @@ func TestListOnlineDisks(t *testing.T) {
 				// and check if that disk
 				// appears in outDatedDisks.
 				tamperedIndex = index
-				dErr := erasureDisks[index].DeleteFile(bucket, pathJoin(object, fi.DataDir, "part.1"))
+				dErr := erasureDisks[index].DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 				if dErr != nil {
 					t.Fatalf("Test %d: Failed to delete %s - %v", i+1,
 						filepath.Join(object, "part.1"), dErr)
@@ -265,6 +266,7 @@ func TestDisksWithAllParts(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Prepare Erasure backend failed - %v", err)
 	}
+	defer obj.Shutdown(context.Background())
 	defer removeRoots(disks)
 
 	bucket := "bucket"

cmd/erasure-healing.go

@@ -29,16 +29,6 @@ import (
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
-func (er erasureObjects) ReloadFormat(ctx context.Context, dryRun bool) error {
-	logger.LogIf(ctx, NotImplemented{})
-	return NotImplemented{}
-}
-
-func (er erasureObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) {
-	logger.LogIf(ctx, NotImplemented{})
-	return madmin.HealResultItem{}, NotImplemented{}
-}
-
 // Heals a bucket if it doesn't exist on one of the disks, additionally
 // also heals the missing entries for bucket metadata files
 // `policy.json, notification.xml, listeners.json`.
@@ -78,7 +68,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
 				afterState[index] = madmin.DriveStateOffline
 				return errDiskNotFound
 			}
-			if _, serr := storageDisks[index].StatVol(bucket); serr != nil {
+			if _, serr := storageDisks[index].StatVol(ctx, bucket); serr != nil {
 				if serr == errDiskNotFound {
 					beforeState[index] = madmin.DriveStateOffline
 					afterState[index] = madmin.DriveStateOffline
@@ -136,7 +126,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
 		index := index
 		g.Go(func() error {
 			if beforeState[index] == madmin.DriveStateMissing {
-				makeErr := storageDisks[index].MakeVol(bucket)
+				makeErr := storageDisks[index].MakeVol(ctx, bucket)
 				if makeErr == nil {
 					afterState[index] = madmin.DriveStateOk
 				}
@@ -171,7 +161,7 @@ func listAllBuckets(storageDisks []StorageAPI, healBuckets map[string]VolInfo) (
 			continue
 		}
 		var volsInfo []VolInfo
-		volsInfo, err = disk.ListVols()
+		volsInfo, err = disk.ListVols(context.TODO())
 		if err != nil {
 			if IsErrIgnored(err, bucketMetadataOpIgnoredErrs...) {
 				continue
@@ -275,7 +265,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			}
 		case errs[i] == errDiskNotFound, dataErrs[i] == errDiskNotFound:
 			driveState = madmin.DriveStateOffline
-		case errs[i] == errFileNotFound, errs[i] == errVolumeNotFound:
+		case errs[i] == errFileNotFound, errs[i] == errFileVersionNotFound, errs[i] == errVolumeNotFound:
 			fallthrough
 		case dataErrs[i] == errFileNotFound, dataErrs[i] == errFileVersionNotFound, dataErrs[i] == errVolumeNotFound:
 			driveState = madmin.DriveStateMissing
@@ -472,7 +462,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		}
 
 		// Attempt a rename now from healed data to final location.
-		if err = disk.RenameData(minioMetaTmpBucket, tmpID, partsMetadata[i].DataDir, bucket, object); err != nil {
+		if err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i].DataDir, bucket, object); err != nil {
 			if err != errIsNotRegular && err != errFileNotFound {
 				logger.LogIf(ctx, err)
 			}
@@ -525,7 +515,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 				wg.Add(1)
 				go func(index int, disk StorageAPI) {
 					defer wg.Done()
-					_ = disk.DeleteFile(bucket, object)
+					_ = disk.DeleteFile(ctx, bucket, object)
 				}(index, disk)
 			}
 			wg.Wait()
@@ -557,7 +547,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 	for i, err := range errs {
 		if err == errVolumeNotFound || err == errFileNotFound {
 			// Bucket or prefix/directory not found
-			merr := storageDisks[i].MakeVol(pathJoin(bucket, object))
+			merr := storageDisks[i].MakeVol(ctx, pathJoin(bucket, object))
 			switch merr {
 			case nil, errVolumeExists:
 				hr.After.Drives[i].State = madmin.DriveStateOk
@@ -642,7 +632,7 @@ func statAllDirs(ctx context.Context, storageDisks []StorageAPI, bucket, prefix
 		}
 		index := index
 		g.Go(func() error {
-			entries, err := storageDisks[index].ListDir(bucket, prefix, 1)
+			entries, err := storageDisks[index].ListDir(ctx, bucket, prefix, 1)
 			if err != nil {
 				return err
 			}

cmd/erasure-healing_test.go

@@ -39,6 +39,7 @@ func TestHealing(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
 	z := obj.(*erasureZones)
@@ -66,7 +67,7 @@ func TestHealing(t *testing.T) {
 	}
 
 	disk := er.getDisks()[0]
-	fileInfoPreHeal, err := disk.ReadVersion(bucket, object, "")
+	fileInfoPreHeal, err := disk.ReadVersion(context.Background(), bucket, object, "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -83,7 +84,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err := disk.ReadVersion(bucket, object, "")
+	fileInfoPostHeal, err := disk.ReadVersion(context.Background(), bucket, object, "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -102,7 +103,7 @@ func TestHealing(t *testing.T) {
 	// gone down when an object was replaced by a new object.
 	fileInfoOutDated := fileInfoPreHeal
 	fileInfoOutDated.ModTime = time.Now()
-	err = disk.WriteMetadata(bucket, object, fileInfoOutDated)
+	err = disk.WriteMetadata(context.Background(), bucket, object, fileInfoOutDated)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -112,7 +113,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err = disk.ReadVersion(bucket, object, "")
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -134,7 +135,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Stat the bucket to make sure that it was created.
-	_, err = er.getDisks()[0].StatVol(bucket)
+	_, err = er.getDisks()[0].StatVol(context.Background(), bucket)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -200,7 +201,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	er := z.zones[0].sets[0]
 	erasureDisks := er.getDisks()
 	firstDisk := erasureDisks[0]
-	err = firstDisk.DeleteFile(bucket, pathJoin(object, xlStorageFormatFile))
+	err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -216,16 +217,16 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
-	if err = firstDisk.CheckFile(bucket, object); err != nil {
+	if err = firstDisk.CheckFile(context.Background(), bucket, object); err != nil {
 		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
 	}
 
-	err = firstDisk.DeleteFile(bucket, pathJoin(object, fi.DataDir, "part.1"))
+	err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
 		t.Errorf("Failure during deleting part.1 - %v", err)
 	}
 
-	err = firstDisk.WriteAll(bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader([]byte{}))
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader([]byte{}))
 	if err != nil {
 		t.Errorf("Failure during creating part.1 - %v", err)
 	}
@@ -245,13 +246,13 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Fatalf("FileInfo not equal after healing")
 	}
 
-	err = firstDisk.DeleteFile(bucket, pathJoin(object, fi.DataDir, "part.1"))
+	err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
 		t.Errorf("Failure during deleting part.1 - %v", err)
 	}
 
 	bdata := bytes.Repeat([]byte("b"), int(nfi.Size))
-	err = firstDisk.WriteAll(bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader(bdata))
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader(bdata))
 	if err != nil {
 		t.Errorf("Failure during creating part.1 - %v", err)
 	}
@@ -274,7 +275,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	// Test 4: checks if HealObject returns an error when xl.meta is not found
 	// in more than read quorum number of disks, to create a corrupted situation.
 	for i := 0; i <= len(er.getDisks())/2; i++ {
-		er.getDisks()[i].DeleteFile(bucket, pathJoin(object, xlStorageFormatFile))
+		er.getDisks()[i].DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
 	}
 
 	// Try healing now, expect to receive errFileNotFound.
@@ -350,7 +351,7 @@ func TestHealObjectErasure(t *testing.T) {
 		t.Fatalf("Failed to complete multipart upload - %v", err)
 	}
 
-	err = firstDisk.DeleteFile(bucket, pathJoin(object, xlStorageFormatFile))
+	err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -360,7 +361,7 @@ func TestHealObjectErasure(t *testing.T) {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	if err = firstDisk.CheckFile(bucket, object); err != nil {
+	if err = firstDisk.CheckFile(context.Background(), bucket, object); err != nil {
 		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
 	}
 
@@ -421,7 +422,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
 	z := obj.(*erasureZones)
 	er := z.zones[0].sets[0]
 	firstDisk := er.getDisks()[0]
-	err = firstDisk.DeleteFile(bucket, object)
+	err = firstDisk.DeleteVol(context.Background(), pathJoin(bucket, encodeDirObject(object)), true)
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -433,7 +434,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
 	}
 
 	// Check if the empty directory is restored in the first disk
-	_, err = firstDisk.StatVol(pathJoin(bucket, object))
+	_, err = firstDisk.StatVol(context.Background(), pathJoin(bucket, encodeDirObject(object)))
 	if err != nil {
 		t.Fatalf("Expected object to be present but stat failed - %v", err)
 	}

cmd/erasure-list-objects.go

@@ -1,58 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-
-	"github.com/minio/minio/pkg/madmin"
-)
-
-// ListObjectVersions - This is not implemented, look for erasure-zones.ListObjectVersions()
-func (er erasureObjects) ListObjectVersions(ctx context.Context, bucket, prefix, marker, versionMarker, delimiter string, maxKeys int) (loi ListObjectVersionsInfo, e error) {
-	return loi, NotImplemented{}
-}
-
-// ListObjectsV2 - This is not implemented/needed anymore, look for erasure-zones.ListObjectsV2()
-func (er erasureObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi ListObjectsV2Info, e error) {
-	return loi, NotImplemented{}
-}
-
-// ListObjects - This is not implemented/needed anymore, look for erasure-zones.ListObjects()
-func (er erasureObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-	return loi, NotImplemented{}
-}
-
-// ListBucketsHeal - This is not implemented/needed anymore, look for erasure-zones.ListBucketHeal()
-func (er erasureObjects) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error) {
-	return nil, NotImplemented{}
-}
-
-// ListObjectsHeal - This is not implemented, look for erasure-zones.ListObjectsHeal()
-func (er erasureObjects) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
-	return ListObjectsInfo{}, NotImplemented{}
-}
-
-// HealObjects - This is not implemented/needed anymore, look for erasure-zones.HealObjects()
-func (er erasureObjects) HealObjects(ctx context.Context, bucket, prefix string, _ madmin.HealOpts, _ HealObjectFn) (e error) {
-	return NotImplemented{}
-}
-
-// Walk - This is not implemented/needed anymore, look for erasure-zones.Walk()
-func (er erasureObjects) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo, _ ObjectOptions) error {
-	return NotImplemented{}
-}

cmd/erasure-metadata-utils.go

@@ -119,14 +119,14 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 	metadataArray := make([]FileInfo, len(disks))
 
 	g := errgroup.WithNErrs(len(disks))
-	// Read `xl.meta` parallelly across disks.
+	// Read `xl.meta` in parallel across disks.
 	for index := range disks {
 		index := index
 		g.Go(func() (err error) {
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			metadataArray[index], err = disks[index].ReadVersion(bucket, object, versionID)
+			metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID)
 			if err != nil {
 				if err != errFileNotFound && err != errVolumeNotFound && err != errFileVersionNotFound {
 					logger.GetReqInfo(ctx).AppendTags("disk", disks[index].String())

cmd/erasure-metadata.go

@@ -96,18 +96,17 @@ func (fi FileInfo) IsValid() bool {
 
 // ToObjectInfo - Converts metadata to object info.
 func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
-	if HasSuffix(object, SlashSeparator) {
-		return ObjectInfo{
-			Bucket: bucket,
-			Name:   object,
-			IsDir:  true,
-		}
+	object = decodeDirObject(object)
+	versionID := fi.VersionID
+	if globalBucketVersioningSys.Enabled(bucket) && versionID == "" {
+		versionID = nullVersionID
 	}
+
 	objInfo := ObjectInfo{
-		IsDir:           false,
+		IsDir:           HasSuffix(object, SlashSeparator),
 		Bucket:          bucket,
 		Name:            object,
-		VersionID:       fi.VersionID,
+		VersionID:       versionID,
 		IsLatest:        fi.IsLatest,
 		DeleteMarker:    fi.Deleted,
 		Size:            fi.Size,
@@ -273,7 +272,7 @@ func renameFileInfo(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			if err := disks[index].RenameData(srcBucket, srcEntry, "", dstBucket, dstEntry); err != nil {
+			if err := disks[index].RenameData(ctx, srcBucket, srcEntry, "", dstBucket, dstEntry); err != nil {
 				if !IsErrIgnored(err, ignoredErr...) {
 					return err
 				}
@@ -304,7 +303,7 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix
 			}
 			// Pick one FileInfo for a disk at index.
 			files[index].Erasure.Index = index + 1
-			return disks[index].WriteMetadata(bucket, prefix, files[index])
+			return disks[index].WriteMetadata(ctx, bucket, prefix, files[index])
 		}, index)
 	}
 

cmd/erasure-multipart.go

@@ -63,7 +63,7 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
 			// Ignoring failure to remove parts that weren't present in CompleteMultipartUpload
 			// requests. xl.meta is the authoritative source of truth on which parts constitute
 			// the object. The presence of parts that don't belong in the object doesn't affect correctness.
-			_ = storageDisks[index].DeleteFile(minioMetaMultipartBucket, curpartPath)
+			_ = storageDisks[index].DeleteFile(context.TODO(), minioMetaMultipartBucket, curpartPath)
 			return nil
 		}, index)
 	}
@@ -71,7 +71,7 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
 }
 
 // Clean-up the old multipart uploads. Should be run in a Go routine.
-func (er erasureObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
+func (er erasureObjects) cleanupStaleUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
 	ticker := time.NewTicker(cleanupInterval)
 	defer ticker.Stop()
 
@@ -81,7 +81,8 @@ func (er erasureObjects) cleanupStaleMultipartUploads(ctx context.Context, clean
 			return
 		case <-ticker.C:
 			var disk StorageAPI
-			for _, d := range er.getLoadBalancedDisks() {
+			// run multiple cleanup's local to this server.
+			for _, d := range er.getLoadBalancedLocalDisks() {
 				if d != nil {
 					disk = d
 					break
@@ -90,26 +91,26 @@ func (er erasureObjects) cleanupStaleMultipartUploads(ctx context.Context, clean
 			if disk == nil {
 				continue
 			}
-			er.cleanupStaleMultipartUploadsOnDisk(ctx, disk, expiry)
+			er.cleanupStaleUploadsOnDisk(ctx, disk, expiry)
 		}
 	}
 }
 
 // Remove the old multipart uploads on the given disk.
-func (er erasureObjects) cleanupStaleMultipartUploadsOnDisk(ctx context.Context, disk StorageAPI, expiry time.Duration) {
+func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk StorageAPI, expiry time.Duration) {
 	now := time.Now()
-	shaDirs, err := disk.ListDir(minioMetaMultipartBucket, "", -1)
+	shaDirs, err := disk.ListDir(ctx, minioMetaMultipartBucket, "", -1)
 	if err != nil {
 		return
 	}
 	for _, shaDir := range shaDirs {
-		uploadIDDirs, err := disk.ListDir(minioMetaMultipartBucket, shaDir, -1)
+		uploadIDDirs, err := disk.ListDir(ctx, minioMetaMultipartBucket, shaDir, -1)
 		if err != nil {
 			continue
 		}
 		for _, uploadIDDir := range uploadIDDirs {
 			uploadIDPath := pathJoin(shaDir, uploadIDDir)
-			fi, err := disk.ReadVersion(minioMetaMultipartBucket, uploadIDPath, "")
+			fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, uploadIDPath, "")
 			if err != nil {
 				continue
 			}
@@ -118,6 +119,19 @@ func (er erasureObjects) cleanupStaleMultipartUploadsOnDisk(ctx context.Context,
 			}
 		}
 	}
+	tmpDirs, err := disk.ListDir(ctx, minioMetaTmpBucket, "", -1)
+	if err != nil {
+		return
+	}
+	for _, tmpDir := range tmpDirs {
+		fi, err := disk.ReadVersion(ctx, minioMetaTmpBucket, tmpDir, "")
+		if err != nil {
+			continue
+		}
+		if now.Sub(fi.ModTime) > expiry {
+			er.deleteObject(ctx, minioMetaTmpBucket, tmpDir, fi.Erasure.DataBlocks+1)
+		}
+	}
 }
 
 // ListMultipartUploads - lists all the pending multipart
@@ -138,7 +152,7 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 		if disk == nil {
 			continue
 		}
-		uploadIDs, err = disk.ListDir(minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
+		uploadIDs, err = disk.ListDir(ctx, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
 		if err != nil {
 			if err == errDiskNotFound {
 				continue
@@ -171,7 +185,7 @@ retry:
 			if populatedUploadIds.Contains(uploadID) {
 				continue
 			}
-			fi, err := disk.ReadVersion(minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "")
+			fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "")
 			if err != nil {
 				if err == errDiskNotFound || err == errFileNotFound {
 					goto retry
@@ -266,11 +280,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 
 	fi.DataDir = mustGetUUID()
 	fi.ModTime = UTCNow()
-	if opts.UserDefined != nil {
-		fi.Metadata = opts.UserDefined
-	} else {
-		fi.Metadata = make(map[string]string)
-	}
+	fi.Metadata = cloneMSS(opts.UserDefined)
 
 	uploadID := mustGetUUID()
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
@@ -336,7 +346,18 @@ func (er erasureObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObjec
 // of the multipart transaction.
 //
 // Implements S3 compatible Upload Part API.
-func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, e error) {
+func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, err error) {
+	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
+	if err = uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+		return PartInfo{}, err
+	}
+	readLocked := true
+	defer func() {
+		if readLocked {
+			uploadIDLock.RUnlock()
+		}
+	}()
+
 	data := r.Reader
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < -1 {
@@ -349,7 +370,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
 
 	// Validates if upload ID exists.
-	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+	if err = er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return pi, toObjectErr(err, bucket, object, uploadID)
 	}
 
@@ -427,7 +448,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	// Should return IncompleteBody{} error when reader has fewer bytes
 	// than specified in request header.
 	if n < data.Size() {
-		return pi, IncompleteBody{}
+		return pi, IncompleteBody{Bucket: bucket, Object: object}
 	}
 
 	for i := range writers {
@@ -436,8 +457,17 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		}
 	}
 
+	// Unlock here before acquiring write locks all concurrent
+	// PutObjectParts would serialize here updating `xl.meta`
+	uploadIDLock.RUnlock()
+	readLocked = false
+	if err = uploadIDLock.GetLock(globalOperationTimeout); err != nil {
+		return PartInfo{}, err
+	}
+	defer uploadIDLock.Unlock()
+
 	// Validates if upload ID exists.
-	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+	if err = er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return pi, toObjectErr(err, bucket, object, uploadID)
 	}
 
@@ -512,6 +542,12 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 		UploadID: uploadID,
 	}
 
+	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
+	if err := uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+		return MultipartInfo{}, err
+	}
+	defer uploadIDLock.RUnlock()
+
 	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return result, toObjectErr(err, bucket, object, uploadID)
 	}
@@ -542,7 +578,7 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 		return result, err
 	}
 
-	result.UserDefined = fi.Metadata
+	result.UserDefined = cloneMSS(fi.Metadata)
 	return result, nil
 }
 
@@ -554,6 +590,12 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 // ListPartsInfo structure is marshaled directly into XML and
 // replied back to the client.
 func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts ObjectOptions) (result ListPartsInfo, e error) {
+	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
+	if err := uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+		return ListPartsInfo{}, err
+	}
+	defer uploadIDLock.RUnlock()
+
 	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return result, toObjectErr(err, bucket, object, uploadID)
 	}
@@ -590,7 +632,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 	result.UploadID = uploadID
 	result.MaxParts = maxParts
 	result.PartNumberMarker = partNumberMarker
-	result.UserDefined = fi.Metadata
+	result.UserDefined = cloneMSS(fi.Metadata)
 
 	// For empty number of parts or maxParts as zero, return right here.
 	if len(fi.Parts) == 0 || maxParts == 0 {
@@ -638,8 +680,16 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 // md5sums of all the parts.
 //
 // Implements S3 compatible Complete multipart API.
-func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, e error) {
-	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
+func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, err error) {
+	// Hold read-locks to verify uploaded parts, also disallows
+	// parallel part uploads as well.
+	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
+	if err = uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+		return oi, err
+	}
+	defer uploadIDLock.RUnlock()
+
+	if err = er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return oi, toObjectErr(err, bucket, object, uploadID)
 	}
 
@@ -787,6 +837,13 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 
+	// Hold namespace to complete the transaction
+	lk := er.NewNSLock(ctx, bucket, object)
+	if err = lk.GetLock(globalOperationTimeout); err != nil {
+		return oi, err
+	}
+	defer lk.Unlock()
+
 	// Rename the multipart object to final location.
 	if onlineDisks, err = renameData(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath,
 		fi.DataDir, bucket, object, writeQuorum, nil); err != nil {
@@ -822,11 +879,13 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 // All parts are purged from all disks and reference to the uploadID
 // would be removed from the system, rollback is not possible on this
 // operation.
-//
-// Implements S3 compatible Abort multipart API, slight difference is
-// that this is an atomic idempotent operation. Subsequent calls have
-// no affect and further requests to the same uploadID would not be honored.
-func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error {
+func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
+	lk := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
+	if err := lk.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
+	defer lk.Unlock()
+
 	// Validates if upload ID exists.
 	if err := er.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return toObjectErr(err, bucket, object, uploadID)

cmd/erasure-object.go

@@ -19,11 +19,13 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
 	"path"
 	"sync"
+	"time"
 
 	"github.com/minio/minio-go/v7/pkg/tags"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -35,30 +37,6 @@ import (
 // list all errors which can be ignored in object operations.
 var objectOpIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errUnformattedDisk)
 
-// putObjectDir hints the bottom layer to create a new directory.
-func (er erasureObjects) putObjectDir(ctx context.Context, bucket, object string, writeQuorum int) error {
-	storageDisks := er.getDisks()
-
-	g := errgroup.WithNErrs(len(storageDisks))
-
-	// Prepare object creation in all disks
-	for index := range storageDisks {
-		if storageDisks[index] == nil {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			err := storageDisks[index].MakeVol(pathJoin(bucket, object))
-			if err != nil && err != errVolumeExists {
-				return err
-			}
-			return nil
-		}, index)
-	}
-
-	return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, writeQuorum)
-}
-
 /// Object Operations
 
 // CopyObject - copy object source object to destination object.
@@ -70,6 +48,11 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		return oi, NotImplemented{}
 	}
 	defer ObjectPathUpdated(path.Join(dstBucket, dstObject))
+	lk := er.NewNSLock(ctx, dstBucket, dstObject)
+	if err := lk.GetLock(globalOperationTimeout); err != nil {
+		return oi, err
+	}
+	defer lk.Unlock()
 
 	// Read metadata associated with the object from all disks.
 	storageDisks := er.getDisks()
@@ -110,18 +93,19 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	fi.VersionID = versionID // set any new versionID we might have created
 	fi.ModTime = modTime     // set modTime for the new versionID
 
+	srcInfo.UserDefined["etag"] = srcInfo.ETag
+
 	// Update `xl.meta` content on each disks.
 	for index := range metaArr {
 		metaArr[index].ModTime = modTime
 		metaArr[index].VersionID = versionID
 		metaArr[index].Metadata = srcInfo.UserDefined
-		metaArr[index].Metadata["etag"] = srcInfo.ETag
 	}
 
 	tempObj := mustGetUUID()
 
 	// Cleanup in case of xl.meta writing failure
-	defer er.deleteObject(ctx, minioMetaTmpBucket, tempObj, writeQuorum)
+	defer er.deleteObject(context.Background(), minioMetaTmpBucket, tempObj, writeQuorum)
 
 	// Write unique `xl.meta` for each disk.
 	if onlineDisks, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaTmpBucket, tempObj, metaArr, writeQuorum); err != nil {
@@ -143,6 +127,32 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		return nil, err
 	}
 
+	var unlockOnDefer bool
+	var nsUnlocker = func() {}
+	defer func() {
+		if unlockOnDefer {
+			nsUnlocker()
+		}
+	}()
+
+	// Acquire lock
+	if lockType != noLock {
+		lock := er.NewNSLock(ctx, bucket, object)
+		switch lockType {
+		case writeLock:
+			if err = lock.GetLock(globalOperationTimeout); err != nil {
+				return nil, err
+			}
+			nsUnlocker = lock.Unlock
+		case readLock:
+			if err = lock.GetRLock(globalOperationTimeout); err != nil {
+				return nil, err
+			}
+			nsUnlocker = lock.RUnlock
+		}
+		unlockOnDefer = true
+	}
+
 	// Handler directory request by returning a reader that
 	// returns no bytes.
 	if HasSuffix(object, SlashSeparator) {
@@ -150,7 +160,8 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		if objInfo, err = er.getObjectInfoDir(ctx, bucket, object); err != nil {
 			return nil, toObjectErr(err, bucket, object)
 		}
-		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts)
+		unlockOnDefer = false
+		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts, nsUnlocker)
 	}
 
 	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts)
@@ -171,7 +182,8 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		}, toObjectErr(errMethodNotAllowed, bucket, object)
 	}
 
-	fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts)
+	unlockOnDefer = false
+	fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts, nsUnlocker)
 	if nErr != nil {
 		return nil, nErr
 	}
@@ -200,6 +212,13 @@ func (er erasureObjects) GetObject(ctx context.Context, bucket, object string, s
 		return err
 	}
 
+	// Lock the object before reading.
+	lk := er.NewNSLock(ctx, bucket, object)
+	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+		return err
+	}
+	defer lk.RUnlock()
+
 	// Start offset cannot be negative.
 	if startOffset < 0 {
 		logger.LogIf(ctx, errUnexpected, logger.Application)
@@ -361,7 +380,7 @@ func (er erasureObjects) getObjectInfoDir(ctx context.Context, bucket, object st
 		index := index
 		g.Go(func() error {
 			// Check if 'prefix' is an object on this 'disk'.
-			entries, err := storageDisks[index].ListDir(bucket, object, 1)
+			entries, err := storageDisks[index].ListDir(ctx, bucket, object, 1)
 			if err != nil {
 				return err
 			}
@@ -384,6 +403,13 @@ func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object strin
 		return info, err
 	}
 
+	// Lock the object before reading.
+	lk := er.NewNSLock(ctx, bucket, object)
+	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+		return ObjectInfo{}, err
+	}
+	defer lk.RUnlock()
+
 	if HasSuffix(object, SlashSeparator) {
 		info, err = er.getObjectInfoDir(ctx, bucket, object)
 		if err != nil {
@@ -458,7 +484,7 @@ func undoRename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry str
 		index := index
 		g.Go(func() error {
 			if errs[index] == nil {
-				_ = disks[index].RenameFile(dstBucket, dstEntry, srcBucket, srcEntry)
+				_ = disks[index].RenameFile(context.TODO(), dstBucket, dstEntry, srcBucket, srcEntry)
 			}
 			return nil
 		}, index)
@@ -478,7 +504,7 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, da
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			if err := disks[index].RenameData(srcBucket, srcEntry, dataDir, dstBucket, dstEntry); err != nil {
+			if err := disks[index].RenameData(ctx, srcBucket, srcEntry, dataDir, dstBucket, dstEntry); err != nil {
 				if !IsErrIgnored(err, ignoredErr...) {
 					return err
 				}
@@ -503,7 +529,7 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, da
 			ug.Go(func() error {
 				// Undo all the partial rename operations.
 				if errs[index] == nil {
-					_ = disks[index].RenameData(dstBucket, dstEntry, dataDir, srcBucket, srcEntry)
+					_ = disks[index].RenameData(context.Background(), dstBucket, dstEntry, dataDir, srcBucket, srcEntry)
 				}
 				return nil
 			}, index)
@@ -531,7 +557,7 @@ func rename(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBuc
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			if err := disks[index].RenameFile(srcBucket, srcEntry, dstBucket, dstEntry); err != nil {
+			if err := disks[index].RenameFile(ctx, srcBucket, srcEntry, dstBucket, dstEntry); err != nil {
 				if !IsErrIgnored(err, ignoredErr...) {
 					return err
 				}
@@ -598,25 +624,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	// Delete temporary object in the event of failure.
 	// If PutObject succeeded there would be no temporary
 	// object to delete.
-	defer er.deleteObject(ctx, minioMetaTmpBucket, tempObj, writeQuorum)
-
-	// This is a special case with size as '0' and object ends with
-	// a slash separator, we treat it like a valid operation and
-	// return success.
-	if isObjectDir(object, data.Size()) {
-		// Check if an object is present as one of the parent dir.
-		// -- FIXME. (needs a new kind of lock).
-		// -- FIXME (this also causes performance issue when disks are down).
-		if er.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-			return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
-		}
-
-		if err = er.putObjectDir(ctx, bucket, object, writeQuorum); err != nil {
-			return ObjectInfo{}, toObjectErr(err, bucket, object)
-		}
-
-		return dirObjectInfo(bucket, object, data.Size(), opts.UserDefined), nil
-	}
+	defer er.deleteObject(context.Background(), minioMetaTmpBucket, tempObj, writeQuorum)
 
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < -1 {
@@ -694,10 +702,15 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	// Should return IncompleteBody{} error when reader has fewer bytes
 	// than specified in request header.
 	if n < data.Size() {
-		logger.LogIf(ctx, IncompleteBody{}, logger.Application)
-		return ObjectInfo{}, IncompleteBody{}
+		return ObjectInfo{}, IncompleteBody{Bucket: bucket, Object: object}
 	}
 
+	lk := er.NewNSLock(ctx, bucket, object)
+	if err := lk.GetLock(globalOperationTimeout); err != nil {
+		return ObjectInfo{}, err
+	}
+	defer lk.Unlock()
+
 	for i, w := range writers {
 		if w == nil {
 			onlineDisks[i] = nil
@@ -775,7 +788,7 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			err := disks[index].DeleteVersion(bucket, object, fi)
+			err := disks[index].DeleteVersion(ctx, bucket, object, fi)
 			if err != nil && err != errVolumeNotFound {
 				return err
 			}
@@ -818,7 +831,9 @@ func (er erasureObjects) deleteObject(ctx context.Context, bucket, object string
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			err := cleanupDir(ctx, disks[index], minioMetaTmpBucket, tmpObj)
+			tctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+			defer cancel()
+			err := cleanupDir(tctx, disks[index], minioMetaTmpBucket, tmpObj)
 			if err != nil && err != errVolumeNotFound {
 				return err
 			}
@@ -860,14 +875,22 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 	versions := make([]FileInfo, len(objects))
 	for i := range objects {
 		if objects[i].VersionID == "" {
-			if opts.Versioned && !HasSuffix(objects[i].ObjectName, SlashSeparator) {
-				versions[i] = FileInfo{
-					Name:      objects[i].ObjectName,
-					VersionID: mustGetUUID(),
-					ModTime:   UTCNow(),
-					Deleted:   true, // delete marker
+			if opts.Versioned || opts.VersionSuspended {
+				if !HasSuffix(objects[i].ObjectName, SlashSeparator) {
+					fi := FileInfo{
+						Name:    objects[i].ObjectName,
+						ModTime: UTCNow(),
+						Deleted: true, // delete marker
+					}
+					if opts.Versioned {
+						fi.VersionID = mustGetUUID()
+					}
+					// versioning suspended means we add `null`
+					// version as delete marker
+
+					versions[i] = fi
+					continue
 				}
-				continue
 			}
 		}
 		versions[i] = FileInfo{
@@ -890,7 +913,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 		wg.Add(1)
 		go func(index int, disk StorageAPI) {
 			defer wg.Done()
-			delObjErrs[index] = disk.DeleteVersions(bucket, versions)
+			delObjErrs[index] = disk.DeleteVersions(ctx, bucket, versions)
 		}(index, disk)
 	}
 
@@ -907,9 +930,11 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 		for i := range delObjErrs {
 			// delObjErrs[i] is not nil when disks[i] is also not nil
 			if delObjErrs[i] != nil {
-				if delObjErrs[i][objIndex] != errFileNotFound {
-					diskErrs[i] = delObjErrs[i][objIndex]
+				if errors.Is(delObjErrs[i][objIndex], errFileNotFound) ||
+					errors.Is(delObjErrs[i][objIndex], errFileVersionNotFound) {
+					continue
 				}
+				diskErrs[i] = delObjErrs[i][objIndex]
 			}
 		}
 		errs[objIndex] = reduceWriteQuorumErrs(ctx, diskErrs, objectOpIgnoredErrs, writeQuorums[objIndex])
@@ -957,22 +982,38 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		return objInfo, err
 	}
 
+	// Acquire a write lock before deleting the object.
+	lk := er.NewNSLock(ctx, bucket, object)
+	if err = lk.GetLock(globalOperationTimeout); err != nil {
+		return ObjectInfo{}, err
+	}
+	defer lk.Unlock()
+
 	storageDisks := er.getDisks()
 	writeQuorum := len(storageDisks)/2 + 1
 
 	if opts.VersionID == "" {
-		if opts.Versioned && !HasSuffix(object, SlashSeparator) {
-			fi := FileInfo{
-				Name:      object,
-				VersionID: mustGetUUID(),
-				Deleted:   true,
-				ModTime:   UTCNow(),
+		if opts.Versioned || opts.VersionSuspended {
+			if !HasSuffix(object, SlashSeparator) {
+				fi := FileInfo{
+					Name:    object,
+					Deleted: true,
+					ModTime: UTCNow(),
+				}
+
+				if opts.Versioned {
+					fi.VersionID = mustGetUUID()
+				}
+
+				// versioning suspended means we add `null`
+				// version as delete marker
+
+				// Add delete marker, since we don't have any version specified explicitly.
+				if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi); err != nil {
+					return objInfo, toObjectErr(err, bucket, object)
+				}
+				return fi.ToObjectInfo(bucket, object), nil
 			}
-			// Add delete marker, since we don't have any version specified explicitly.
-			if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi); err != nil {
-				return objInfo, toObjectErr(err, bucket, object)
-			}
-			return fi.ToObjectInfo(bucket, object), nil
 		}
 	}
 

cmd/erasure-object_test.go

@@ -42,6 +42,7 @@ func TestRepeatPutObjectPart(t *testing.T) {
 	}
 
 	// cleaning up of temporary test directories
+	defer objLayer.Shutdown(context.Background())
 	defer removeRoots(disks)
 
 	err = objLayer.MakeBucketWithLocation(ctx, "bucket1", BucketOptions{})
@@ -90,6 +91,7 @@ func TestErasureDeleteObjectBasic(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer xl.Shutdown(context.Background())
 
 	err = xl.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
 	if err != nil {
@@ -200,6 +202,7 @@ func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Cleanup backend directories
+	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
 	z := obj.(*erasureZones)
@@ -269,6 +272,7 @@ func TestGetObjectNoQuorum(t *testing.T) {
 		t.Fatal(err)
 	}
 	// Cleanup backend directories.
+	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
 	z := obj.(*erasureZones)
@@ -331,6 +335,7 @@ func TestPutObjectNoQuorum(t *testing.T) {
 	}
 
 	// Cleanup backend directories.
+	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
 	z := obj.(*erasureZones)

cmd/erasure-sets.go

@@ -75,13 +75,11 @@ type erasureSets struct {
 	endpointStrings []string
 
 	// Total number of sets and the number of disks per set.
-	setCount, drivesPerSet int
+	setCount, setDriveCount int
+	listTolerancePerSet     int
 
 	disksConnectEvent chan diskConnectInfo
 
-	// Done channel to control monitoring loop.
-	disksConnectDoneCh chan struct{}
-
 	// Distribution algorithm of choice.
 	distributionAlgo string
 	deploymentID     [16]byte
@@ -112,9 +110,9 @@ func (s *erasureSets) getDiskMap() map[string]StorageAPI {
 	defer s.erasureDisksMu.RUnlock()
 
 	for i := 0; i < s.setCount; i++ {
-		for j := 0; j < s.drivesPerSet; j++ {
+		for j := 0; j < s.setDriveCount; j++ {
 			disk := s.erasureDisks[i][j]
-			if disk == nil {
+			if disk == OfflineDisk {
 				continue
 			}
 			if !disk.IsOnline() {
@@ -136,13 +134,10 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 
 	format, err := loadFormatErasure(disk)
 	if err != nil {
-		// Close the internal connection to avoid connection leaks.
-		disk.Close()
 		if errors.Is(err, errUnformattedDisk) {
-			info, derr := disk.DiskInfo()
+			info, derr := disk.DiskInfo(context.TODO())
 			if derr != nil && info.RootDisk {
-				return nil, nil, fmt.Errorf("Disk: %s returned %w but its a root disk refusing to use it",
-					disk, derr) // make sure to '%w' to wrap the error
+				return nil, nil, fmt.Errorf("Disk: %s returned %w", disk, derr) // make sure to '%w' to wrap the error
 			}
 		}
 		return nil, nil, fmt.Errorf("Disk: %s returned %w", disk, err) // make sure to '%w' to wrap the error
@@ -212,14 +207,24 @@ func (s *erasureSets) connectDisks() {
 			defer wg.Done()
 			disk, format, err := connectEndpoint(endpoint)
 			if err != nil {
-				printEndpointError(endpoint, err, true)
+				if endpoint.IsLocal && errors.Is(err, errUnformattedDisk) {
+					globalBackgroundHealState.pushHealLocalDisks(endpoint)
+					logger.Info(fmt.Sprintf("Found unformatted drive %s, attempting to heal...", endpoint))
+				} else {
+					printEndpointError(endpoint, err, true)
+				}
 				return
 			}
+			s.erasureDisksMu.RLock()
 			setIndex, diskIndex, err := findDiskIndex(s.format, format)
+			s.erasureDisksMu.RUnlock()
 			if err != nil {
-				// Close the internal connection to avoid connection leaks.
-				disk.Close()
-				printEndpointError(endpoint, err, false)
+				if endpoint.IsLocal {
+					globalBackgroundHealState.pushHealLocalDisks(endpoint)
+					logger.Info(fmt.Sprintf("Found inconsistent drive %s with format.json, attempting to heal...", endpoint))
+				} else {
+					printEndpointError(endpoint, err, false)
+				}
 				return
 			}
 			disk.SetDiskID(format.Erasure.This)
@@ -228,7 +233,7 @@ func (s *erasureSets) connectDisks() {
 				s.erasureDisks[setIndex][diskIndex].Close()
 			}
 			s.erasureDisks[setIndex][diskIndex] = disk
-			s.endpointStrings[setIndex*s.drivesPerSet+diskIndex] = disk.String()
+			s.endpointStrings[setIndex*s.setDriveCount+diskIndex] = disk.String()
 			s.erasureDisksMu.Unlock()
 			go func(setIndex int) {
 				// Send a new disk connect event with a timeout
@@ -250,8 +255,6 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 		select {
 		case <-ctx.Done():
 			return
-		case <-s.disksConnectDoneCh:
-			return
 		case <-time.After(monitorInterval):
 			s.connectDisks()
 		}
@@ -260,7 +263,7 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 
 func (s *erasureSets) GetLockers(setIndex int) func() []dsync.NetLocker {
 	return func() []dsync.NetLocker {
-		lockers := make([]dsync.NetLocker, s.drivesPerSet)
+		lockers := make([]dsync.NetLocker, s.setDriveCount)
 		copy(lockers, s.erasureLockers[setIndex])
 		return lockers
 	}
@@ -271,9 +274,9 @@ func (s *erasureSets) GetEndpoints(setIndex int) func() []string {
 		s.erasureDisksMu.RLock()
 		defer s.erasureDisksMu.RUnlock()
 
-		eps := make([]string, s.drivesPerSet)
-		for i := 0; i < s.drivesPerSet; i++ {
-			eps[i] = s.endpointStrings[setIndex*s.drivesPerSet+i]
+		eps := make([]string, s.setDriveCount)
+		for i := 0; i < s.setDriveCount; i++ {
+			eps[i] = s.endpointStrings[setIndex*s.setDriveCount+i]
 		}
 		return eps
 	}
@@ -284,71 +287,71 @@ func (s *erasureSets) GetDisks(setIndex int) func() []StorageAPI {
 	return func() []StorageAPI {
 		s.erasureDisksMu.RLock()
 		defer s.erasureDisksMu.RUnlock()
-		disks := make([]StorageAPI, s.drivesPerSet)
+		disks := make([]StorageAPI, s.setDriveCount)
 		copy(disks, s.erasureDisks[setIndex])
 		return disks
 	}
 }
 
-const defaultMonitorConnectEndpointInterval = time.Second * 10 // Set to 10 secs.
+// defaultMonitorConnectEndpointInterval is the interval to monitor endpoint connections.
+// Must be bigger than defaultMonitorNewDiskInterval.
+const defaultMonitorConnectEndpointInterval = defaultMonitorNewDiskInterval + time.Second*5
 
 // Initialize new set of erasure coded sets.
 func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []StorageAPI, format *formatErasureV3) (*erasureSets, error) {
 	setCount := len(format.Erasure.Sets)
-	drivesPerSet := len(format.Erasure.Sets[0])
+	setDriveCount := len(format.Erasure.Sets[0])
 
 	endpointStrings := make([]string, len(endpoints))
 	// Initialize the erasure sets instance.
 	s := &erasureSets{
-		sets:               make([]*erasureObjects, setCount),
-		erasureDisks:       make([][]StorageAPI, setCount),
-		erasureLockers:     make([][]dsync.NetLocker, setCount),
-		endpoints:          endpoints,
-		endpointStrings:    endpointStrings,
-		setCount:           setCount,
-		drivesPerSet:       drivesPerSet,
-		format:             format,
-		disksConnectEvent:  make(chan diskConnectInfo),
-		disksConnectDoneCh: make(chan struct{}),
-		distributionAlgo:   format.Erasure.DistributionAlgo,
-		deploymentID:       uuid.MustParse(format.ID),
-		pool:               NewMergeWalkPool(globalMergeLookupTimeout),
-		poolSplunk:         NewMergeWalkPool(globalMergeLookupTimeout),
-		poolVersions:       NewMergeWalkVersionsPool(globalMergeLookupTimeout),
-		mrfOperations:      make(map[healSource]int),
+		sets:                make([]*erasureObjects, setCount),
+		erasureDisks:        make([][]StorageAPI, setCount),
+		erasureLockers:      make([][]dsync.NetLocker, setCount),
+		endpoints:           endpoints,
+		endpointStrings:     endpointStrings,
+		setCount:            setCount,
+		setDriveCount:       setDriveCount,
+		listTolerancePerSet: setDriveCount / 2,
+		format:              format,
+		disksConnectEvent:   make(chan diskConnectInfo),
+		distributionAlgo:    format.Erasure.DistributionAlgo,
+		deploymentID:        uuid.MustParse(format.ID),
+		pool:                NewMergeWalkPool(globalMergeLookupTimeout),
+		poolSplunk:          NewMergeWalkPool(globalMergeLookupTimeout),
+		poolVersions:        NewMergeWalkVersionsPool(globalMergeLookupTimeout),
+		mrfOperations:       make(map[healSource]int),
 	}
 
 	mutex := newNSLock(globalIsDistErasure)
 
 	// Initialize byte pool once for all sets, bpool size is set to
-	// setCount * drivesPerSet with each memory upto blockSizeV1.
-	bp := bpool.NewBytePoolCap(setCount*drivesPerSet, blockSizeV1, blockSizeV1*2)
+	// setCount * setDriveCount with each memory upto blockSizeV1.
+	bp := bpool.NewBytePoolCap(setCount*setDriveCount, blockSizeV1, blockSizeV1*2)
 
 	for i := 0; i < setCount; i++ {
-		s.erasureDisks[i] = make([]StorageAPI, drivesPerSet)
-		s.erasureLockers[i] = make([]dsync.NetLocker, drivesPerSet)
+		s.erasureDisks[i] = make([]StorageAPI, setDriveCount)
+		s.erasureLockers[i] = make([]dsync.NetLocker, setDriveCount)
 	}
 
 	for i := 0; i < setCount; i++ {
-		for j := 0; j < drivesPerSet; j++ {
-			endpoint := endpoints[i*drivesPerSet+j]
+		for j := 0; j < setDriveCount; j++ {
+			endpoint := endpoints[i*setDriveCount+j]
 			// Rely on endpoints list to initialize, init lockers and available disks.
 			s.erasureLockers[i][j] = newLockAPI(endpoint)
-			disk := storageDisks[i*drivesPerSet+j]
+			disk := storageDisks[i*setDriveCount+j]
 			if disk == nil {
 				continue
 			}
 			diskID, derr := disk.GetDiskID()
 			if derr != nil {
-				disk.Close()
 				continue
 			}
 			m, n, err := findDiskIndexByDiskID(format, diskID)
 			if err != nil {
-				disk.Close()
 				continue
 			}
-			s.endpointStrings[m*drivesPerSet+n] = disk.String()
+			s.endpointStrings[m*setDriveCount+n] = disk.String()
 			s.erasureDisks[m][n] = disk
 		}
 
@@ -362,8 +365,8 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 			mrfOpCh:      make(chan partialOperation, 10000),
 		}
 
-		go s.sets[i].cleanupStaleMultipartUploads(ctx,
-			GlobalMultipartCleanupInterval, GlobalMultipartExpiry)
+		go s.sets[i].cleanupStaleUploads(ctx,
+			GlobalStaleUploadsCleanupInterval, GlobalStaleUploadsExpiry)
 	}
 
 	// Start the disk monitoring and connect routine.
@@ -384,7 +387,7 @@ func (s *erasureSets) NewNSLock(ctx context.Context, bucket string, objects ...s
 
 // SetDriveCount returns the current drives per set.
 func (s *erasureSets) SetDriveCount() int {
-	return s.drivesPerSet
+	return s.setDriveCount
 }
 
 // StorageUsageInfo - combines output of StorageInfo across all erasure coded object sets.
@@ -458,13 +461,13 @@ func (s *erasureSets) StorageInfo(ctx context.Context, local bool) (StorageInfo,
 
 	scParity := globalStorageClass.GetParityForSC(storageclass.STANDARD)
 	if scParity == 0 {
-		scParity = s.drivesPerSet / 2
+		scParity = s.setDriveCount / 2
 	}
-	storageInfo.Backend.StandardSCData = s.drivesPerSet - scParity
+	storageInfo.Backend.StandardSCData = s.setDriveCount - scParity
 	storageInfo.Backend.StandardSCParity = scParity
 
 	rrSCParity := globalStorageClass.GetParityForSC(storageclass.RRS)
-	storageInfo.Backend.RRSCData = s.drivesPerSet - rrSCParity
+	storageInfo.Backend.RRSCData = s.setDriveCount - rrSCParity
 	storageInfo.Backend.RRSCParity = rrSCParity
 
 	if local {
@@ -481,11 +484,6 @@ func (s *erasureSets) StorageInfo(ctx context.Context, local bool) (StorageInfo,
 	return storageInfo, errs
 }
 
-func (s *erasureSets) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo) error {
-	// Use the zone-level implementation instead.
-	return NotImplemented{API: "CrawlAndGetDataUsage"}
-}
-
 // Shutdown shutsdown all erasure coded sets in parallel
 // returns error upon first error.
 func (s *erasureSets) Shutdown(ctx context.Context) error {
@@ -503,7 +501,14 @@ func (s *erasureSets) Shutdown(ctx context.Context) error {
 			return err
 		}
 	}
-
+	select {
+	case _, ok := <-s.disksConnectEvent:
+		if ok {
+			close(s.disksConnectEvent)
+		}
+	default:
+		close(s.disksConnectEvent)
+	}
 	return nil
 }
 
@@ -582,11 +587,6 @@ func (s *erasureSets) GetBucketInfo(ctx context.Context, bucket string) (bucketI
 	return s.getHashedSet("").GetBucketInfo(ctx, bucket)
 }
 
-// ListObjectsV2 lists all objects in bucket filtered by prefix
-func (s *erasureSets) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
-	return result, NotImplemented{}
-}
-
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
 func (s *erasureSets) IsNotificationSupported() bool {
 	return s.getHashedSet("").IsNotificationSupported()
@@ -758,9 +758,13 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
 
 	// Check if this request is only metadata update.
 	if cpSrcDstSame && srcInfo.metadataOnly {
+		// Version ID is set for the destination and source == destination version ID.
+		// perform an in-place update.
 		if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
 			return srcSet.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
+		// Destination is not versioned and source version ID is empty
+		// perform an in-place update.
 		if !dstOpts.Versioned && srcOpts.VersionID == "" {
 			return srcSet.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
@@ -838,8 +842,8 @@ func (f *FileInfoCh) Push(fi FileInfo) {
 // if the caller wishes to list N entries to call lexicallySortedEntry
 // N times until this boolean is 'false'.
 func lexicallySortedEntry(entryChs []FileInfoCh, entries []FileInfo, entriesValid []bool) (FileInfo, int, bool) {
-	for i := range entryChs {
-		entries[i], entriesValid[i] = entryChs[i].Pop()
+	for j := range entryChs {
+		entries[j], entriesValid[j] = entryChs[j].Pop()
 	}
 
 	var isTruncated = false
@@ -902,8 +906,8 @@ func lexicallySortedEntry(entryChs []FileInfoCh, entries []FileInfo, entriesVali
 // if the caller wishes to list N entries to call lexicallySortedEntry
 // N times until this boolean is 'false'.
 func lexicallySortedEntryVersions(entryChs []FileInfoVersionsCh, entries []FileInfoVersions, entriesValid []bool) (FileInfoVersions, int, bool) {
-	for i := range entryChs {
-		entries[i], entriesValid[i] = entryChs[i].Pop()
+	for j := range entryChs {
+		entries[j], entriesValid[j] = entryChs[j].Pop()
 	}
 
 	var isTruncated = false
@@ -959,7 +963,7 @@ func lexicallySortedEntryVersions(entryChs []FileInfoVersionsCh, entries []FileI
 }
 
 func (s *erasureSets) startMergeWalks(ctx context.Context, bucket, prefix, marker string, recursive bool, endWalkCh <-chan struct{}) []FileInfoCh {
-	return s.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, -1)
+	return s.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, -1, false)
 }
 
 func (s *erasureSets) startMergeWalksVersions(ctx context.Context, bucket, prefix, marker string, recursive bool, endWalkCh <-chan struct{}) []FileInfoVersionsCh {
@@ -970,109 +974,67 @@ func (s *erasureSets) startMergeWalksVersions(ctx context.Context, bucket, prefi
 // FileInfoCh which can be read from.
 func (s *erasureSets) startMergeWalksVersionsN(ctx context.Context, bucket, prefix, marker string, recursive bool, endWalkCh <-chan struct{}, ndisks int) []FileInfoVersionsCh {
 	var entryChs []FileInfoVersionsCh
-	var success int
+	var wg sync.WaitGroup
+	var mutex sync.Mutex
 	for _, set := range s.sets {
 		// Reset for the next erasure set.
-		success = ndisks
-		for _, disk := range set.getLoadBalancedDisks() {
-			if disk == nil {
-				// Disk can be offline
-				continue
-			}
-			entryCh, err := disk.WalkVersions(bucket, prefix, marker, recursive, endWalkCh)
-			if err != nil {
-				logger.LogIf(ctx, err)
-				// Disk walk returned error, ignore it.
-				continue
-			}
-			entryChs = append(entryChs, FileInfoVersionsCh{
-				Ch: entryCh,
-			})
-			success--
-			if success == 0 {
-				break
-			}
+		for _, disk := range set.getLoadBalancedNDisks(ndisks) {
+			wg.Add(1)
+			go func(disk StorageAPI) {
+				defer wg.Done()
+				entryCh, err := disk.WalkVersions(GlobalContext, bucket, prefix, marker, recursive, endWalkCh)
+				if err != nil {
+					return
+				}
+
+				mutex.Lock()
+				entryChs = append(entryChs, FileInfoVersionsCh{
+					Ch: entryCh,
+				})
+				mutex.Unlock()
+			}(disk)
 		}
 	}
+	wg.Wait()
 	return entryChs
 }
 
 // Starts a walk channel across n number of disks and returns a slice of
 // FileInfoCh which can be read from.
-func (s *erasureSets) startMergeWalksN(ctx context.Context, bucket, prefix, marker string, recursive bool, endWalkCh <-chan struct{}, ndisks int) []FileInfoCh {
+func (s *erasureSets) startMergeWalksN(ctx context.Context, bucket, prefix, marker string, recursive bool, endWalkCh <-chan struct{}, ndisks int, splunk bool) []FileInfoCh {
 	var entryChs []FileInfoCh
-	var success int
+	var wg sync.WaitGroup
+	var mutex sync.Mutex
 	for _, set := range s.sets {
 		// Reset for the next erasure set.
-		success = ndisks
-		for _, disk := range set.getLoadBalancedDisks() {
-			if disk == nil {
-				// Disk can be offline
-				continue
-			}
-			entryCh, err := disk.Walk(bucket, prefix, marker, recursive, endWalkCh)
-			if err != nil {
-				// Disk walk returned error, ignore it.
-				continue
-			}
-			entryChs = append(entryChs, FileInfoCh{
-				Ch: entryCh,
-			})
-			success--
-			if success == 0 {
-				break
-			}
+		for _, disk := range set.getLoadBalancedNDisks(ndisks) {
+			wg.Add(1)
+			go func(disk StorageAPI) {
+				defer wg.Done()
+
+				var entryCh chan FileInfo
+				var err error
+				if splunk {
+					entryCh, err = disk.WalkSplunk(GlobalContext, bucket, prefix, marker, endWalkCh)
+				} else {
+					entryCh, err = disk.Walk(GlobalContext, bucket, prefix, marker, recursive, endWalkCh)
+				}
+				if err != nil {
+					// Disk walk returned error, ignore it.
+					return
+				}
+				mutex.Lock()
+				entryChs = append(entryChs, FileInfoCh{
+					Ch: entryCh,
+				})
+				mutex.Unlock()
+			}(disk)
 		}
 	}
+	wg.Wait()
 	return entryChs
 }
 
-// Starts a walk channel across n number of disks and returns a slice of
-// FileInfo channels which can be read from.
-func (s *erasureSets) startSplunkMergeWalksN(ctx context.Context, bucket, prefix, marker string, endWalkCh <-chan struct{}, ndisks int) []FileInfoCh {
-	var entryChs []FileInfoCh
-	var success int
-	for _, set := range s.sets {
-		// Reset for the next erasure set.
-		success = ndisks
-		for _, disk := range set.getLoadBalancedDisks() {
-			if disk == nil {
-				// Disk can be offline
-				continue
-			}
-			entryCh, err := disk.WalkSplunk(bucket, prefix, marker, endWalkCh)
-			if err != nil {
-				// Disk walk returned error, ignore it.
-				continue
-			}
-			entryChs = append(entryChs, FileInfoCh{
-				Ch: entryCh,
-			})
-			success--
-			if success == 0 {
-				break
-			}
-		}
-	}
-	return entryChs
-}
-
-// ListObjectVersions - implements listing of objects across disks, each disk is indepenently
-// walked and merged at this layer. Resulting value through the merge process sends
-// the data in lexically sorted order.
-func (s *erasureSets) ListObjectVersions(ctx context.Context, bucket, prefix, marker, versionIDMarker, delimiter string, maxKeys int) (loi ListObjectVersionsInfo, err error) {
-	// Shouldn't be called directly, caller Zones already has an implementation
-	return loi, NotImplemented{}
-}
-
-// ListObjects - implements listing of objects across disks, each disk is indepenently
-// walked and merged at this layer. Resulting value through the merge process sends
-// the data in lexically sorted order.
-func (s *erasureSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
-	// Shouldn't be called directly, caller Zones already has an implementation
-	return loi, NotImplemented{}
-}
-
 func (s *erasureSets) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
 	// In list multipart uploads we are going to treat input prefix as the object,
 	// this means that we are not supporting directory navigation.
@@ -1108,8 +1070,8 @@ func (s *erasureSets) ListObjectParts(ctx context.Context, bucket, object, uploa
 }
 
 // Aborts an in-progress multipart operation on hashedSet based on the object name.
-func (s *erasureSets) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error {
-	return s.getHashedSet(object).AbortMultipartUpload(ctx, bucket, object, uploadID)
+func (s *erasureSets) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
+	return s.getHashedSet(object).AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 }
 
 // CompleteMultipartUpload - completes a pending multipart transaction, on hashedSet based on object name.
@@ -1216,7 +1178,7 @@ func (s *erasureSets) ReloadFormat(ctx context.Context, dryRun bool) (err error)
 	}(storageDisks)
 
 	formats, _ := loadFormatErasureAll(storageDisks, false)
-	if err = checkFormatErasureValues(formats, s.drivesPerSet); err != nil {
+	if err = checkFormatErasureValues(formats, s.setDriveCount); err != nil {
 		return err
 	}
 
@@ -1225,16 +1187,12 @@ func (s *erasureSets) ReloadFormat(ctx context.Context, dryRun bool) (err error)
 		return err
 	}
 
-	// kill the monitoring loop such that we stop writing
-	// to indicate that we will re-initialize everything
-	// with new format.
-	s.disksConnectDoneCh <- struct{}{}
+	s.erasureDisksMu.Lock()
 
 	// Replace with new reference format.
 	s.format = refFormat
 
 	// Close all existing disks and reconnect all the disks.
-	s.erasureDisksMu.Lock()
 	for _, disk := range storageDisks {
 		if disk == nil {
 			continue
@@ -1242,13 +1200,11 @@ func (s *erasureSets) ReloadFormat(ctx context.Context, dryRun bool) (err error)
 
 		diskID, err := disk.GetDiskID()
 		if err != nil {
-			disk.Close()
 			continue
 		}
 
 		m, n, err := findDiskIndexByDiskID(refFormat, diskID)
 		if err != nil {
-			disk.Close()
 			continue
 		}
 
@@ -1256,13 +1212,11 @@ func (s *erasureSets) ReloadFormat(ctx context.Context, dryRun bool) (err error)
 			s.erasureDisks[m][n].Close()
 		}
 
-		s.endpointStrings[m*s.drivesPerSet+n] = disk.String()
+		s.endpointStrings[m*s.setDriveCount+n] = disk.String()
 		s.erasureDisks[m][n] = disk
 	}
-	s.erasureDisksMu.Unlock()
 
-	// Restart monitoring loop to monitor reformatted disks again.
-	go s.monitorAndConnectEndpoints(GlobalContext, defaultMonitorConnectEndpointInterval)
+	s.erasureDisksMu.Unlock()
 
 	return nil
 }
@@ -1272,17 +1226,14 @@ func (s *erasureSets) ReloadFormat(ctx context.Context, dryRun bool) (err error)
 func isTestSetup(infos []DiskInfo, errs []error) bool {
 	rootDiskCount := 0
 	for i := range errs {
-		if errs[i] != nil && errs[i] != errUnformattedDisk {
-			// On any error which is not unformatted disk
-			// it is safer to reject healing.
-			return false
-		}
-		if infos[i].RootDisk {
-			rootDiskCount++
+		if errs[i] == nil || errs[i] == errUnformattedDisk {
+			if infos[i].RootDisk {
+				rootDiskCount++
+			}
 		}
 	}
-	// It is a test setup if all disks are root disks.
-	return rootDiskCount == len(infos)
+	// It is a test setup if all disks are root disks in quorum.
+	return rootDiskCount >= len(infos)/2+1
 }
 
 func getHealDiskInfos(storageDisks []StorageAPI, errs []error) ([]DiskInfo, []error) {
@@ -1298,7 +1249,7 @@ func getHealDiskInfos(storageDisks []StorageAPI, errs []error) ([]DiskInfo, []er
 				return errDiskNotFound
 			}
 			var err error
-			infos[index], err = storageDisks[index].DiskInfo()
+			infos[index], err = storageDisks[index].DiskInfo(context.TODO())
 			return err
 		}, index)
 	}
@@ -1338,18 +1289,31 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 	}(storageDisks)
 
 	formats, sErrs := loadFormatErasureAll(storageDisks, true)
-	if err = checkFormatErasureValues(formats, s.drivesPerSet); err != nil {
+	if err = checkFormatErasureValues(formats, s.setDriveCount); err != nil {
 		return madmin.HealResultItem{}, err
 	}
 
 	// Mark all root disks down
 	markRootDisksAsDown(storageDisks, sErrs)
 
+	refFormat, err := getFormatErasureInQuorum(formats)
+	if err != nil {
+		return res, err
+	}
+
+	for i, format := range formats {
+		if format != nil {
+			if ferr := formatErasureV3Check(refFormat, format); ferr != nil {
+				sErrs[i] = errUnformattedDisk
+			}
+		}
+	}
+
 	// Prepare heal-result
 	res = madmin.HealResultItem{
 		Type:      madmin.HealItemMetadata,
 		Detail:    "disk-format",
-		DiskCount: s.setCount * s.drivesPerSet,
+		DiskCount: s.setCount * s.setDriveCount,
 		SetCount:  s.setCount,
 	}
 
@@ -1370,17 +1334,12 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 		return res, errNoHealRequired
 	}
 
-	refFormat, err := getFormatErasureInQuorum(formats)
-	if err != nil {
-		return res, err
-	}
-
 	// Mark all UUIDs which might be offline, use list
 	// of formats to mark them appropriately.
 	markUUIDsOffline(refFormat, formats)
 
 	// Initialize a new set of set formats which will be written to disk.
-	newFormatSets := newHealFormatSets(refFormat, s.setCount, s.drivesPerSet, formats, sErrs)
+	newFormatSets := newHealFormatSets(refFormat, s.setCount, s.setDriveCount, formats, sErrs)
 
 	// Look for all offline/unformatted disks in our reference format,
 	// such that we can fill them up with new UUIDs, this looping also
@@ -1397,7 +1356,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 						newFormatSets[i][l].Erasure.This = mustGetUUID()
 						refFormat.Erasure.Sets[i][j] = newFormatSets[i][l].Erasure.This
 						for m, v := range res.After.Drives {
-							if v.Endpoint == s.endpoints.GetString(i*s.drivesPerSet+l) {
+							if v.Endpoint == s.endpoints.GetString(i*s.setDriveCount+l) {
 								res.After.Drives[m].UUID = newFormatSets[i][l].Erasure.This
 								res.After.Drives[m].State = madmin.DriveStateOk
 							}
@@ -1410,14 +1369,14 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 	}
 
 	if !dryRun {
-		var tmpNewFormats = make([]*formatErasureV3, s.setCount*s.drivesPerSet)
+		var tmpNewFormats = make([]*formatErasureV3, s.setCount*s.setDriveCount)
 		for i := range newFormatSets {
 			for j := range newFormatSets[i] {
 				if newFormatSets[i][j] == nil {
 					continue
 				}
-				tmpNewFormats[i*s.drivesPerSet+j] = newFormatSets[i][j]
-				tmpNewFormats[i*s.drivesPerSet+j].Erasure.Sets = refFormat.Erasure.Sets
+				tmpNewFormats[i*s.setDriveCount+j] = newFormatSets[i][j]
+				tmpNewFormats[i*s.setDriveCount+j].Erasure.Sets = refFormat.Erasure.Sets
 			}
 		}
 
@@ -1431,16 +1390,12 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 			return madmin.HealResultItem{}, err
 		}
 
-		// kill the monitoring loop such that we stop writing
-		// to indicate that we will re-initialize everything
-		// with new format.
-		s.disksConnectDoneCh <- struct{}{}
+		s.erasureDisksMu.Lock()
 
 		// Replace with new reference format.
 		s.format = refFormat
 
 		// Disconnect/relinquish all existing disks, lockers and reconnect the disks, lockers.
-		s.erasureDisksMu.Lock()
 		for _, disk := range storageDisks {
 			if disk == nil {
 				continue
@@ -1448,13 +1403,11 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 
 			diskID, err := disk.GetDiskID()
 			if err != nil {
-				disk.Close()
 				continue
 			}
 
 			m, n, err := findDiskIndexByDiskID(refFormat, diskID)
 			if err != nil {
-				disk.Close()
 				continue
 			}
 
@@ -1462,13 +1415,11 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 				s.erasureDisks[m][n].Close()
 			}
 
-			s.endpointStrings[m*s.drivesPerSet+n] = disk.String()
+			s.endpointStrings[m*s.setDriveCount+n] = disk.String()
 			s.erasureDisks[m][n] = disk
 		}
-		s.erasureDisksMu.Unlock()
 
-		// Restart our monitoring loop to start monitoring newly formatted disks.
-		go s.monitorAndConnectEndpoints(GlobalContext, defaultMonitorConnectEndpointInterval)
+		s.erasureDisksMu.Unlock()
 	}
 
 	return res, nil
@@ -1480,7 +1431,7 @@ func (s *erasureSets) HealBucket(ctx context.Context, bucket string, dryRun, rem
 	result = madmin.HealResultItem{
 		Type:      madmin.HealItemBucket,
 		Bucket:    bucket,
-		DiskCount: s.setCount * s.drivesPerSet,
+		DiskCount: s.setCount * s.setDriveCount,
 		SetCount:  s.setCount,
 	}
 
@@ -1496,7 +1447,7 @@ func (s *erasureSets) HealBucket(ctx context.Context, bucket string, dryRun, rem
 
 	// Check if we had quorum to write, if not return an appropriate error.
 	_, afterDriveOnline := result.GetOnlineCounts()
-	if afterDriveOnline < ((s.setCount*s.drivesPerSet)/2)+1 {
+	if afterDriveOnline < ((s.setCount*s.setDriveCount)/2)+1 {
 		return result, toObjectErr(errErasureWriteQuorum, bucket)
 	}
 
@@ -1552,7 +1503,7 @@ func (s *erasureSets) Walk(ctx context.Context, bucket, prefix string, results c
 					return
 				}
 
-				if quorumCount >= s.drivesPerSet/2 {
+				if quorumCount >= s.setDriveCount/2 {
 					// Read quorum exists proceed
 					for _, version := range entry.Versions {
 						results <- version.ToObjectInfo(bucket, version.Name)
@@ -1579,7 +1530,7 @@ func (s *erasureSets) Walk(ctx context.Context, bucket, prefix string, results c
 				return
 			}
 
-			if quorumCount >= s.drivesPerSet/2 {
+			if quorumCount >= s.setDriveCount/2 {
 				// Read quorum exists proceed
 				results <- entry.ToObjectInfo(bucket, entry.Name)
 			}
@@ -1606,14 +1557,14 @@ func (s *erasureSets) HealObjects(ctx context.Context, bucket, prefix string, op
 			break
 		}
 
-		if quorumCount == s.drivesPerSet && opts.ScanMode == madmin.HealNormalScan {
+		if quorumCount == s.setDriveCount && opts.ScanMode == madmin.HealNormalScan {
 			// Skip good entries.
 			continue
 		}
 
 		for _, version := range entry.Versions {
 			// Wait and proceed if there are active requests
-			waitForLowHTTPReq(int32(s.drivesPerSet), time.Second)
+			waitForLowHTTPReq(int32(s.setDriveCount), time.Second)
 
 			if err := healObject(bucket, version.Name, version.VersionID); err != nil {
 				return toObjectErr(err, bucket, version.Name)
@@ -1639,18 +1590,6 @@ func (s *erasureSets) GetObjectTags(ctx context.Context, bucket, object string,
 	return s.getHashedSet(object).GetObjectTags(ctx, bucket, object, opts)
 }
 
-// GetMetrics - no op
-func (s *erasureSets) GetMetrics(ctx context.Context) (*Metrics, error) {
-	logger.LogIf(ctx, NotImplemented{})
-	return &Metrics{}, NotImplemented{}
-}
-
-// Health shouldn't be called directly - will panic
-func (s *erasureSets) Health(ctx context.Context, _ HealthOptions) HealthResult {
-	logger.CriticalIf(ctx, NotImplemented{})
-	return HealthResult{}
-}
-
 // maintainMRFList gathers the list of successful partial uploads
 // from all underlying er.sets and puts them in a global map which
 // should not have more than 10000 entries.

cmd/erasure-sets_test.go

@@ -206,6 +206,7 @@ func TestHashedLayer(t *testing.T) {
 		if err != nil {
 			t.Fatal("Unable to initialize 'Erasure' object layer.", err)
 		}
+		defer obj.Shutdown(ctx)
 
 		// Remove all dirs.
 		for _, dir := range fsDirs {

cmd/erasure-zones.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"math/rand"
@@ -40,6 +41,9 @@ type erasureZones struct {
 	GatewayUnsupported
 
 	zones []*erasureSets
+
+	// Shut down async operations
+	shutdown context.CancelFunc
 }
 
 func (z *erasureZones) SingleZone() bool {
@@ -79,7 +83,7 @@ func newErasureZones(ctx context.Context, endpointZones EndpointZones) (ObjectLa
 			return nil, err
 		}
 	}
-
+	ctx, z.shutdown = context.WithCancel(ctx)
 	go intDataUpdateTracker.start(ctx, localDrives...)
 	return z, nil
 }
@@ -218,6 +222,7 @@ func (z *erasureZones) getZoneIdx(ctx context.Context, bucket, object string, op
 }
 
 func (z *erasureZones) Shutdown(ctx context.Context) error {
+	defer z.shutdown()
 	if z.SingleZone() {
 		return z.zones[0].Shutdown(ctx)
 	}
@@ -237,7 +242,6 @@ func (z *erasureZones) Shutdown(ctx context.Context) error {
 		}
 		// let's the rest shutdown
 	}
-
 	return nil
 }
 
@@ -456,24 +460,7 @@ func (z *erasureZones) MakeBucketWithLocation(ctx context.Context, bucket string
 }
 
 func (z *erasureZones) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	var nsUnlocker = func() {}
-
-	// Acquire lock
-	if lockType != noLock {
-		lock := z.NewNSLock(ctx, bucket, object)
-		switch lockType {
-		case writeLock:
-			if err = lock.GetLock(globalOperationTimeout); err != nil {
-				return nil, err
-			}
-			nsUnlocker = lock.Unlock
-		case readLock:
-			if err = lock.GetRLock(globalOperationTimeout); err != nil {
-				return nil, err
-			}
-			nsUnlocker = lock.RUnlock
-		}
-	}
+	object = encodeDirObject(object)
 
 	for _, zone := range z.zones {
 		gr, err = zone.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
@@ -481,13 +468,10 @@ func (z *erasureZones) GetObjectNInfo(ctx context.Context, bucket, object string
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				continue
 			}
-			nsUnlocker()
 			return gr, err
 		}
-		gr.cleanUpFns = append(gr.cleanUpFns, nsUnlocker)
 		return gr, nil
 	}
-	nsUnlocker()
 	if opts.VersionID != "" {
 		return gr, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
 	}
@@ -495,16 +479,7 @@ func (z *erasureZones) GetObjectNInfo(ctx context.Context, bucket, object string
 }
 
 func (z *erasureZones) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
-	// Lock the object before reading.
-	lk := z.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer lk.RUnlock()
-
-	if z.SingleZone() {
-		return z.zones[0].GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
-	}
+	object = encodeDirObject(object)
 
 	for _, zone := range z.zones {
 		if err := zone.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts); err != nil {
@@ -515,20 +490,14 @@ func (z *erasureZones) GetObject(ctx context.Context, bucket, object string, sta
 		}
 		return nil
 	}
+	if opts.VersionID != "" {
+		return VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
+	}
 	return ObjectNotFound{Bucket: bucket, Object: object}
 }
 
 func (z *erasureZones) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	// Lock the object before reading.
-	lk := z.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
-		return ObjectInfo{}, err
-	}
-	defer lk.RUnlock()
-
-	if z.SingleZone() {
-		return z.zones[0].GetObjectInfo(ctx, bucket, object, opts)
-	}
+	object = encodeDirObject(object)
 	for _, zone := range z.zones {
 		objInfo, err = zone.GetObjectInfo(ctx, bucket, object, opts)
 		if err != nil {
@@ -539,6 +508,7 @@ func (z *erasureZones) GetObjectInfo(ctx context.Context, bucket, object string,
 		}
 		return objInfo, nil
 	}
+	object = decodeDirObject(object)
 	if opts.VersionID != "" {
 		return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
 	}
@@ -547,12 +517,7 @@ func (z *erasureZones) GetObjectInfo(ctx context.Context, bucket, object string,
 
 // PutObject - writes an object to least used erasure zone.
 func (z *erasureZones) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
-	// Lock the object.
-	lk := z.NewNSLock(ctx, bucket, object)
-	if err := lk.GetLock(globalOperationTimeout); err != nil {
-		return ObjectInfo{}, err
-	}
-	defer lk.Unlock()
+	object = encodeDirObject(object)
 
 	if z.SingleZone() {
 		return z.zones[0].PutObject(ctx, bucket, object, data, opts)
@@ -568,12 +533,7 @@ func (z *erasureZones) PutObject(ctx context.Context, bucket string, object stri
 }
 
 func (z *erasureZones) DeleteObject(ctx context.Context, bucket string, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	// Acquire a write lock before deleting the object.
-	lk := z.NewNSLock(ctx, bucket, object)
-	if err = lk.GetLock(globalOperationTimeout); err != nil {
-		return ObjectInfo{}, err
-	}
-	defer lk.Unlock()
+	object = encodeDirObject(object)
 
 	if z.SingleZone() {
 		return z.zones[0].DeleteObject(ctx, bucket, object, opts)
@@ -595,6 +555,8 @@ func (z *erasureZones) DeleteObjects(ctx context.Context, bucket string, objects
 	dobjects := make([]DeletedObject, len(objects))
 	objSets := set.NewStringSet()
 	for i := range derrs {
+		objects[i].ObjectName = encodeDirObject(objects[i].ObjectName)
+
 		derrs[i] = checkDelObjArgs(ctx, bucket, objects[i].ObjectName)
 		objSets.Add(objects[i].ObjectName)
 	}
@@ -626,15 +588,10 @@ func (z *erasureZones) DeleteObjects(ctx context.Context, bucket string, objects
 }
 
 func (z *erasureZones) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
-	// Check if this request is only metadata update.
+	srcObject = encodeDirObject(srcObject)
+	dstObject = encodeDirObject(dstObject)
+
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
-	if !cpSrcDstSame {
-		lk := z.NewNSLock(ctx, dstBucket, dstObject)
-		if err := lk.GetLock(globalOperationTimeout); err != nil {
-			return objInfo, err
-		}
-		defer lk.Unlock()
-	}
 
 	zoneIdx, err := z.getZoneIdx(ctx, dstBucket, dstObject, dstOpts, srcInfo.Size)
 	if err != nil {
@@ -642,12 +599,19 @@ func (z *erasureZones) CopyObject(ctx context.Context, srcBucket, srcObject, dst
 	}
 
 	if cpSrcDstSame && srcInfo.metadataOnly {
+		// Version ID is set for the destination and source == destination version ID.
 		if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
 			return z.zones[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
+		// Destination is not versioned and source version ID is empty
+		// perform an in-place update.
 		if !dstOpts.Versioned && srcOpts.VersionID == "" {
 			return z.zones[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		}
+		// Destination is versioned, source is not destination version,
+		// as a special case look for if the source object is not legacy
+		// from older format, for older format we will rewrite them as
+		// newer using PutObject() - this is an optimization to save space
 		if dstOpts.Versioned && srcOpts.VersionID != dstOpts.VersionID && !srcInfo.Legacy {
 			// CopyObject optimization where we don't create an entire copy
 			// of the content, instead we add a reference.
@@ -689,24 +653,24 @@ func (z *erasureZones) ListObjectsV2(ctx context.Context, bucket, prefix, contin
 
 func (z *erasureZones) listObjectsNonSlash(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
 
-	var zonesEntryChs [][]FileInfoCh
-	var zonesDrivesPerSet []int
+	zonesEntryChs := make([][]FileInfoCh, 0, len(z.zones))
+	zonesListTolerancePerSet := make([]int, 0, len(z.zones))
 
 	endWalkCh := make(chan struct{})
 	defer close(endWalkCh)
 
 	for _, zone := range z.zones {
 		zonesEntryChs = append(zonesEntryChs,
-			zone.startMergeWalksN(ctx, bucket, prefix, "", true, endWalkCh, zone.drivesPerSet))
-		zonesDrivesPerSet = append(zonesDrivesPerSet, zone.drivesPerSet)
+			zone.startMergeWalksN(ctx, bucket, prefix, "", true, endWalkCh, zone.setDriveCount, false))
+		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet)
 	}
 
 	var objInfos []ObjectInfo
 	var eof bool
 	var prevPrefix string
 
-	var zonesEntriesInfos [][]FileInfo
-	var zonesEntriesValid [][]bool
+	zonesEntriesInfos := make([][]FileInfo, 0, len(zonesEntryChs))
+	zonesEntriesValid := make([][]bool, 0, len(zonesEntryChs))
 	for _, entryChs := range zonesEntryChs {
 		zonesEntriesInfos = append(zonesEntriesInfos, make([]FileInfo, len(entryChs)))
 		zonesEntriesValid = append(zonesEntriesValid, make([]bool, len(entryChs)))
@@ -723,7 +687,7 @@ func (z *erasureZones) listObjectsNonSlash(ctx context.Context, bucket, prefix,
 			break
 		}
 
-		if quorumCount < zonesDrivesPerSet[zoneIndex]/2 {
+		if quorumCount < zonesListTolerancePerSet[zoneIndex] {
 			// Skip entries which are not found on upto ndisks/2.
 			continue
 		}
@@ -808,22 +772,22 @@ func (z *erasureZones) listObjectsSplunk(ctx context.Context, bucket, prefix, ma
 
 	recursive := true
 
-	var zonesEntryChs [][]FileInfoCh
-	var zonesEndWalkCh []chan struct{}
-	var drivesPerSets []int
+	zonesEntryChs := make([][]FileInfoCh, 0, len(z.zones))
+	zonesEndWalkCh := make([]chan struct{}, 0, len(z.zones))
+	zonesListTolerancePerSet := make([]int, 0, len(z.zones))
 
 	for _, zone := range z.zones {
 		entryChs, endWalkCh := zone.poolSplunk.Release(listParams{bucket, recursive, marker, prefix})
 		if entryChs == nil {
 			endWalkCh = make(chan struct{})
-			entryChs = zone.startSplunkMergeWalksN(ctx, bucket, prefix, marker, endWalkCh, zone.drivesPerSet)
+			entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.setDriveCount, true)
 		}
 		zonesEntryChs = append(zonesEntryChs, entryChs)
 		zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh)
-		drivesPerSets = append(drivesPerSets, zone.drivesPerSet)
+		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet)
 	}
 
-	entries := mergeZonesEntriesCh(zonesEntryChs, maxKeys, drivesPerSets)
+	entries := mergeZonesEntriesCh(zonesEntryChs, maxKeys, zonesListTolerancePerSet)
 	if len(entries.Files) == 0 {
 		return loi, nil
 	}
@@ -900,22 +864,22 @@ func (z *erasureZones) listObjects(ctx context.Context, bucket, prefix, marker,
 		recursive = false
 	}
 
-	var zonesEntryChs [][]FileInfoCh
-	var zonesEndWalkCh []chan struct{}
-	var drivesPerSets []int
+	zonesEntryChs := make([][]FileInfoCh, 0, len(z.zones))
+	zonesEndWalkCh := make([]chan struct{}, 0, len(z.zones))
+	zonesListTolerancePerSet := make([]int, 0, len(z.zones))
 
 	for _, zone := range z.zones {
 		entryChs, endWalkCh := zone.pool.Release(listParams{bucket, recursive, marker, prefix})
 		if entryChs == nil {
 			endWalkCh = make(chan struct{})
-			entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.drivesPerSet)
+			entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.setDriveCount, false)
 		}
 		zonesEntryChs = append(zonesEntryChs, entryChs)
 		zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh)
-		drivesPerSets = append(drivesPerSets, zone.drivesPerSet)
+		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet)
 	}
 
-	entries := mergeZonesEntriesCh(zonesEntryChs, maxKeys, drivesPerSets)
+	entries := mergeZonesEntriesCh(zonesEntryChs, maxKeys, zonesListTolerancePerSet)
 	if len(entries.Files) == 0 {
 		return loi, nil
 	}
@@ -951,18 +915,9 @@ func (z *erasureZones) listObjects(ctx context.Context, bucket, prefix, marker,
 // N times until this boolean is 'false'.
 func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileInfo, zoneEntriesValid [][]bool) (FileInfo, int, int, bool) {
 	for i, entryChs := range zoneEntryChs {
-		i := i
-		var wg sync.WaitGroup
 		for j := range entryChs {
-			j := j
-			wg.Add(1)
-			// Pop() entries in parallel for large drive setups.
-			go func() {
-				defer wg.Done()
-				zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
-			}()
+			zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
 		}
-		wg.Wait()
 	}
 
 	var isTruncated = false
@@ -995,7 +950,16 @@ func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileI
 				zoneIndex = i
 				continue
 			}
-			if zoneEntries[i][j].Name < lentry.Name {
+			str1 := zoneEntries[i][j].Name
+			str2 := lentry.Name
+			if HasSuffix(str1, globalDirSuffix) {
+				str1 = strings.TrimSuffix(str1, globalDirSuffix) + slashSeparator
+			}
+			if HasSuffix(str2, globalDirSuffix) {
+				str2 = strings.TrimSuffix(str2, globalDirSuffix) + slashSeparator
+			}
+
+			if str1 < str2 {
 				lentry = zoneEntries[i][j]
 				zoneIndex = i
 			}
@@ -1028,6 +992,10 @@ func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileI
 		}
 	}
 
+	if HasSuffix(lentry.Name, globalDirSuffix) {
+		lentry.Name = strings.TrimSuffix(lentry.Name, globalDirSuffix) + slashSeparator
+	}
+
 	return lentry, lexicallySortedEntryCount, zoneIndex, isTruncated
 }
 
@@ -1040,18 +1008,9 @@ func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileI
 // N times until this boolean is 'false'.
 func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneEntries [][]FileInfoVersions, zoneEntriesValid [][]bool) (FileInfoVersions, int, int, bool) {
 	for i, entryChs := range zoneEntryChs {
-		i := i
-		var wg sync.WaitGroup
 		for j := range entryChs {
-			j := j
-			wg.Add(1)
-			// Pop() entries in parallel for large drive setups.
-			go func() {
-				defer wg.Done()
-				zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
-			}()
+			zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
 		}
-		wg.Wait()
 	}
 
 	var isTruncated = false
@@ -1082,7 +1041,16 @@ func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneE
 				zoneIndex = i
 				continue
 			}
-			if zoneEntries[i][j].Name < lentry.Name {
+			str1 := zoneEntries[i][j].Name
+			str2 := lentry.Name
+			if HasSuffix(str1, globalDirSuffix) {
+				str1 = strings.TrimSuffix(str1, globalDirSuffix) + slashSeparator
+			}
+			if HasSuffix(str2, globalDirSuffix) {
+				str2 = strings.TrimSuffix(str2, globalDirSuffix) + slashSeparator
+			}
+
+			if str1 < str2 {
 				lentry = zoneEntries[i][j]
 				zoneIndex = i
 			}
@@ -1115,18 +1083,23 @@ func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneE
 		}
 	}
 
+	if HasSuffix(lentry.Name, globalDirSuffix) {
+		lentry.Name = strings.TrimSuffix(lentry.Name, globalDirSuffix) + slashSeparator
+	}
+
 	return lentry, lexicallySortedEntryCount, zoneIndex, isTruncated
 }
 
 // mergeZonesEntriesVersionsCh - merges FileInfoVersions channel to entries upto maxKeys.
-func mergeZonesEntriesVersionsCh(zonesEntryChs [][]FileInfoVersionsCh, maxKeys int, drivesPerSets []int) (entries FilesInfoVersions) {
+func mergeZonesEntriesVersionsCh(zonesEntryChs [][]FileInfoVersionsCh, maxKeys int, zonesListTolerancePerSet []int) (entries FilesInfoVersions) {
 	var i = 0
-	var zonesEntriesInfos [][]FileInfoVersions
-	var zonesEntriesValid [][]bool
+	zonesEntriesInfos := make([][]FileInfoVersions, 0, len(zonesEntryChs))
+	zonesEntriesValid := make([][]bool, 0, len(zonesEntryChs))
 	for _, entryChs := range zonesEntryChs {
 		zonesEntriesInfos = append(zonesEntriesInfos, make([]FileInfoVersions, len(entryChs)))
 		zonesEntriesValid = append(zonesEntriesValid, make([]bool, len(entryChs)))
 	}
+
 	for {
 		fi, quorumCount, zoneIndex, ok := lexicallySortedEntryZoneVersions(zonesEntryChs, zonesEntriesInfos, zonesEntriesValid)
 		if !ok {
@@ -1134,8 +1107,8 @@ func mergeZonesEntriesVersionsCh(zonesEntryChs [][]FileInfoVersionsCh, maxKeys i
 			break
 		}
 
-		if quorumCount < drivesPerSets[zoneIndex]/2 {
-			// Skip entries which are not found on upto ndisks/2.
+		if quorumCount < zonesListTolerancePerSet[zoneIndex] {
+			// Skip entries which are not found upto the expected tolerance
 			continue
 		}
 
@@ -1150,14 +1123,15 @@ func mergeZonesEntriesVersionsCh(zonesEntryChs [][]FileInfoVersionsCh, maxKeys i
 }
 
 // mergeZonesEntriesCh - merges FileInfo channel to entries upto maxKeys.
-func mergeZonesEntriesCh(zonesEntryChs [][]FileInfoCh, maxKeys int, drivesPerSets []int) (entries FilesInfo) {
+func mergeZonesEntriesCh(zonesEntryChs [][]FileInfoCh, maxKeys int, zonesListTolerancePerSet []int) (entries FilesInfo) {
 	var i = 0
-	var zonesEntriesInfos [][]FileInfo
-	var zonesEntriesValid [][]bool
+	zonesEntriesInfos := make([][]FileInfo, 0, len(zonesEntryChs))
+	zonesEntriesValid := make([][]bool, 0, len(zonesEntryChs))
 	for _, entryChs := range zonesEntryChs {
 		zonesEntriesInfos = append(zonesEntriesInfos, make([]FileInfo, len(entryChs)))
 		zonesEntriesValid = append(zonesEntriesValid, make([]bool, len(entryChs)))
 	}
+	var prevEntry string
 	for {
 		fi, quorumCount, zoneIndex, ok := lexicallySortedEntryZone(zonesEntryChs, zonesEntriesInfos, zonesEntriesValid)
 		if !ok {
@@ -1165,8 +1139,12 @@ func mergeZonesEntriesCh(zonesEntryChs [][]FileInfoCh, maxKeys int, drivesPerSet
 			break
 		}
 
-		if quorumCount < drivesPerSets[zoneIndex]/2 {
-			// Skip entries which are not found on upto ndisks/2.
+		if quorumCount < zonesListTolerancePerSet[zoneIndex] {
+			// Skip entries which are not found upto configured tolerance.
+			continue
+		}
+
+		if HasSuffix(fi.Name, slashSeparator) && fi.Name == prevEntry {
 			continue
 		}
 
@@ -1176,24 +1154,16 @@ func mergeZonesEntriesCh(zonesEntryChs [][]FileInfoCh, maxKeys int, drivesPerSet
 			entries.IsTruncated = isTruncatedZones(zonesEntryChs, zonesEntriesInfos, zonesEntriesValid)
 			break
 		}
+		prevEntry = fi.Name
 	}
 	return entries
 }
 
 func isTruncatedZones(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileInfo, zoneEntriesValid [][]bool) bool {
 	for i, entryChs := range zoneEntryChs {
-		i := i
-		var wg sync.WaitGroup
 		for j := range entryChs {
-			j := j
-			wg.Add(1)
-			// Pop() entries in parallel for large drive setups.
-			go func() {
-				defer wg.Done()
-				zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
-			}()
+			zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
 		}
-		wg.Wait()
 	}
 
 	var isTruncated = false
@@ -1214,24 +1184,16 @@ func isTruncatedZones(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileInfo, zon
 				zoneEntryChs[i][j].Push(zoneEntries[i][j])
 			}
 		}
+
 	}
 	return isTruncated
 }
 
 func isTruncatedZonesVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneEntries [][]FileInfoVersions, zoneEntriesValid [][]bool) bool {
 	for i, entryChs := range zoneEntryChs {
-		i := i
-		var wg sync.WaitGroup
 		for j := range entryChs {
-			j := j
-			wg.Add(1)
-			// Pop() entries in parallel for large drive setups.
-			go func() {
-				defer wg.Done()
-				zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
-			}()
+			zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
 		}
-		wg.Wait()
 	}
 
 	var isTruncated = false
@@ -1305,21 +1267,21 @@ func (z *erasureZones) listObjectVersions(ctx context.Context, bucket, prefix, m
 		recursive = false
 	}
 
-	var zonesEntryChs [][]FileInfoVersionsCh
-	var zonesEndWalkCh []chan struct{}
-	var drivesPerSets []int
+	zonesEntryChs := make([][]FileInfoVersionsCh, 0, len(z.zones))
+	zonesEndWalkCh := make([]chan struct{}, 0, len(z.zones))
+	zonesListTolerancePerSet := make([]int, 0, len(z.zones))
 	for _, zone := range z.zones {
 		entryChs, endWalkCh := zone.poolVersions.Release(listParams{bucket, recursive, marker, prefix})
 		if entryChs == nil {
 			endWalkCh = make(chan struct{})
-			entryChs = zone.startMergeWalksVersionsN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.drivesPerSet)
+			entryChs = zone.startMergeWalksVersionsN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.setDriveCount)
 		}
 		zonesEntryChs = append(zonesEntryChs, entryChs)
 		zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh)
-		drivesPerSets = append(drivesPerSets, zone.drivesPerSet)
+		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet)
 	}
 
-	entries := mergeZonesEntriesVersionsCh(zonesEntryChs, maxKeys, drivesPerSets)
+	entries := mergeZonesEntriesVersionsCh(zonesEntryChs, maxKeys, zonesListTolerancePerSet)
 	if len(entries.FilesVersions) == 0 {
 		return loi, nil
 	}
@@ -1416,12 +1378,6 @@ func (z *erasureZones) PutObjectPart(ctx context.Context, bucket, object, upload
 		return PartInfo{}, err
 	}
 
-	uploadIDLock := z.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := uploadIDLock.GetLock(globalOperationTimeout); err != nil {
-		return PartInfo{}, err
-	}
-	defer uploadIDLock.Unlock()
-
 	if z.SingleZone() {
 		return z.zones[0].PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 	}
@@ -1452,12 +1408,6 @@ func (z *erasureZones) GetMultipartInfo(ctx context.Context, bucket, object, upl
 		return MultipartInfo{}, err
 	}
 
-	uploadIDLock := z.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
-		return MultipartInfo{}, err
-	}
-	defer uploadIDLock.RUnlock()
-
 	if z.SingleZone() {
 		return z.zones[0].GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 	}
@@ -1488,12 +1438,6 @@ func (z *erasureZones) ListObjectParts(ctx context.Context, bucket, object, uplo
 		return ListPartsInfo{}, err
 	}
 
-	uploadIDLock := z.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
-		return ListPartsInfo{}, err
-	}
-	defer uploadIDLock.RUnlock()
-
 	if z.SingleZone() {
 		return z.zones[0].ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
 	}
@@ -1516,25 +1460,19 @@ func (z *erasureZones) ListObjectParts(ctx context.Context, bucket, object, uplo
 }
 
 // Aborts an in-progress multipart operation on hashedSet based on the object name.
-func (z *erasureZones) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error {
+func (z *erasureZones) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
 	if err := checkAbortMultipartArgs(ctx, bucket, object, z); err != nil {
 		return err
 	}
 
-	uploadIDLock := z.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := uploadIDLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer uploadIDLock.Unlock()
-
 	if z.SingleZone() {
-		return z.zones[0].AbortMultipartUpload(ctx, bucket, object, uploadID)
+		return z.zones[0].AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 	}
 
 	for _, zone := range z.zones {
-		_, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, ObjectOptions{})
+		_, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
 		if err == nil {
-			return zone.AbortMultipartUpload(ctx, bucket, object, uploadID)
+			return zone.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
 		}
 		switch err.(type) {
 		case InvalidUploadID:
@@ -1556,22 +1494,6 @@ func (z *erasureZones) CompleteMultipartUpload(ctx context.Context, bucket, obje
 		return objInfo, err
 	}
 
-	// Hold read-locks to verify uploaded parts, also disallows
-	// parallel part uploads as well.
-	uploadIDLock := z.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err = uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
-		return objInfo, err
-	}
-	defer uploadIDLock.RUnlock()
-
-	// Hold namespace to complete the transaction, only hold
-	// if uploadID can be held exclusively.
-	lk := z.NewNSLock(ctx, bucket, object)
-	if err = lk.GetLock(globalOperationTimeout); err != nil {
-		return objInfo, err
-	}
-	defer lk.Unlock()
-
 	if z.SingleZone() {
 		return z.zones[0].CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
 	}
@@ -1735,13 +1657,8 @@ func (z *erasureZones) ListBuckets(ctx context.Context) (buckets []BucketInfo, e
 }
 
 func (z *erasureZones) ReloadFormat(ctx context.Context, dryRun bool) error {
-	// Acquire lock on format.json
-	formatLock := z.NewNSLock(ctx, minioMetaBucket, formatConfigFile)
-	if err := formatLock.GetRLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer formatLock.RUnlock()
-
+	// No locks needed since reload happens in HealFormat under
+	// write lock across all nodes.
 	for _, zone := range z.zones {
 		if err := zone.ReloadFormat(ctx, dryRun); err != nil {
 			return err
@@ -1766,13 +1683,13 @@ func (z *erasureZones) HealFormat(ctx context.Context, dryRun bool) (madmin.Heal
 	var countNoHeal int
 	for _, zone := range z.zones {
 		result, err := zone.HealFormat(ctx, dryRun)
-		if err != nil && err != errNoHealRequired {
+		if err != nil && !errors.Is(err, errNoHealRequired) {
 			logger.LogIf(ctx, err)
 			continue
 		}
 		// Count errNoHealRequired across all zones,
 		// to return appropriate error to the caller
-		if err == errNoHealRequired {
+		if errors.Is(err, errNoHealRequired) {
 			countNoHeal++
 		}
 		r.DiskCount += result.DiskCount
@@ -1780,10 +1697,21 @@ func (z *erasureZones) HealFormat(ctx context.Context, dryRun bool) (madmin.Heal
 		r.Before.Drives = append(r.Before.Drives, result.Before.Drives...)
 		r.After.Drives = append(r.After.Drives, result.After.Drives...)
 	}
+
+	// Healing succeeded notify the peers to reload format and re-initialize disks.
+	// We will not notify peers if healing is not required.
+	for _, nerr := range globalNotificationSys.ReloadFormat(dryRun) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+
 	// No heal returned by all zones, return errNoHealRequired
 	if countNoHeal == len(z.zones) {
 		return r, errNoHealRequired
 	}
+
 	return r, nil
 }
 
@@ -1830,7 +1758,7 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results
 
 		var zoneDrivesPerSet []int
 		for _, zone := range z.zones {
-			zoneDrivesPerSet = append(zoneDrivesPerSet, zone.drivesPerSet)
+			zoneDrivesPerSet = append(zoneDrivesPerSet, zone.setDriveCount)
 		}
 
 		var zonesEntriesInfos [][]FileInfoVersions
@@ -1864,18 +1792,15 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results
 		return nil
 	}
 
-	var zonesEntryChs [][]FileInfoCh
+	zonesEntryChs := make([][]FileInfoCh, 0, len(z.zones))
+	zoneDrivesPerSet := make([]int, 0, len(z.zones))
 	for _, zone := range z.zones {
 		zonesEntryChs = append(zonesEntryChs, zone.startMergeWalks(ctx, bucket, prefix, "", true, ctx.Done()))
+		zoneDrivesPerSet = append(zoneDrivesPerSet, zone.setDriveCount)
 	}
 
-	var zoneDrivesPerSet []int
-	for _, zone := range z.zones {
-		zoneDrivesPerSet = append(zoneDrivesPerSet, zone.drivesPerSet)
-	}
-
-	var zonesEntriesInfos [][]FileInfo
-	var zonesEntriesValid [][]bool
+	zonesEntriesInfos := make([][]FileInfo, 0, len(zonesEntryChs))
+	zonesEntriesValid := make([][]bool, 0, len(zonesEntryChs))
 	for _, entryChs := range zonesEntryChs {
 		zonesEntriesInfos = append(zonesEntriesInfos, make([]FileInfo, len(entryChs)))
 		zonesEntriesValid = append(zonesEntriesValid, make([]bool, len(entryChs)))
@@ -1906,23 +1831,20 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results
 type HealObjectFn func(bucket, object, versionID string) error
 
 func (z *erasureZones) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObject HealObjectFn) error {
-	var zonesEntryChs [][]FileInfoVersionsCh
-
 	endWalkCh := make(chan struct{})
 	defer close(endWalkCh)
 
+	zonesEntryChs := make([][]FileInfoVersionsCh, 0, len(z.zones))
+	zoneDrivesPerSet := make([]int, 0, len(z.zones))
+
 	for _, zone := range z.zones {
 		zonesEntryChs = append(zonesEntryChs,
 			zone.startMergeWalksVersions(ctx, bucket, prefix, "", true, endWalkCh))
+		zoneDrivesPerSet = append(zoneDrivesPerSet, zone.setDriveCount)
 	}
 
-	var zoneDrivesPerSet []int
-	for _, zone := range z.zones {
-		zoneDrivesPerSet = append(zoneDrivesPerSet, zone.drivesPerSet)
-	}
-
-	var zonesEntriesInfos [][]FileInfoVersions
-	var zonesEntriesValid [][]bool
+	zonesEntriesInfos := make([][]FileInfoVersions, 0, len(zonesEntryChs))
+	zonesEntriesValid := make([][]bool, 0, len(zonesEntryChs))
 	for _, entryChs := range zonesEntryChs {
 		zonesEntriesInfos = append(zonesEntriesInfos, make([]FileInfoVersions, len(entryChs)))
 		zonesEntriesValid = append(zonesEntriesValid, make([]bool, len(entryChs)))
@@ -1962,6 +1884,8 @@ func (z *erasureZones) HealObjects(ctx context.Context, bucket, prefix string, o
 }
 
 func (z *erasureZones) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
+	object = encodeDirObject(object)
+
 	lk := z.NewNSLock(ctx, bucket, object)
 	if bucket == minioMetaBucket {
 		// For .minio.sys bucket heals we should hold write locks.
@@ -2080,27 +2004,49 @@ func (z *erasureZones) Health(ctx context.Context, opts HealthOptions) HealthRes
 
 	reqInfo := (&logger.ReqInfo{}).AppendTags("maintenance", strconv.FormatBool(opts.Maintenance))
 
+	parityDrives := globalStorageClass.GetParityForSC(storageclass.STANDARD)
+	diskCount := z.SetDriveCount()
+
+	if parityDrives == 0 {
+		parityDrives = getDefaultParityBlocks(diskCount)
+	}
+	dataDrives := diskCount - parityDrives
+	writeQuorum := dataDrives
+	if dataDrives == parityDrives {
+		writeQuorum++
+	}
+
+	var aggHealStateResult madmin.BgHealState
+	if opts.Maintenance {
+		// check if local disks are being healed, if they are being healed
+		// we need to tell healthy status as 'false' so that this server
+		// is not taken down for maintenance
+		var err error
+		aggHealStateResult, err = getAggregatedBackgroundHealState(ctx)
+		if err != nil {
+			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Unable to verify global heal status: %w", err))
+			return HealthResult{
+				Healthy: false,
+			}
+		}
+
+		if len(aggHealStateResult.HealDisks) > 0 {
+			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Total drives to be healed %d", len(aggHealStateResult.HealDisks)))
+		}
+	}
+
 	for zoneIdx := range erasureSetUpCount {
-		parityDrives := globalStorageClass.GetParityForSC(storageclass.STANDARD)
-		diskCount := z.zones[zoneIdx].drivesPerSet
-		if parityDrives == 0 {
-			parityDrives = getDefaultParityBlocks(diskCount)
-		}
-		dataDrives := diskCount - parityDrives
-		writeQuorum := dataDrives
-		if dataDrives == parityDrives {
-			writeQuorum++
-		}
 		for setIdx := range erasureSetUpCount[zoneIdx] {
 			if erasureSetUpCount[zoneIdx][setIdx] < writeQuorum {
 				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
 					fmt.Errorf("Write quorum may be lost on zone: %d, set: %d, expected write quorum: %d",
 						zoneIdx, setIdx, writeQuorum))
 				return HealthResult{
-					Healthy:     false,
-					ZoneID:      zoneIdx,
-					SetID:       setIdx,
-					WriteQuorum: writeQuorum,
+					Healthy:       false,
+					HealingDrives: len(aggHealStateResult.HealDisks),
+					ZoneID:        zoneIdx,
+					SetID:         setIdx,
+					WriteQuorum:   writeQuorum,
 				}
 			}
 		}
@@ -2110,35 +2056,21 @@ func (z *erasureZones) Health(ctx context.Context, opts HealthOptions) HealthRes
 	// to look at the healing side of the code.
 	if !opts.Maintenance {
 		return HealthResult{
-			Healthy: true,
+			Healthy:     true,
+			WriteQuorum: writeQuorum,
 		}
 	}
 
-	// check if local disks are being healed, if they are being healed
-	// we need to tell healthy status as 'false' so that this server
-	// is not taken down for maintenance
-	aggHealStateResult, err := getAggregatedBackgroundHealState(ctx, true)
-	if err != nil {
-		logger.LogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Unable to verify global heal status: %w", err))
-		return HealthResult{
-			Healthy: false,
-		}
-	}
-
-	if len(aggHealStateResult.HealDisks) > 0 {
-		logger.LogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Total drives to be healed %d", len(aggHealStateResult.HealDisks)))
-	}
-
-	healthy := len(aggHealStateResult.HealDisks) == 0
-
 	return HealthResult{
-		Healthy:       healthy,
+		Healthy:       len(aggHealStateResult.HealDisks) == 0,
 		HealingDrives: len(aggHealStateResult.HealDisks),
+		WriteQuorum:   writeQuorum,
 	}
 }
 
 // PutObjectTags - replace or add tags to an existing object
 func (z *erasureZones) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) error {
+	object = encodeDirObject(object)
 	if z.SingleZone() {
 		return z.zones[0].PutObjectTags(ctx, bucket, object, tags, opts)
 	}
@@ -2168,6 +2100,7 @@ func (z *erasureZones) PutObjectTags(ctx context.Context, bucket, object string,
 
 // DeleteObjectTags - delete object tags from an existing object
 func (z *erasureZones) DeleteObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+	object = encodeDirObject(object)
 	if z.SingleZone() {
 		return z.zones[0].DeleteObjectTags(ctx, bucket, object, opts)
 	}
@@ -2196,6 +2129,7 @@ func (z *erasureZones) DeleteObjectTags(ctx context.Context, bucket, object stri
 
 // GetObjectTags - get object tags from an existing object
 func (z *erasureZones) GetObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (*tags.Tags, error) {
+	object = encodeDirObject(object)
 	if z.SingleZone() {
 		return z.zones[0].GetObjectTags(ctx, bucket, object, opts)
 	}

cmd/erasure.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bpool"
+	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/dsync"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/sync/errgroup"
@@ -80,6 +81,14 @@ func (er erasureObjects) SetDriveCount() int {
 func (er erasureObjects) Shutdown(ctx context.Context) error {
 	// Add any object layer shutdown activities here.
 	closeStorageDisks(er.getDisks())
+	select {
+	case _, ok := <-er.mrfOpCh:
+		if ok {
+			close(er.mrfOpCh)
+		}
+	default:
+		close(er.mrfOpCh)
+	}
 	return nil
 }
 
@@ -131,6 +140,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 		index := index
 		g.Go(func() error {
 			if disks[index] == OfflineDisk {
+				logger.LogIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]))
 				disksInfo[index] = madmin.Disk{
 					State:    diskErrToDriveState(errDiskNotFound),
 					Endpoint: endpoints[index],
@@ -138,13 +148,16 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 				// Storage disk is empty, perhaps ignored disk or not available.
 				return errDiskNotFound
 			}
-			info, err := disks[index].DiskInfo()
+			info, err := disks[index].DiskInfo(context.TODO())
 			if err != nil {
-				if !IsErr(err, baseErrs...) {
-					reqInfo := (&logger.ReqInfo{}).AppendTags("disk", disks[index].String())
-					ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-					logger.LogIf(ctx, err)
+				reqInfo := (&logger.ReqInfo{}).AppendTags("disk", disks[index].String())
+				ctx := logger.SetReqInfo(GlobalContext, reqInfo)
+				logger.LogIf(ctx, err)
+				disksInfo[index] = madmin.Disk{
+					State:    diskErrToDriveState(err),
+					Endpoint: endpoints[index],
 				}
+				return err
 			}
 			di := madmin.Disk{
 				Endpoint:       endpoints[index],
@@ -239,37 +252,49 @@ func (er erasureObjects) StorageInfo(ctx context.Context, local bool) (StorageIn
 	return getStorageInfo(disks, endpoints)
 }
 
-// GetMetrics - is not implemented and shouldn't be called.
-func (er erasureObjects) GetMetrics(ctx context.Context) (*Metrics, error) {
-	logger.LogIf(ctx, NotImplemented{})
-	return &Metrics{}, NotImplemented{}
-}
-
-// CrawlAndGetDataUsage collects usage from all buckets.
-// updates are sent as different parts of the underlying
-// structure has been traversed.
-func (er erasureObjects) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo) error {
-	return NotImplemented{API: "CrawlAndGetDataUsage"}
-}
-
 // CrawlAndGetDataUsage will start crawling buckets and send updated totals as they are traversed.
 // Updates are sent on a regular basis and the caller *must* consume them.
 func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, updates chan<- dataUsageCache) error {
-	var disks []StorageAPI
+	if len(buckets) == 0 {
+		return nil
+	}
 
+	// Collect any disk healing.
+	healing, err := getAggregatedBackgroundHealState(ctx)
+	if err != nil {
+		return err
+	}
+
+	healDisks := make(map[string]struct{}, len(healing.HealDisks))
+	for _, disk := range healing.HealDisks {
+		healDisks[disk] = struct{}{}
+	}
+
+	// Collect disks we can use.
+	var disks []StorageAPI
 	for _, d := range er.getLoadBalancedDisks() {
 		if d == nil || !d.IsOnline() {
 			continue
 		}
+		di, err := d.DiskInfo(ctx)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		if _, ok := healDisks[di.Endpoint]; ok {
+			logger.Info(color.Green("data-crawl:")+" Disk %q is Healing, skipping disk.", di.Endpoint)
+			continue
+		}
 		disks = append(disks, d)
 	}
-	if len(disks) == 0 || len(buckets) == 0 {
+	if len(disks) == 0 {
+		logger.Info(color.Green("data-crawl:") + " No disks found, skipping crawl")
 		return nil
 	}
 
 	// Load bucket totals
 	oldCache := dataUsageCache{}
-	err := oldCache.load(ctx, er, dataUsageCacheName)
+	err = oldCache.load(ctx, er, dataUsageCacheName)
 	if err != nil {
 		return err
 	}
@@ -408,9 +433,3 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
 
 	return nil
 }
-
-// Health shouldn't be called directly - will panic
-func (er erasureObjects) Health(ctx context.Context, _ HealthOptions) HealthResult {
-	logger.CriticalIf(ctx, NotImplemented{})
-	return HealthResult{}
-}

cmd/erasure_test.go

@@ -134,7 +134,7 @@ func newErasureTestSetup(dataBlocks int, parityBlocks int, blockSize int64) (*er
 		if err != nil {
 			return nil, err
 		}
-		err = disks[i].MakeVol("testbucket")
+		err = disks[i].MakeVol(context.Background(), "testbucket")
 		if err != nil {
 			return nil, err
 		}

cmd/format-erasure.go

@@ -355,15 +355,15 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3) error {
 	tmpFormat := mustGetUUID()
 
 	// Purge any existing temporary file, okay to ignore errors here.
-	defer disk.DeleteFile(minioMetaBucket, tmpFormat)
+	defer disk.DeleteFile(context.TODO(), minioMetaBucket, tmpFormat)
 
 	// write to unique file.
-	if err = disk.WriteAll(minioMetaBucket, tmpFormat, bytes.NewReader(formatBytes)); err != nil {
+	if err = disk.WriteAll(context.TODO(), minioMetaBucket, tmpFormat, bytes.NewReader(formatBytes)); err != nil {
 		return err
 	}
 
 	// Rename file `uuid.json` --> `format.json`.
-	if err = disk.RenameFile(minioMetaBucket, tmpFormat, minioMetaBucket, formatConfigFile); err != nil {
+	if err = disk.RenameFile(context.TODO(), minioMetaBucket, tmpFormat, minioMetaBucket, formatConfigFile); err != nil {
 		return err
 	}
 
@@ -392,13 +392,13 @@ func isHiddenDirectories(vols ...VolInfo) bool {
 
 // loadFormatErasure - loads format.json from disk.
 func loadFormatErasure(disk StorageAPI) (format *formatErasureV3, err error) {
-	buf, err := disk.ReadAll(minioMetaBucket, formatConfigFile)
+	buf, err := disk.ReadAll(context.TODO(), minioMetaBucket, formatConfigFile)
 	if err != nil {
 		// 'file not found' and 'volume not found' as
 		// same. 'volume not found' usually means its a fresh disk.
 		if err == errFileNotFound || err == errVolumeNotFound {
 			var vols []VolInfo
-			vols, err = disk.ListVols()
+			vols, err = disk.ListVols(context.TODO())
 			if err != nil {
 				return nil, err
 			}
@@ -439,7 +439,7 @@ func checkFormatErasureValue(formatErasure *formatErasureV3) error {
 }
 
 // Check all format values.
-func checkFormatErasureValues(formats []*formatErasureV3, drivesPerSet int) error {
+func checkFormatErasureValues(formats []*formatErasureV3, setDriveCount int) error {
 	for i, formatErasure := range formats {
 		if formatErasure == nil {
 			continue
@@ -454,8 +454,8 @@ func checkFormatErasureValues(formats []*formatErasureV3, drivesPerSet int) erro
 		// Only if custom erasure drive count is set,
 		// we should fail here other proceed to honor what
 		// is present on the disk.
-		if globalCustomErasureDriveCount && len(formatErasure.Erasure.Sets[0]) != drivesPerSet {
-			return fmt.Errorf("%s disk is already formatted with %d drives per erasure set. This cannot be changed to %d, please revert your MINIO_ERASURE_SET_DRIVE_COUNT setting", humanize.Ordinal(i+1), len(formatErasure.Erasure.Sets[0]), drivesPerSet)
+		if globalCustomErasureDriveCount && len(formatErasure.Erasure.Sets[0]) != setDriveCount {
+			return fmt.Errorf("%s disk is already formatted with %d drives per erasure set. This cannot be changed to %d, please revert your MINIO_ERASURE_SET_DRIVE_COUNT setting", humanize.Ordinal(i+1), len(formatErasure.Erasure.Sets[0]), setDriveCount)
 		}
 	}
 	return nil
@@ -788,22 +788,22 @@ func fixFormatErasureV3(storageDisks []StorageAPI, endpoints Endpoints, formats
 }
 
 // initFormatErasure - save Erasure format configuration on all disks.
-func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount, drivesPerSet int, deploymentID string, sErrs []error) (*formatErasureV3, error) {
-	format := newFormatErasureV3(setCount, drivesPerSet)
+func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount, setDriveCount int, deploymentID string, sErrs []error) (*formatErasureV3, error) {
+	format := newFormatErasureV3(setCount, setDriveCount)
 	formats := make([]*formatErasureV3, len(storageDisks))
-	wantAtMost := ecDrivesNoConfig(drivesPerSet)
+	wantAtMost := ecDrivesNoConfig(setDriveCount)
 
 	for i := 0; i < setCount; i++ {
-		hostCount := make(map[string]int, drivesPerSet)
-		for j := 0; j < drivesPerSet; j++ {
-			disk := storageDisks[i*drivesPerSet+j]
+		hostCount := make(map[string]int, setDriveCount)
+		for j := 0; j < setDriveCount; j++ {
+			disk := storageDisks[i*setDriveCount+j]
 			newFormat := format.Clone()
 			newFormat.Erasure.This = format.Erasure.Sets[i][j]
 			if deploymentID != "" {
 				newFormat.ID = deploymentID
 			}
 			hostCount[disk.Hostname()]++
-			formats[i*drivesPerSet+j] = newFormat
+			formats[i*setDriveCount+j] = newFormat
 		}
 		if len(hostCount) > 0 {
 			var once sync.Once
@@ -817,8 +817,8 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 							return
 						}
 						logger.Info(" * Set %v:", i+1)
-						for j := 0; j < drivesPerSet; j++ {
-							disk := storageDisks[i*drivesPerSet+j]
+						for j := 0; j < setDriveCount; j++ {
+							disk := storageDisks[i*setDriveCount+j]
 							logger.Info("   - Drive: %s", disk.String())
 						}
 					})
@@ -842,15 +842,15 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 
 // ecDrivesNoConfig returns the erasure coded drives in a set if no config has been set.
 // It will attempt to read it from env variable and fall back to drives/2.
-func ecDrivesNoConfig(drivesPerSet int) int {
+func ecDrivesNoConfig(setDriveCount int) int {
 	ecDrives := globalStorageClass.GetParityForSC(storageclass.STANDARD)
 	if ecDrives == 0 {
-		cfg, err := storageclass.LookupConfig(nil, drivesPerSet)
+		cfg, err := storageclass.LookupConfig(nil, setDriveCount)
 		if err == nil {
 			ecDrives = cfg.Standard.Parity
 		}
 		if ecDrives == 0 {
-			ecDrives = drivesPerSet / 2
+			ecDrives = setDriveCount / 2
 		}
 	}
 	return ecDrives
@@ -862,7 +862,7 @@ func makeFormatErasureMetaVolumes(disk StorageAPI) error {
 		return errDiskNotFound
 	}
 	// Attempt to create MinIO internal buckets.
-	return disk.MakeVolBulk(minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, dataUsageBucket)
+	return disk.MakeVolBulk(context.TODO(), minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, dataUsageBucket)
 }
 
 // Get all UUIDs which are present in reference format should
@@ -920,14 +920,14 @@ func markUUIDsOffline(refFormat *formatErasureV3, formats []*formatErasureV3) {
 }
 
 // Initialize a new set of set formats which will be written to all disks.
-func newHealFormatSets(refFormat *formatErasureV3, setCount, drivesPerSet int, formats []*formatErasureV3, errs []error) [][]*formatErasureV3 {
+func newHealFormatSets(refFormat *formatErasureV3, setCount, setDriveCount int, formats []*formatErasureV3, errs []error) [][]*formatErasureV3 {
 	newFormats := make([][]*formatErasureV3, setCount)
 	for i := range refFormat.Erasure.Sets {
-		newFormats[i] = make([]*formatErasureV3, drivesPerSet)
+		newFormats[i] = make([]*formatErasureV3, setDriveCount)
 	}
 	for i := range refFormat.Erasure.Sets {
 		for j := range refFormat.Erasure.Sets[i] {
-			if errs[i*drivesPerSet+j] == errUnformattedDisk || errs[i*drivesPerSet+j] == nil {
+			if errs[i*setDriveCount+j] == errUnformattedDisk || errs[i*setDriveCount+j] == nil {
 				newFormats[i][j] = &formatErasureV3{}
 				newFormats[i][j].Version = refFormat.Version
 				newFormats[i][j].ID = refFormat.ID
@@ -935,13 +935,13 @@ func newHealFormatSets(refFormat *formatErasureV3, setCount, drivesPerSet int, f
 				newFormats[i][j].Erasure.Version = refFormat.Erasure.Version
 				newFormats[i][j].Erasure.DistributionAlgo = refFormat.Erasure.DistributionAlgo
 			}
-			if errs[i*drivesPerSet+j] == errUnformattedDisk {
+			if errs[i*setDriveCount+j] == errUnformattedDisk {
 				newFormats[i][j].Erasure.This = ""
 				newFormats[i][j].Erasure.Sets = nil
 				continue
 			}
-			if errs[i*drivesPerSet+j] == nil {
-				newFormats[i][j].Erasure.This = formats[i*drivesPerSet+j].Erasure.This
+			if errs[i*setDriveCount+j] == nil {
+				newFormats[i][j].Erasure.This = formats[i*setDriveCount+j].Erasure.This
 				newFormats[i][j].Erasure.Sets = nil
 			}
 		}

cmd/format-erasure_test.go

@@ -324,16 +324,16 @@ func TestCheckFormatErasureValue(t *testing.T) {
 // Tests getFormatErasureInQuorum()
 func TestGetFormatErasureInQuorumCheck(t *testing.T) {
 	setCount := 2
-	drivesPerSet := 16
+	setDriveCount := 16
 
-	format := newFormatErasureV3(setCount, drivesPerSet)
+	format := newFormatErasureV3(setCount, setDriveCount)
 	formats := make([]*formatErasureV3, 32)
 
 	for i := 0; i < setCount; i++ {
-		for j := 0; j < drivesPerSet; j++ {
+		for j := 0; j < setDriveCount; j++ {
 			newFormat := format.Clone()
 			newFormat.Erasure.This = format.Erasure.Sets[i][j]
-			formats[i*drivesPerSet+j] = newFormat
+			formats[i*setDriveCount+j] = newFormat
 		}
 	}
 
@@ -390,16 +390,16 @@ func TestGetFormatErasureInQuorumCheck(t *testing.T) {
 // Tests formatErasureGetDeploymentID()
 func TestGetErasureID(t *testing.T) {
 	setCount := 2
-	drivesPerSet := 8
+	setDriveCount := 8
 
-	format := newFormatErasureV3(setCount, drivesPerSet)
+	format := newFormatErasureV3(setCount, setDriveCount)
 	formats := make([]*formatErasureV3, 16)
 
 	for i := 0; i < setCount; i++ {
-		for j := 0; j < drivesPerSet; j++ {
+		for j := 0; j < setDriveCount; j++ {
 			newFormat := format.Clone()
 			newFormat.Erasure.This = format.Erasure.Sets[i][j]
-			formats[i*drivesPerSet+j] = newFormat
+			formats[i*setDriveCount+j] = newFormat
 		}
 	}
 
@@ -445,17 +445,17 @@ func TestGetErasureID(t *testing.T) {
 // Initialize new format sets.
 func TestNewFormatSets(t *testing.T) {
 	setCount := 2
-	drivesPerSet := 16
+	setDriveCount := 16
 
-	format := newFormatErasureV3(setCount, drivesPerSet)
+	format := newFormatErasureV3(setCount, setDriveCount)
 	formats := make([]*formatErasureV3, 32)
 	errs := make([]error, 32)
 
 	for i := 0; i < setCount; i++ {
-		for j := 0; j < drivesPerSet; j++ {
+		for j := 0; j < setDriveCount; j++ {
 			newFormat := format.Clone()
 			newFormat.Erasure.This = format.Erasure.Sets[i][j]
-			formats[i*drivesPerSet+j] = newFormat
+			formats[i*setDriveCount+j] = newFormat
 		}
 	}
 
@@ -467,7 +467,7 @@ func TestNewFormatSets(t *testing.T) {
 	// 16th disk is unformatted.
 	errs[15] = errUnformattedDisk
 
-	newFormats := newHealFormatSets(quorumFormat, setCount, drivesPerSet, formats, errs)
+	newFormats := newHealFormatSets(quorumFormat, setCount, setDriveCount, formats, errs)
 	if newFormats == nil {
 		t.Fatal("Unexpected failure")
 	}

cmd/fs-v1-helpers.go

@@ -41,9 +41,8 @@ func fsRemoveFile(ctx context.Context, filePath string) (err error) {
 		return err
 	}
 
-	if err = os.Remove((filePath)); err != nil {
-		err = osErrToFileErr(err)
-		if err != errFileNotFound {
+	if err = os.Remove(filePath); err != nil {
+		if err = osErrToFileErr(err); err != errFileNotFound {
 			logger.LogIf(ctx, err)
 		}
 	}

cmd/fs-v1-metadata.go

@@ -225,7 +225,6 @@ func (m *fsMetaV1) ReadFrom(ctx context.Context, lk *lock.LockedFile) (n int64,
 	}
 
 	if len(fsMetaBuf) == 0 {
-		logger.LogIf(ctx, io.EOF)
 		return 0, io.EOF
 	}