optionally allow strict quorum listing (#10649)

```
export MINIO_API_LIST_STRICT_QUORUM=on
```

would enable listing in quorum if necessary

.github/workflows/codeql.yml

@@ -1,51 +0,0 @@
-name: "Code scanning - action"
-
-on:
-  push:
-  pull_request:
-  schedule:
-    - cron: '0 19 * * 0'
-
-jobs:
-  CodeQL-Build:
-
-    # CodeQL runs on ubuntu-latest and windows-latest
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        # We must fetch at least the immediate parents so that if this is
-        # a pull request then we can checkout the head.
-        fetch-depth: 2
-
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-      
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-         languages: go, javascript
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.github/workflows/go.yml

@@ -37,6 +37,8 @@ jobs:
           GO111MODULE: on
           MINIO_CI_CD: 1
         run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           sudo apt-get install devscripts shellcheck
           nancy_version=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/sonatype-nexus-community/nancy/releases/latest | sed "s/https:\/\/github.com\/sonatype-nexus-community\/nancy\/releases\/tag\///")
           curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-linux.amd64-${nancy_version} && chmod +x nancy

.nancy-ignore

@@ -1,6 +0,0 @@
-CVE-2018-17075
-CVE-2018-17848
-CVE-2018-17846
-CVE-2018-17847
-CVE-2018-17143
-CVE-2018-17142

Makefile

@@ -18,11 +18,17 @@ getdeps:
 	@mkdir -p ${GOPATH}/bin
 	@which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0)
 	@which ruleguard 1>/dev/null || (echo "Installing ruleguard" && GO111MODULE=off go get github.com/quasilyte/go-ruleguard/...)
+	@which msgp 1>/dev/null || (echo "Installing msgp" && GO111MODULE=off go get github.com/tinylib/msgp)
+	@which stringer 1>/dev/null || (echo "Installing stringer" && GO111MODULE=off go get golang.org/x/tools/cmd/stringer)
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
 
-verifiers: getdeps fmt lint ruleguard
+verifiers: getdeps fmt lint ruleguard check-gen
+
+check-gen:
+	@go generate ./... >/dev/null
+	@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
 
 fmt:
 	@echo "Running $@ check"

browser/app/js/browser/BrowserDropdown.js

@@ -57,22 +57,6 @@ export class BrowserDropdown extends React.Component {
     const { fetchServerInfo } = this.props
     fetchServerInfo()
   }
-  fullScreen(e) {
-    e.preventDefault()
-    let el = document.documentElement
-    if (el.requestFullscreen) {
-      el.requestFullscreen()
-    }
-    if (el.mozRequestFullScreen) {
-      el.mozRequestFullScreen()
-    }
-    if (el.webkitRequestFullscreen) {
-      el.webkitRequestFullscreen()
-    }
-    if (el.msRequestFullscreen) {
-      el.msRequestFullscreen()
-    }
-  }
   logout(e) {
     e.preventDefault()
     web.Logout()
@@ -87,24 +71,30 @@ export class BrowserDropdown extends React.Component {
             <i className="fas fa-bars" />
           </Dropdown.Toggle>
           <Dropdown.Menu className="dropdown-menu-right">
+            <li>
+              <a href="" onClick={this.showChangePassword.bind(this)}>
+                Change Password <i className="fas fa-cog" />
+              </a>
+              {this.state.showChangePasswordModal && (
+                <ChangePasswordModal
+                  serverInfo={serverInfo}
+                  hideChangePassword={this.hideChangePassword.bind(this)}
+                />
+              )}
+            </li>
+            <li>
+              <a target="_blank" href="https://docs.min.io/?ref=ob">
+                Documentation <i className="fas fa-book" />
+              </a>
+            </li>
             <li>
               <a target="_blank" href="https://github.com/minio/minio">
                 GitHub <i className="fab fa-github" />
               </a>
             </li>
             <li>
-              <a href="" onClick={this.fullScreen}>
-                Fullscreen <i className="fas fa-expand" />
-              </a>
-            </li>
-            <li>
-              <a target="_blank" href="https://docs.min.io/">
-                Documentation <i className="fas fa-book" />
-              </a>
-            </li>
-            <li>
-              <a target="_blank" href="https://slack.min.io">
-                Ask for help <i className="fas fa-question-circle" />
+              <a target="_blank" href="https://min.io/pricing?ref=ob">
+                Get Support <i className="fas fa-question-circle" />
               </a>
             </li>
             <li>
@@ -118,20 +108,9 @@ export class BrowserDropdown extends React.Component {
                 />
               )}
             </li>
-            <li>
-              <a href="" onClick={this.showChangePassword.bind(this)}>
-                Change Password <i className="fas fa-cog" />
-              </a>
-              {this.state.showChangePasswordModal && (
-                <ChangePasswordModal
-                  serverInfo={serverInfo}
-                  hideChangePassword={this.hideChangePassword.bind(this)}
-                />
-              )}
-            </li>
             <li>
               <a href="" id="logout" onClick={this.logout}>
-                Sign Out <i className="fas fa-sign-out-alt" />
+                Logout <i className="fas fa-sign-out-alt" />
               </a>
             </li>
           </Dropdown.Menu>

browser/app/js/browser/Header.js

@@ -15,6 +15,7 @@
  */
 
 import React from "react"
+import ObjectsSearch from "../objects/ObjectsSearch"
 import Path from "../objects/Path"
 import StorageInfo from "./StorageInfo"
 import BrowserDropdown from "./BrowserDropdown"
@@ -27,6 +28,7 @@ export const Header = () => {
     <header className="fe-header">
       <Path />
       {loggedIn && <StorageInfo />}
+      {loggedIn && <ObjectsSearch />}
       <ul className="feh-actions">
         {loggedIn ? (
           <BrowserDropdown />

browser/app/js/buckets/selectors.js

@@ -22,7 +22,8 @@ const bucketsFilterSelector = state => state.buckets.filter
 export const getFilteredBuckets = createSelector(
   bucketsSelector,
   bucketsFilterSelector,
-  (buckets, filter) => buckets.filter(bucket => bucket.indexOf(filter) > -1)
+  (buckets, filter) => buckets.filter(
+    bucket => bucket.toLowerCase().indexOf(filter.toLowerCase()) > -1)
 )
 
 export const getCurrentBucket = state => state.buckets.currentBucket

browser/app/js/objects/ObjectsListContainer.js

@@ -18,6 +18,7 @@ import React from "react"
 import { connect } from "react-redux"
 import InfiniteScroll from "react-infinite-scroller"
 import ObjectsList from "./ObjectsList"
+import { getFilteredObjects } from "./selectors"
 
 export class ObjectsListContainer extends React.Component {
   constructor(props) {
@@ -39,22 +40,29 @@ export class ObjectsListContainer extends React.Component {
       })
     }
   }
+  componentDidUpdate(prevProps) {
+    if (this.props.filter !== prevProps.filter) {
+      this.setState({
+        page: 1
+      })
+    }
+  }
   loadNextPage() {
     this.setState(state => {
       return { page: state.page + 1 }
     })
   }
   render() {
-    const { objects, listLoading } = this.props
+    const { filteredObjects, listLoading } = this.props
 
-    const visibleObjects = objects.slice(0, this.state.page * 100)
+    const visibleObjects = filteredObjects.slice(0, this.state.page * 100)
 
     return (
       <div style={{ position: "relative" }}>
         <InfiniteScroll
           pageStart={0}
           loadMore={this.loadNextPage}
-          hasMore={objects.length > visibleObjects.length}
+          hasMore={filteredObjects.length > visibleObjects.length}
           useWindow={true}
           initialLoad={false}
         >
@@ -70,7 +78,8 @@ const mapStateToProps = state => {
   return {
     currentBucket: state.buckets.currentBucket,
     currentPrefix: state.objects.currentPrefix,
-    objects: state.objects.list,
+    filteredObjects: getFilteredObjects(state),
+    filter: state.objects.filter,
     sortBy: state.objects.sortBy,
     sortOrder: state.objects.sortOrder,
     listLoading: state.objects.listLoading

browser/app/js/objects/ObjectsSearch.js

@@ -0,0 +1,43 @@
+/*
+ * MinIO Cloud Storage (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from "react"
+import { connect } from "react-redux"
+import * as actionsObjects from "./actions"
+
+export const ObjectsSearch = ({ onChange }) => (
+  <div
+    className="input-group ig-left ig-search-dark"
+    style={{ display: "block" }}
+  >
+    <input
+      className="ig-text"
+      type="input"
+      placeholder="Search Objects..."
+      onChange={e => onChange(e.target.value)}
+    />
+    <i className="ig-helpers" />
+  </div>
+)
+
+const mapDispatchToProps = dispatch => {
+  return {
+    onChange: filter =>
+      dispatch(actionsObjects.setFilter(filter))
+  }
+}
+
+export default connect(undefined, mapDispatchToProps)(ObjectsSearch)

browser/app/js/objects/__tests__/ObjectsListContainer.test.js

@@ -20,13 +20,13 @@ import { ObjectsListContainer } from "../ObjectsListContainer"
 
 describe("ObjectsList", () => {
   it("should render without crashing", () => {
-    shallow(<ObjectsListContainer objects={[]} />)
+    shallow(<ObjectsListContainer filteredObjects={[]} />)
   })
 
   it("should render ObjectsList with objects", () => {
     const wrapper = shallow(
       <ObjectsListContainer
-        objects={[{ name: "test1.jpg" }, { name: "test2.jpg" }]}
+        filteredObjects={[{ name: "test1.jpg" }, { name: "test2.jpg" }]}
       />
     )
     expect(wrapper.find("ObjectsList").length).toBe(1)
@@ -40,7 +40,7 @@ describe("ObjectsList", () => {
     const wrapper = shallow(
       <ObjectsListContainer
         currentBucket="test1"
-        objects={[]}
+        filteredObjects={[]}
         listLoading={true}
       />
     )

browser/app/js/objects/__tests__/ObjectsSearch.test.js

@@ -0,0 +1,32 @@
+/*
+ * MinIO Cloud Storage (C) 2018 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from "react"
+import { shallow } from "enzyme"
+import { ObjectsSearch } from "../ObjectsSearch"
+
+describe("ObjectsSearch", () => {
+  it("should render without crashing", () => {
+    shallow(<ObjectsSearch />)
+  })
+
+  it("should call onChange with search text", () => {
+    const onChange = jest.fn()
+    const wrapper = shallow(<ObjectsSearch onChange={onChange} />)
+    wrapper.find("input").simulate("change", { target: { value: "test" } })
+    expect(onChange).toHaveBeenCalledWith("test")
+  })
+})

browser/app/js/objects/__tests__/reducer.test.js

@@ -23,6 +23,7 @@ describe("objects reducer", () => {
     const initialState = reducer(undefined, {})
     expect(initialState).toEqual({
       list: [],
+      filter: "",
       listLoading: false,
       sortBy: "",
       sortOrder: SORT_ORDER_ASC,

browser/app/js/objects/actions.js

@@ -36,6 +36,7 @@ import { getServerInfo, hasServerPublicDomain } from '../browser/selectors'
 
 export const SET_LIST = "objects/SET_LIST"
 export const RESET_LIST = "objects/RESET_LIST"
+export const SET_FILTER = "objects/SET_FILTER"
 export const APPEND_LIST = "objects/APPEND_LIST"
 export const REMOVE = "objects/REMOVE"
 export const SET_SORT_BY = "objects/SET_SORT_BY"
@@ -57,6 +58,13 @@ export const resetList = () => ({
   type: RESET_LIST,
 })
 
+export const setFilter = filter => {
+  return {
+    type: SET_FILTER,
+    filter
+  }
+}
+
 export const setListLoading = (listLoading) => ({
   type: SET_LIST_LOADING,
   listLoading,

browser/app/js/objects/reducer.js

@@ -28,6 +28,7 @@ const removeObject = (list, objectToRemove, lookup) => {
 export default (
   state = {
     list: [],
+    filter: "",
     listLoading: false,
     sortBy: "",
     sortOrder: SORT_ORDER_ASC,
@@ -53,6 +54,11 @@ export default (
         ...state,
         list: []
       }
+    case actionsObjects.SET_FILTER:
+      return {
+        ...state,
+        filter: action.filter
+      }
     case actionsObjects.SET_LIST_LOADING:
       return {
         ...state,

browser/app/js/objects/selectors.js

@@ -21,3 +21,13 @@ export const getCurrentPrefix = state => state.objects.currentPrefix
 export const getCheckedList = state => state.objects.checkedList
 
 export const getPrefixWritable = state => state.objects.prefixWritable
+
+const objectsSelector = state => state.objects.list
+const objectsFilterSelector = state => state.objects.filter
+
+export const getFilteredObjects = createSelector(
+  objectsSelector,
+  objectsFilterSelector,
+  (objects, filter) => objects.filter(
+    object => object.name.toLowerCase().startsWith(filter.toLowerCase()))
+)

browser/app/js/uploads/Dropzone.js

@@ -36,7 +36,7 @@ export class Dropzone extends React.Component {
     // Overwrite the default styling from react-dropzone; otherwise it
     // won't handle child elements correctly.
     const style = {
-      height: "100%",
+      flex: "1",
       borderWidth: "0",
       borderStyle: "dashed",
       borderColor: "#fff"

browser/app/less/inc/file-explorer.less

@@ -20,7 +20,8 @@
 	@media(max-width: @screen-sm-max) {
 		padding: 75px 0 80px;
 	}
-
+  display: flex;
+  flex-direction: column;
 	min-height:100vh;
 	overflow: auto;
 }

browser/app/less/inc/form.less

@@ -169,6 +169,24 @@ select.form-control {
     }
 }
 
+.ig-search-dark {
+    &:before {
+        font-family: @font-family-icon;
+        font-weight: 900;
+        content: '\f002';
+        font-size: 15px;
+        position: absolute;
+        left: 2px;
+        top: 8px;
+        color: rgba(0, 0, 0, 0.5);
+    }
+
+    .ig-text {
+        padding-left: 25px;
+        .placeholder(rgba(0, 0, 0, 0.5))
+    }
+}
+
 .ig-search {
     &:before {
         font-family: @font-family-icon;
@@ -270,4 +288,4 @@ select.form-control {
 .set-expire-decrease {
     bottom: -27px;
     .rotate(-180deg);
-}
+}

buildscripts/verify-build.sh

@@ -76,8 +76,8 @@ function start_minio_zone_erasure_sets_ipv6()
     export MINIO_ACCESS_KEY=$ACCESS_KEY
     export MINIO_SECRET_KEY=$SECRET_KEY
 
-    "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9000.log" 2>&1 &
-    "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-9001.log" 2>&1 &
+    "${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-ipv6-9000.log" 2>&1 &
+    "${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/zone-disk-sets{1...4}" "http://[::1]:9001${WORK_DIR}/zone-disk-sets{5...8}" >"$WORK_DIR/zone-minio-ipv6-9001.log" 2>&1 &
 
     sleep 40
 }

cmd/admin-bucket-handlers.go

@@ -226,6 +226,11 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 		return
 	}
 	if bucket != "" {
+		// Check if bucket exists.
+		if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
 		if _, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket); err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return

cmd/admin-handlers-config-kv.go

@@ -42,7 +42,7 @@ import (
 
 func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *http.Request) (auth.Credentials, ObjectLayer) {
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return auth.Credentials{}, nil

cmd/admin-handlers-users.go

@@ -35,7 +35,7 @@ func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.R
 	var adminAPIErr APIErrorCode
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return nil, cred
@@ -386,7 +386,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	defer logger.AuditLog(w, r, "AddServiceAccount", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
@@ -465,7 +465,7 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 	defer logger.AuditLog(w, r, "ListServiceAccounts", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
@@ -520,7 +520,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 	defer logger.AuditLog(w, r, "DeleteServiceAccount", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
@@ -579,7 +579,7 @@ func (a adminAPIHandlers) AccountUsageInfoHandler(w http.ResponseWriter, r *http
 	defer logger.AuditLog(w, r, "AccountUsageInfo", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
@@ -722,7 +722,10 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	json.NewEncoder(w).Encode(policy)
+	if err = json.NewEncoder(w).Encode(policy); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 	w.(http.Flusher).Flush()
 }
 

cmd/admin-handlers.go

@@ -37,7 +37,6 @@ import (
 	"github.com/gorilla/mux"
 
 	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/config/notify"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -54,16 +53,13 @@ const (
 	maxEConfigJSONSize = 262272
 )
 
-// Type-safe query params.
-type mgmtQueryKey string
-
 // Only valid query params for mgmt admin APIs.
 const (
-	mgmtBucket      mgmtQueryKey = "bucket"
-	mgmtPrefix                   = "prefix"
-	mgmtClientToken              = "clientToken"
-	mgmtForceStart               = "forceStart"
-	mgmtForceStop                = "forceStop"
+	mgmtBucket      = "bucket"
+	mgmtPrefix      = "prefix"
+	mgmtClientToken = "clientToken"
+	mgmtForceStart  = "forceStart"
+	mgmtForceStop   = "forceStop"
 )
 
 func updateServer(u *url.URL, sha256Sum []byte, lrTime time.Time, mode string) (us madmin.ServerUpdateStatus, err error) {
@@ -298,6 +294,20 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 	// ignores any errors here.
 	storageInfo, _ := objectAPI.StorageInfo(ctx, false)
 
+	// Collect any disk healing.
+	healing, _ := getAggregatedBackgroundHealState(ctx)
+	healDisks := make(map[string]struct{}, len(healing.HealDisks))
+	for _, disk := range healing.HealDisks {
+		healDisks[disk] = struct{}{}
+	}
+
+	// find all disks which belong to each respective endpoints
+	for i, disk := range storageInfo.Disks {
+		if _, ok := healDisks[disk.Endpoint]; ok {
+			storageInfo.Disks[i].Healing = true
+		}
+	}
+
 	// Marshal API response
 	jsonBytes, err := json.Marshal(storageInfo)
 	if err != nil {
@@ -339,23 +349,26 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
 	writeSuccessResponseJSON(w, dataUsageInfoJSON)
 }
 
-func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
+func lriToLockEntry(l lockRequesterInfo, resource, server string, rquorum, wquorum int) *madmin.LockEntry {
 	entry := &madmin.LockEntry{
 		Timestamp:  l.Timestamp,
 		Resource:   resource,
 		ServerList: []string{server},
 		Source:     l.Source,
+		Owner:      l.Owner,
 		ID:         l.UID,
 	}
 	if l.Writer {
 		entry.Type = "WRITE"
+		entry.Quorum = wquorum
 	} else {
 		entry.Type = "READ"
+		entry.Quorum = rquorum
 	}
 	return entry
 }
 
-func topLockEntries(peerLocks []*PeerLocks, count int) madmin.LockEntries {
+func topLockEntries(peerLocks []*PeerLocks, rquorum, wquorum int, stale bool) madmin.LockEntries {
 	entryMap := make(map[string]*madmin.LockEntry)
 	for _, peerLock := range peerLocks {
 		if peerLock == nil {
@@ -367,20 +380,23 @@ func topLockEntries(peerLocks []*PeerLocks, count int) madmin.LockEntries {
 					if val, ok := entryMap[lockReqInfo.UID]; ok {
 						val.ServerList = append(val.ServerList, peerLock.Addr)
 					} else {
-						entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
+						entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr, rquorum, wquorum)
 					}
 				}
 			}
 		}
 	}
-	var lockEntries = make(madmin.LockEntries, 0, len(entryMap))
+	var lockEntries madmin.LockEntries
 	for _, v := range entryMap {
-		lockEntries = append(lockEntries, *v)
+		if stale {
+			lockEntries = append(lockEntries, *v)
+			continue
+		}
+		if len(v.ServerList) >= v.Quorum {
+			lockEntries = append(lockEntries, *v)
+		}
 	}
 	sort.Sort(lockEntries)
-	if len(lockEntries) > count {
-		lockEntries = lockEntries[:count]
-	}
 	return lockEntries
 }
 
@@ -410,23 +426,20 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
 			return
 		}
 	}
+	stale := r.URL.Query().Get("stale") == "true" // list also stale locks
 
-	peerLocks := globalNotificationSys.GetLocks(ctx)
-	// Once we have received all the locks currently used from peers
-	// add the local peer locks list as well.
-	var getRespLocks GetLocksResp
-	for _, llocker := range globalLockServers {
-		getRespLocks = append(getRespLocks, llocker.DupLockMap())
+	peerLocks := globalNotificationSys.GetLocks(ctx, r)
+
+	rquorum := getReadQuorum(objectAPI.SetDriveCount())
+	wquorum := getWriteQuorum(objectAPI.SetDriveCount())
+
+	topLocks := topLockEntries(peerLocks, rquorum, wquorum, stale)
+
+	// Marshal API response upto requested count.
+	if len(topLocks) > count && count > 0 {
+		topLocks = topLocks[:count]
 	}
 
-	peerLocks = append(peerLocks, &PeerLocks{
-		Addr:  getHostName(r),
-		Locks: getRespLocks,
-	})
-
-	topLocks := topLockEntries(peerLocks, count)
-
-	// Marshal API response
 	jsonBytes, err := json.Marshal(topLocks)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -572,8 +585,8 @@ type healInitParams struct {
 
 // extractHealInitParams - Validates params for heal init API.
 func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reader) (hip healInitParams, err APIErrorCode) {
-	hip.bucket = vars[string(mgmtBucket)]
-	hip.objPrefix = vars[string(mgmtPrefix)]
+	hip.bucket = vars[mgmtBucket]
+	hip.objPrefix = vars[mgmtPrefix]
 
 	if hip.bucket == "" {
 		if hip.objPrefix != "" {
@@ -592,13 +605,13 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 		return
 	}
 
-	if len(qParms[string(mgmtClientToken)]) > 0 {
-		hip.clientToken = qParms[string(mgmtClientToken)][0]
+	if len(qParms[mgmtClientToken]) > 0 {
+		hip.clientToken = qParms[mgmtClientToken][0]
 	}
-	if _, ok := qParms[string(mgmtForceStart)]; ok {
+	if _, ok := qParms[mgmtForceStart]; ok {
 		hip.forceStart = true
 	}
-	if _, ok := qParms[string(mgmtForceStop)]; ok {
+	if _, ok := qParms[mgmtForceStop]; ok {
 		hip.forceStop = true
 	}
 
@@ -886,7 +899,7 @@ func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Reques
 	var cred auth.Credentials
 	var adminAPIErr APIErrorCode
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return nil, cred
@@ -1425,12 +1438,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	cfg, err := readServerConfig(ctx, objectAPI)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
 	buckets := madmin.Buckets{}
 	objects := madmin.Objects{}
 	usage := madmin.Usage{}
@@ -1442,7 +1449,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
 	}
 
-	vault := fetchVaultStatus(cfg)
+	vault := fetchVaultStatus()
 
 	ldap := madmin.LDAP{}
 	if globalLDAPConfig.Enabled {
@@ -1458,10 +1465,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
-	log, audit := fetchLoggerInfo(cfg)
+	log, audit := fetchLoggerInfo()
 
 	// Get the notification target info
-	notifyTarget := fetchLambdaInfo(cfg)
+	notifyTarget := fetchLambdaInfo()
 
 	// Fetching the Storage information, ignore any errors.
 	storageInfo, _ := objectAPI.StorageInfo(ctx, false)
@@ -1483,11 +1490,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
-	mode := "safemode"
-	if newObjectLayerFn() != nil {
-		mode = "online"
-	}
-
+	mode := "online"
 	server := getLocalServerProperty(globalEndpoints, r)
 	servers := globalNotificationSys.ServerInfo()
 	servers = append(servers, server)
@@ -1501,20 +1504,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		Notifications: notifyTarget,
 	}
 
-	// Collect any disk healing.
-	healing, _ := getAggregatedBackgroundHealState(ctx)
-	healDisks := make(map[string]struct{}, len(healing.HealDisks))
-	for _, disk := range healing.HealDisks {
-		healDisks[disk] = struct{}{}
-	}
-
 	// find all disks which belong to each respective endpoints
 	for i := range servers {
 		for _, disk := range storageInfo.Disks {
 			if strings.Contains(disk.Endpoint, servers[i].Endpoint) {
-				if _, ok := healDisks[disk.Endpoint]; ok {
-					disk.Healing = true
-				}
 				servers[i].Disks = append(servers[i].Disks, disk)
 			}
 		}
@@ -1526,9 +1519,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 			continue
 		}
 		if disk.Endpoint == disk.DrivePath {
-			if _, ok := healDisks[disk.Endpoint]; ok {
-				disk.Healing = true
-			}
 			servers[len(servers)-1].Disks = append(servers[len(servers)-1].Disks, disk)
 		}
 	}
@@ -1554,27 +1544,33 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	//Reply with storage information (across nodes in a
+	// Reply with storage information (across nodes in a
 	// distributed setup) as json.
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
-func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
-
-	// Fetch the configured targets
-	tr := NewGatewayHTTPTransport()
-	defer tr.CloseIdleConnections()
-	targetList, err := notify.FetchRegisteredTargets(GlobalContext, cfg, tr, true, false)
-	if err != nil && err != notify.ErrTargetsOffline {
-		logger.LogIf(GlobalContext, err)
-		return nil
-	}
+func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
 
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
 
-	for targetID, target := range targetList.TargetMap() {
+	for _, tgt := range globalConfigTargetList.Targets() {
 		targetIDStatus := make(map[string]madmin.Status)
-		active, _ := target.IsActive()
+		active, _ := tgt.IsActive()
+		targetID := tgt.ID()
+		if active {
+			targetIDStatus[targetID.ID] = madmin.Status{Status: "Online"}
+		} else {
+			targetIDStatus[targetID.ID] = madmin.Status{Status: "Offline"}
+		}
+		list := lambdaMap[targetID.Name]
+		list = append(list, targetIDStatus)
+		lambdaMap[targetID.Name] = list
+	}
+
+	for _, tgt := range globalEnvTargetList.Targets() {
+		targetIDStatus := make(map[string]madmin.Status)
+		active, _ := tgt.IsActive()
+		targetID := tgt.ID()
 		if active {
 			targetIDStatus[targetID.ID] = madmin.Status{Status: "Online"}
 		} else {
@@ -1583,8 +1579,6 @@ func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
 		list := lambdaMap[targetID.Name]
 		list = append(list, targetIDStatus)
 		lambdaMap[targetID.Name] = list
-		// Close any leaking connections
-		_ = target.Close()
 	}
 
 	notify := make([]map[string][]madmin.TargetIDStatus, len(lambdaMap))
@@ -1599,7 +1593,7 @@ func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
 }
 
 // fetchVaultStatus fetches Vault Info
-func fetchVaultStatus(cfg config.Config) madmin.Vault {
+func fetchVaultStatus() madmin.Vault {
 	vault := madmin.Vault{}
 	if GlobalKMS == nil {
 		vault.Status = "disabled"
@@ -1642,41 +1636,42 @@ func fetchVaultStatus(cfg config.Config) madmin.Vault {
 }
 
 // fetchLoggerDetails return log info
-func fetchLoggerInfo(cfg config.Config) ([]madmin.Logger, []madmin.Audit) {
-	loggerCfg, _ := logger.LookupConfig(cfg)
-
-	var logger []madmin.Logger
-	var auditlogger []madmin.Audit
-	for log, l := range loggerCfg.HTTP {
-		if l.Enabled {
-			err := checkConnection(l.Endpoint, 15*time.Second)
+func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
+	var loggerInfo []madmin.Logger
+	var auditloggerInfo []madmin.Audit
+	for _, target := range logger.Targets {
+		if target.Endpoint() != "" {
+			tgt := target.String()
+			err := checkConnection(target.Endpoint(), 15*time.Second)
 			if err == nil {
 				mapLog := make(map[string]madmin.Status)
-				mapLog[log] = madmin.Status{Status: "Online"}
-				logger = append(logger, mapLog)
+				mapLog[tgt] = madmin.Status{Status: "Online"}
+				loggerInfo = append(loggerInfo, mapLog)
 			} else {
 				mapLog := make(map[string]madmin.Status)
-				mapLog[log] = madmin.Status{Status: "offline"}
-				logger = append(logger, mapLog)
+				mapLog[tgt] = madmin.Status{Status: "offline"}
+				loggerInfo = append(loggerInfo, mapLog)
 			}
 		}
 	}
 
-	for audit, l := range loggerCfg.Audit {
-		if l.Enabled {
-			err := checkConnection(l.Endpoint, 15*time.Second)
+	for _, target := range logger.AuditTargets {
+		if target.Endpoint() != "" {
+			tgt := target.String()
+			err := checkConnection(target.Endpoint(), 15*time.Second)
 			if err == nil {
 				mapAudit := make(map[string]madmin.Status)
-				mapAudit[audit] = madmin.Status{Status: "Online"}
-				auditlogger = append(auditlogger, mapAudit)
+				mapAudit[tgt] = madmin.Status{Status: "Online"}
+				auditloggerInfo = append(auditloggerInfo, mapAudit)
 			} else {
 				mapAudit := make(map[string]madmin.Status)
-				mapAudit[audit] = madmin.Status{Status: "Offline"}
-				auditlogger = append(auditlogger, mapAudit)
+				mapAudit[tgt] = madmin.Status{Status: "Offline"}
+				auditloggerInfo = append(auditloggerInfo, mapAudit)
 			}
 		}
 	}
-	return logger, auditlogger
+
+	return loggerInfo, auditloggerInfo
 }
 
 // checkConnection - ping an endpoint , return err in case of no connection

cmd/admin-handlers_test.go

@@ -327,13 +327,13 @@ func TestExtractHealInitParams(t *testing.T) {
 	mkParams := func(clientToken string, forceStart, forceStop bool) url.Values {
 		v := url.Values{}
 		if clientToken != "" {
-			v.Add(string(mgmtClientToken), clientToken)
+			v.Add(mgmtClientToken, clientToken)
 		}
 		if forceStart {
-			v.Add(string(mgmtForceStart), "")
+			v.Add(mgmtForceStart, "")
 		}
 		if forceStop {
-			v.Add(string(mgmtForceStop), "")
+			v.Add(mgmtForceStop, "")
 		}
 		return v
 	}
@@ -351,11 +351,11 @@ func TestExtractHealInitParams(t *testing.T) {
 	}
 	varsArr := []map[string]string{
 		// Invalid cases
-		{string(mgmtPrefix): "objprefix"},
+		{mgmtPrefix: "objprefix"},
 		// Valid cases
 		{},
-		{string(mgmtBucket): "bucket"},
-		{string(mgmtBucket): "bucket", string(mgmtPrefix): "objprefix"},
+		{mgmtBucket: "bucket"},
+		{mgmtBucket: "bucket", mgmtPrefix: "objprefix"},
 	}
 
 	// Body is always valid - we do not test JSON decoding.

cmd/admin-heal-ops.go

@@ -85,7 +85,7 @@ type healSequenceStatus struct {
 
 // structure to hold state of all heal sequences in server memory
 type allHealState struct {
-	sync.Mutex
+	sync.RWMutex
 
 	// map of heal path to heal sequence
 	healSeqMap     map[string]*healSequence
@@ -105,21 +105,21 @@ func newHealState() *allHealState {
 }
 
 func (ahs *allHealState) healDriveCount() int {
-	ahs.Lock()
-	defer ahs.Unlock()
+	ahs.RLock()
+	defer ahs.RUnlock()
 
 	return len(ahs.healLocalDisks)
 }
 
 func (ahs *allHealState) getHealLocalDisks() Endpoints {
-	ahs.Lock()
-	defer ahs.Unlock()
+	ahs.RLock()
+	defer ahs.RUnlock()
 
-	var healLocalDisks Endpoints
+	var endpoints Endpoints
 	for ep := range ahs.healLocalDisks {
-		healLocalDisks = append(healLocalDisks, ep)
+		endpoints = append(endpoints, ep)
 	}
-	return healLocalDisks
+	return endpoints
 }
 
 func (ahs *allHealState) popHealLocalDisks(healLocalDisks ...Endpoint) {
@@ -528,7 +528,7 @@ func (h *healSequence) hasEnded() bool {
 	if h.clientToken == bgHealingUUID {
 		return false
 	}
-	return len(h.currentStatus.Items) == 0 || h.currentStatus.Summary == healStoppedStatus || h.currentStatus.Summary == healFinishedStatus
+	return !h.endTime.IsZero()
 }
 
 // stops the heal sequence - safe to call multiple times.
@@ -808,7 +808,7 @@ func (h *healSequence) traverseAndHeal() {
 func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
 	return func() error {
 		// Get current object layer instance.
-		objectAPI := newObjectLayerWithoutSafeModeFn()
+		objectAPI := newObjectLayerFn()
 		if objectAPI == nil {
 			return errServerNotInitialized
 		}
@@ -844,7 +844,7 @@ func (h *healSequence) healDiskFormat() error {
 	}
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -864,7 +864,7 @@ func (h *healSequence) healBuckets(bucketsOnly bool) error {
 	}
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -886,7 +886,7 @@ func (h *healSequence) healBuckets(bucketsOnly bool) error {
 // healBucket - traverses and heals given bucket
 func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -932,7 +932,7 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 // healObject - heal the given object and record result
 func (h *healSequence) healObject(bucket, object, versionID string) error {
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}

cmd/admin-router.go

@@ -180,19 +180,19 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 				// PutBucketQuotaConfig
 				adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
 					httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
-			}
-			// Bucket replication operations
-			// GetBucketTargetHandler
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
-				httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
-			// SetRemoteTargetHandler
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
-				httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
-			// SetRemoteTargetHandler
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
-				httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
-		}
 
+				// Bucket replication operations
+				// GetBucketTargetHandler
+				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
+					httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
+				// SetRemoteTargetHandler
+				adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
+					httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
+				// SetRemoteTargetHandler
+				adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
+					httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
+			}
+		}
 		// -- Top APIs --
 		// Top locks
 		if globalIsDistErasure {
@@ -218,6 +218,6 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 	}
 
 	// If none of the routes match add default error handler routes
-	adminRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(errorResponseHandler))
-	adminRouter.MethodNotAllowedHandler = http.HandlerFunc(httpTraceAll(errorResponseHandler))
+	adminRouter.NotFoundHandler = httpTraceAll(errorResponseHandler)
+	adminRouter.MethodNotAllowedHandler = httpTraceAll(errorResponseHandler)
 }

cmd/api-errors.go

@@ -106,16 +106,17 @@ const (
 	ErrNoSuchCORSConfiguration
 	ErrNoSuchWebsiteConfiguration
 	ErrReplicationConfigurationNotFoundError
-	ErrReplicationDestinationNotFoundError
+	ErrRemoteDestinationNotFoundError
 	ErrReplicationDestinationMissingLock
-	ErrReplicationTargetNotFoundError
+	ErrRemoteTargetNotFoundError
 	ErrReplicationRemoteConnectionError
 	ErrBucketRemoteIdenticalToSource
 	ErrBucketRemoteAlreadyExists
+	ErrBucketRemoteLabelInUse
 	ErrBucketRemoteArnTypeInvalid
 	ErrBucketRemoteArnInvalid
 	ErrBucketRemoteRemoveDisallowed
-	ErrReplicationTargetNotVersionedError
+	ErrRemoteTargetNotVersionedError
 	ErrReplicationSourceNotVersionedError
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
@@ -809,9 +810,9 @@ var errorCodes = errorCodeMap{
 		Description:    "The replication configuration was not found",
 		HTTPStatusCode: http.StatusNotFound,
 	},
-	ErrReplicationDestinationNotFoundError: {
-		Code:           "ReplicationDestinationNotFoundError",
-		Description:    "The replication destination bucket does not exist",
+	ErrRemoteDestinationNotFoundError: {
+		Code:           "RemoteDestinationNotFoundError",
+		Description:    "The remote destination bucket does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrReplicationDestinationMissingLock: {
@@ -819,29 +820,34 @@ var errorCodes = errorCodeMap{
 		Description:    "The replication destination bucket does not have object locking enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrReplicationTargetNotFoundError: {
-		Code:           "XminioAdminReplicationTargetNotFoundError",
-		Description:    "The replication target does not exist",
+	ErrRemoteTargetNotFoundError: {
+		Code:           "XMinioAdminRemoteTargetNotFoundError",
+		Description:    "The remote target does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrReplicationRemoteConnectionError: {
-		Code:           "XminioAdminReplicationRemoteConnectionError",
+		Code:           "XMinioAdminReplicationRemoteConnectionError",
 		Description:    "Remote service endpoint or target bucket not available",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrBucketRemoteIdenticalToSource: {
-		Code:           "XminioAdminRemoteIdenticalToSource",
+		Code:           "XMinioAdminRemoteIdenticalToSource",
 		Description:    "The remote target cannot be identical to source",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrBucketRemoteAlreadyExists: {
-		Code:           "XminioAdminBucketRemoteAlreadyExists",
+		Code:           "XMinioAdminBucketRemoteAlreadyExists",
 		Description:    "The remote target already exists",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrBucketRemoteLabelInUse: {
+		Code:           "XMinioAdminBucketRemoteLabelInUse",
+		Description:    "The remote target with this label already exists",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrBucketRemoteRemoveDisallowed: {
 		Code:           "XMinioAdminRemoteRemoveDisallowed",
-		Description:    "Replication configuration exists with this ARN.",
+		Description:    "This ARN is in use by an existing configuration",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrBucketRemoteArnTypeInvalid: {
@@ -854,9 +860,9 @@ var errorCodes = errorCodeMap{
 		Description:    "The bucket remote ARN does not have correct format",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrReplicationTargetNotVersionedError: {
-		Code:           "ReplicationTargetNotVersionedError",
-		Description:    "The replication target does not have versioning enabled",
+	ErrRemoteTargetNotVersionedError: {
+		Code:           "RemoteTargetNotVersionedError",
+		Description:    "The remote target does not have versioning enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrReplicationSourceNotVersionedError: {
@@ -1906,24 +1912,26 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminNoSuchQuotaConfiguration
 	case BucketReplicationConfigNotFound:
 		apiErr = ErrReplicationConfigurationNotFoundError
-	case BucketReplicationDestinationNotFound:
-		apiErr = ErrReplicationDestinationNotFoundError
+	case BucketRemoteDestinationNotFound:
+		apiErr = ErrRemoteDestinationNotFoundError
 	case BucketReplicationDestinationMissingLock:
 		apiErr = ErrReplicationDestinationMissingLock
 	case BucketRemoteTargetNotFound:
-		apiErr = ErrReplicationTargetNotFoundError
+		apiErr = ErrRemoteTargetNotFoundError
 	case BucketRemoteConnectionErr:
 		apiErr = ErrReplicationRemoteConnectionError
 	case BucketRemoteAlreadyExists:
 		apiErr = ErrBucketRemoteAlreadyExists
+	case BucketRemoteLabelInUse:
+		apiErr = ErrBucketRemoteLabelInUse
 	case BucketRemoteArnTypeInvalid:
 		apiErr = ErrBucketRemoteArnTypeInvalid
 	case BucketRemoteArnInvalid:
 		apiErr = ErrBucketRemoteArnInvalid
 	case BucketRemoteRemoveDisallowed:
 		apiErr = ErrBucketRemoteRemoveDisallowed
-	case BucketReplicationTargetNotVersioned:
-		apiErr = ErrReplicationTargetNotVersionedError
+	case BucketRemoteTargetNotVersioned:
+		apiErr = ErrRemoteTargetNotVersionedError
 	case BucketReplicationSourceNotVersioned:
 		apiErr = ErrReplicationSourceNotVersionedError
 	case BucketQuotaExceeded:
@@ -1956,6 +1964,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrBackendDown
 	case ObjectNameTooLong:
 		apiErr = ErrKeyTooLongError
+	case dns.ErrInvalidBucketName:
+		apiErr = ErrInvalidBucketName
 	default:
 		var ie, iw int
 		// This work-around is to handle the issue golang/go#30648
@@ -1992,6 +2002,12 @@ func toAPIError(ctx context.Context, err error) APIError {
 	}
 
 	var apiErr = errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
+	e, ok := err.(dns.ErrInvalidBucketName)
+	if ok {
+		code := toAPIErrorCode(ctx, e)
+		apiErr = errorCodes.ToAPIErrWithErr(code, e)
+	}
+
 	if apiErr.Code == "InternalError" {
 		// If we see an internal error try to interpret
 		// any underlying errors if possible depending on

cmd/api-headers.go

@@ -84,7 +84,7 @@ func setPartsCountHeaders(w http.ResponseWriter, objInfo ObjectInfo) {
 }
 
 // Write object header
-func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSpec) (err error) {
+func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSpec, opts ObjectOptions) (err error) {
 	// set common headers
 	setCommonHeaders(w)
 
@@ -147,15 +147,26 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		}
 	}
 
+	var start, rangeLen int64
 	totalObjectSize, err := objInfo.GetActualSize()
 	if err != nil {
 		return err
 	}
 
-	// for providing ranged content
-	start, rangeLen, err := rs.GetOffsetLength(totalObjectSize)
-	if err != nil {
-		return err
+	if opts.PartNumber > 0 {
+		var start, end int64
+		for i := 0; i < len(objInfo.Parts) && i < opts.PartNumber; i++ {
+			start = end
+			end = start + objInfo.Parts[i].ActualSize - 1
+		}
+		rs = &HTTPRangeSpec{Start: start, End: end}
+		rangeLen = end - start + 1
+	} else {
+		// for providing ranged content
+		start, rangeLen, err = rs.GetOffsetLength(totalObjectSize)
+		if err != nil {
+			return err
+		}
 	}
 
 	// Set content length.

cmd/api-response.go

@@ -703,10 +703,6 @@ func generateMultiDeleteResponse(quiet bool, deletedObjects []DeletedObject, err
 }
 
 func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
-	if newObjectLayerFn() == nil {
-		// Server still in safe mode.
-		w.Header().Set(xhttp.MinIOServerStatus, "safemode")
-	}
 	setCommonHeaders(w)
 	if mType != mimeNone {
 		w.Header().Set(xhttp.ContentType, string(mType))
@@ -773,10 +769,6 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 		// The request is from browser and also if browser
 		// is enabled we need to redirect.
 		if browser && globalBrowserEnabled {
-			if newObjectLayerFn() == nil {
-				// server still in safe mode.
-				w.Header().Set(xhttp.MinIOServerStatus, "safemode")
-			}
 			w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
 			w.WriteHeader(http.StatusTemporaryRedirect)
 			return

cmd/api-router.go

@@ -32,28 +32,15 @@ func newHTTPServerFn() *xhttp.Server {
 	return globalHTTPServer
 }
 
-func newObjectLayerWithoutSafeModeFn() ObjectLayer {
-	globalObjLayerMutex.Lock()
-	defer globalObjLayerMutex.Unlock()
-	return globalObjectAPI
-}
-
 func newObjectLayerFn() ObjectLayer {
 	globalObjLayerMutex.Lock()
 	defer globalObjLayerMutex.Unlock()
-	if globalSafeMode {
-		return nil
-	}
 	return globalObjectAPI
 }
 
 func newCachedObjectLayerFn() CacheObjectLayer {
 	globalObjLayerMutex.Lock()
 	defer globalObjLayerMutex.Unlock()
-
-	if globalSafeMode {
-		return nil
-	}
 	return globalCacheObjectAPI
 }
 
@@ -88,7 +75,10 @@ func registerAPIRouter(router *mux.Router) {
 	for _, domainName := range globalDomainNames {
 		if IsKubernetes() {
 			routers = append(routers, apiRouter.MatcherFunc(func(r *http.Request, match *mux.RouteMatch) bool {
-				host, _, _ := net.SplitHostPort(getHost(r))
+				host, _, err := net.SplitHostPort(getHost(r))
+				if err != nil {
+					host = r.Host
+				}
 				// Make sure to skip matching minio.<domain>` this is
 				// specifically meant for operator/k8s deployment
 				// The reason we need to skip this is for a special
@@ -329,8 +319,8 @@ func registerAPIRouter(router *mux.Router) {
 		maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler))))
 
 	// If none of the routes match add default error handler routes
-	apiRouter.NotFoundHandler = http.HandlerFunc(collectAPIStats("notfound", httpTraceAll(errorResponseHandler)))
-	apiRouter.MethodNotAllowedHandler = http.HandlerFunc(collectAPIStats("methodnotallowed", httpTraceAll(errorResponseHandler)))
+	apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler))
+	apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(errorResponseHandler))
 
 }
 

cmd/background-newdisks-heal-ops.go

@@ -26,7 +26,17 @@ import (
 	"github.com/minio/minio/cmd/logger"
 )
 
-const defaultMonitorNewDiskInterval = time.Second * 10
+const (
+	defaultMonitorNewDiskInterval = time.Second * 10
+	healingTrackerFilename        = ".healing.bin"
+)
+
+//go:generate msgp -file $GOFILE -unexported
+type healingTracker struct {
+	ID string
+
+	// future add more tracking capabilities
+}
 
 func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 	z, ok := objAPI.(*erasureZones)
@@ -47,9 +57,7 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 		time.Sleep(time.Second)
 	}
 
-	for _, ep := range getLocalDisksToHeal() {
-		globalBackgroundHealState.pushHealLocalDisks(ep)
-	}
+	globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
 
 	if drivesToHeal := globalBackgroundHealState.healDriveCount(); drivesToHeal > 0 {
 		logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content...",
@@ -76,9 +84,11 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 			}
 			// Try to connect to the current endpoint
 			// and reformat if the current disk is not formatted
-			_, _, err := connectEndpoint(endpoint)
+			disk, _, err := connectEndpoint(endpoint)
 			if errors.Is(err, errUnformattedDisk) {
 				disksToHeal = append(disksToHeal, endpoint)
+			} else if err == nil && disk != nil && disk.Healing() {
+				disksToHeal = append(disksToHeal, disk.Endpoint())
 			}
 		}
 	}
@@ -106,7 +116,8 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS
 		case <-time.After(defaultMonitorNewDiskInterval):
 			waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()), time.Second)
 
-			var erasureSetInZoneEndpointToHeal []map[int]Endpoints
+			var erasureSetInZoneDisksToHeal []map[int][]StorageAPI
+
 			healDisks := globalBackgroundHealState.getHealLocalDisks()
 			if len(healDisks) > 0 {
 				// Reformat disks
@@ -118,22 +129,21 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS
 				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
 					len(healDisks)))
 
-				erasureSetInZoneEndpointToHeal = make([]map[int]Endpoints, len(z.zones))
+				erasureSetInZoneDisksToHeal = make([]map[int][]StorageAPI, len(z.zones))
 				for i := range z.zones {
-					erasureSetInZoneEndpointToHeal[i] = map[int]Endpoints{}
+					erasureSetInZoneDisksToHeal[i] = map[int][]StorageAPI{}
 				}
 			}
 
 			// heal only if new disks found.
 			for _, endpoint := range healDisks {
-				// Load the new format of this passed endpoint
-				_, format, err := connectEndpoint(endpoint)
+				disk, format, err := connectEndpoint(endpoint)
 				if err != nil {
 					printEndpointError(endpoint, err, true)
 					continue
 				}
 
-				zoneIdx := globalEndpoints.GetLocalZoneIdx(endpoint)
+				zoneIdx := globalEndpoints.GetLocalZoneIdx(disk.Endpoint())
 				if zoneIdx < 0 {
 					continue
 				}
@@ -145,21 +155,31 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS
 					continue
 				}
 
-				erasureSetInZoneEndpointToHeal[zoneIdx][setIndex] = append(erasureSetInZoneEndpointToHeal[zoneIdx][setIndex], endpoint)
+				erasureSetInZoneDisksToHeal[zoneIdx][setIndex] = append(erasureSetInZoneDisksToHeal[zoneIdx][setIndex], disk)
 			}
 
-			for i, setMap := range erasureSetInZoneEndpointToHeal {
-				for setIndex, endpoints := range setMap {
-					for _, ep := range endpoints {
-						logger.Info("Healing disk '%s' on %s zone", ep, humanize.Ordinal(i+1))
+			buckets, _ := z.ListBucketsHeal(ctx)
+			for i, setMap := range erasureSetInZoneDisksToHeal {
+				for setIndex, disks := range setMap {
+					for _, disk := range disks {
+						logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
 
-						if err := healErasureSet(ctx, setIndex, z.zones[i].sets[setIndex], z.zones[i].setDriveCount); err != nil {
+						lbDisks := z.zones[i].sets[setIndex].getLoadBalancedNDisks(z.zones[i].listTolerancePerSet)
+						if err := healErasureSet(ctx, setIndex, buckets, lbDisks); err != nil {
+							logger.LogIf(ctx, err)
+							continue
+						}
+
+						logger.Info("Healing disk '%s' on %s zone complete", disk, humanize.Ordinal(i+1))
+
+						if err := disk.DeleteFile(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix),
+							healingTrackerFilename); err != nil {
 							logger.LogIf(ctx, err)
 							continue
 						}
 
 						// Only upon success pop the healed disk.
-						globalBackgroundHealState.popHealLocalDisks(ep)
+						globalBackgroundHealState.popHealLocalDisks(disk.Endpoint())
 					}
 				}
 			}

cmd/background-newdisks-heal-ops_gen.go

@@ -0,0 +1,110 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "ID":
+			z.ID, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "ID")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 1
+	// write "ID"
+	err = en.Append(0x81, 0xa2, 0x49, 0x44)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.ID)
+	if err != nil {
+		err = msgp.WrapError(err, "ID")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 1
+	// string "ID"
+	o = append(o, 0x81, 0xa2, 0x49, 0x44)
+	o = msgp.AppendString(o, z.ID)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "ID":
+			z.ID, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ID")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z healingTracker) Msgsize() (s int) {
+	s = 1 + 3 + msgp.StringPrefixSize + len(z.ID)
+	return
+}

cmd/background-newdisks-heal-ops_gen_test.go

@@ -0,0 +1,123 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalhealingTracker(t *testing.T) {
+	v := healingTracker{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsghealingTracker(b *testing.B) {
+	v := healingTracker{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsghealingTracker(b *testing.B) {
+	v := healingTracker{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalhealingTracker(b *testing.B) {
+	v := healingTracker{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodehealingTracker(t *testing.T) {
+	v := healingTracker{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodehealingTracker Msgsize() is inaccurate")
+	}
+
+	vn := healingTracker{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodehealingTracker(b *testing.B) {
+	v := healingTracker{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodehealingTracker(b *testing.B) {
+	v := healingTracker{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/bitrot-streaming.go

@@ -140,8 +140,8 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	b.h.Write(buf)
 
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		err := &errHashMismatch{fmt.Sprintf("Disk: %s - content hash does not match - expected %s, got %s",
-			b.disk, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))}
+		err := &errHashMismatch{fmt.Sprintf("Disk: %s  -> %s/%s - content hash does not match - expected %s, got %s",
+			b.disk, b.volume, b.filePath, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))}
 		logger.LogIf(GlobalContext, err)
 		return 0, err
 	}

cmd/bitrot-whole.go

@@ -71,12 +71,12 @@ func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error)
 	if b.buf == nil {
 		b.buf = make([]byte, b.tillOffset-offset)
 		if _, err := b.disk.ReadFile(context.TODO(), b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
-			logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
+			logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, err))
 			return 0, err
 		}
 	}
 	if len(b.buf) < len(buf) {
-		logger.LogIf(GlobalContext, errLessData)
+		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, errLessData))
 		return 0, errLessData
 	}
 	n = copy(buf, b.buf)

cmd/bootstrap-peer-server.go

@@ -188,7 +188,7 @@ func verifyServerSystemConfig(ctx context.Context, endpointZones EndpointZones)
 			retries++
 			// after 5 retries start logging that servers are not reachable yet
 			if retries >= 5 {
-				logger.Info(fmt.Sprintf("Waiting for atleast %d servers to be online for bootstrap check", len(clnts)/2))
+				logger.Info(fmt.Sprintf("Waiting for atleast %d remote servers to be online for bootstrap check", len(clnts)/2))
 				logger.Info(fmt.Sprintf("Following servers are currently offline or unreachable %s", offlineEndpoints))
 				retries = 0 // reset to log again after 5 retries.
 			}
@@ -233,7 +233,7 @@ func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
 		}
 	}
 
-	trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
+	trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultTimeout)
 	restClient := rest.NewClient(serverURL, trFn, newAuthToken)
 	restClient.HealthCheckFn = func() bool {
 		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)

cmd/bucket-encryption.go

@@ -34,7 +34,7 @@ func NewBucketSSEConfigSys() *BucketSSEConfigSys {
 // Get - gets bucket encryption config for the given bucket.
 func (sys *BucketSSEConfigSys) Get(bucket string) (*bucketsse.BucketSSEConfig, error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}

cmd/bucket-lifecycle.go

@@ -31,7 +31,7 @@ type LifecycleSys struct{}
 // Get - gets lifecycle config associated to a given bucket name.
 func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}

cmd/bucket-listobjects-handlers.go

@@ -27,6 +27,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 
 	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
@@ -113,7 +114,12 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if proxyRequestByBucket(ctx, w, r, bucket) {
+	// Forward the request using Source IP or bucket
+	forwardStr := handlers.GetSourceIPFromHeaders(r)
+	if forwardStr == "" {
+		forwardStr = bucket
+	}
+	if proxyRequestByStringHash(ctx, w, r, forwardStr) {
 		return
 	}
 
@@ -340,8 +346,8 @@ func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http
 	return proxyRequest(ctx, w, r, ep)
 }
 
-func proxyRequestByBucket(ctx context.Context, w http.ResponseWriter, r *http.Request, bucket string) (success bool) {
-	return proxyRequestByNodeIndex(ctx, w, r, crcHashMod(bucket, len(globalProxyEndpoints)))
+func proxyRequestByStringHash(ctx context.Context, w http.ResponseWriter, r *http.Request, str string) (success bool) {
+	return proxyRequestByNodeIndex(ctx, w, r, crcHashMod(str, len(globalProxyEndpoints)))
 }
 
 // ListObjectsV1Handler - GET Bucket (List Objects) Version 1.
@@ -382,7 +388,12 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	if proxyRequestByBucket(ctx, w, r, bucket) {
+	// Forward the request using Source IP or bucket
+	forwardStr := handlers.GetSourceIPFromHeaders(r)
+	if forwardStr == "" {
+		forwardStr = bucket
+	}
+	if proxyRequestByStringHash(ctx, w, r, forwardStr) {
 		return
 	}
 

cmd/bucket-metadata-sys.go

@@ -72,7 +72,7 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
 // Update update bucket metadata for the specified config file.
 // The configData data should not be modified after being sent here.
 func (sys *BucketMetadataSys) Update(bucket string, configFile string, configData []byte) error {
-	objAPI := newObjectLayerWithoutSafeModeFn()
+	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
@@ -275,7 +275,7 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
 func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
 	if globalIsGateway && globalGatewayName == NASBackendGateway {
 		// Only needed in case of NAS gateway.
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}
@@ -313,7 +313,7 @@ func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEC
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}
@@ -376,7 +376,7 @@ func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.Buc
 // GetConfig returns a specific configuration from the bucket metadata.
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetConfig(bucket string) (BucketMetadata, error) {
-	objAPI := newObjectLayerWithoutSafeModeFn()
+	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return newBucketMetadata(bucket), errServerNotInitialized
 	}

cmd/bucket-metadata.go

@@ -107,6 +107,10 @@ func newBucketMetadata(name string) BucketMetadata {
 // Load - loads the metadata of bucket by name from ObjectLayer api.
 // If an error is returned the returned metadata will be default initialized.
 func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
+	if name == "" {
+		logger.LogIf(ctx, errors.New("bucket name cannot be empty"))
+		return errors.New("bucket name cannot be empty")
+	}
 	configFile := path.Join(bucketConfigPrefix, name, bucketMetadataFile)
 	data, err := readConfig(ctx, api, configFile)
 	if err != nil {
@@ -128,20 +132,22 @@ func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string)
 	}
 	// OK, parse data.
 	_, err = b.UnmarshalMsg(data[4:])
+	b.Name = name // in-case parsing failed for some reason, make sure bucket name is not empty.
 	return err
 }
 
 // loadBucketMetadata loads and migrates to bucket metadata.
 func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
 	b := newBucketMetadata(bucket)
-	err := b.Load(ctx, objectAPI, bucket)
+	err := b.Load(ctx, objectAPI, b.Name)
 	if err == nil {
 		return b, b.convertLegacyConfigs(ctx, objectAPI)
 	}
 
-	if err != errConfigNotFound {
+	if !errors.Is(err, errConfigNotFound) {
 		return b, err
 	}
+
 	// Old bucket without bucket metadata. Hence we migrate existing settings.
 	return b, b.convertLegacyConfigs(ctx, objectAPI)
 }

cmd/bucket-object-lock.go

@@ -33,7 +33,7 @@ type BucketObjectLockSys struct{}
 // Get - Get retention configuration.
 func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention, err error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return r, errServerNotInitialized
 		}

cmd/bucket-quota.go

@@ -63,7 +63,7 @@ func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota,
 }
 
 func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64) error {
-	objAPI := newObjectLayerWithoutSafeModeFn()
+	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
@@ -71,6 +71,8 @@ func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64)
 	sys.bucketStorageCache.Once.Do(func() {
 		sys.bucketStorageCache.TTL = 1 * time.Second
 		sys.bucketStorageCache.Update = func() (interface{}, error) {
+			ctx, done := context.WithTimeout(context.Background(), 5*time.Second)
+			defer done()
 			return loadDataUsageFromBackend(ctx, objAPI)
 		}
 	})

cmd/bucket-replication.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"runtime"
 	"strings"
 	"time"
 
@@ -37,7 +38,7 @@ import (
 // gets replication config associated to a given bucket name.
 func getReplicationConfig(ctx context.Context, bucketName string) (rc *replication.Config, err error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}
@@ -51,12 +52,12 @@ func getReplicationConfig(ctx context.Context, bucketName string) (rc *replicati
 // validateReplicationDestination returns error if replication destination bucket missing or not configured
 // It also returns true if replication destination is same as this server.
 func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config) (bool, error) {
-	clnt := globalBucketTargetSys.GetReplicationTargetClient(ctx, rCfg.RoleArn)
+	clnt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rCfg.RoleArn)
 	if clnt == nil {
 		return false, BucketRemoteTargetNotFound{Bucket: bucket}
 	}
 	if found, _ := clnt.BucketExists(ctx, rCfg.GetDestination().Bucket); !found {
-		return false, BucketReplicationDestinationNotFound{Bucket: rCfg.GetDestination().Bucket}
+		return false, BucketRemoteDestinationNotFound{Bucket: rCfg.GetDestination().Bucket}
 	}
 	if ret, err := globalBucketObjectLockSys.Get(bucket); err == nil {
 		if ret.LockEnabled {
@@ -86,7 +87,7 @@ func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object strin
 
 // mustReplicate returns true if object meets replication criteria.
 func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
-	if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
+	if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, "", r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
 		return false
 	}
 	return mustReplicater(ctx, r, bucket, object, meta, replStatus)
@@ -138,15 +139,17 @@ func putReplicationOpts(dest replication.Destination, objInfo ObjectInfo) (putOp
 		sc = objInfo.StorageClass
 	}
 	putOpts = miniogo.PutObjectOptions{
-		UserMetadata:         meta,
-		UserTags:             tag.ToMap(),
-		ContentType:          objInfo.ContentType,
-		ContentEncoding:      objInfo.ContentEncoding,
-		StorageClass:         sc,
-		ReplicationVersionID: objInfo.VersionID,
-		ReplicationStatus:    miniogo.ReplicationStatusReplica,
-		ReplicationMTime:     objInfo.ModTime,
-		ReplicationETag:      objInfo.ETag,
+		UserMetadata:    meta,
+		UserTags:        tag.ToMap(),
+		ContentType:     objInfo.ContentType,
+		ContentEncoding: objInfo.ContentEncoding,
+		StorageClass:    sc,
+		Internal: miniogo.AdvancedPutOptions{
+			SourceVersionID:   objInfo.VersionID,
+			ReplicationStatus: miniogo.ReplicationStatusReplica,
+			SourceMTime:       objInfo.ModTime,
+			SourceETag:        objInfo.ETag,
+		},
 	}
 	if mode, ok := objInfo.UserDefined[xhttp.AmzObjectLockMode]; ok {
 		rmode := miniogo.RetentionMode(mode)
@@ -179,8 +182,7 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 		logger.LogIf(ctx, err)
 		return
 	}
-
-	tgt := globalBucketTargetSys.GetReplicationTargetClient(ctx, cfg.RoleArn)
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
 	if tgt == nil {
 		return
 	}
@@ -293,29 +295,56 @@ func (r *replicationState) queueReplicaTask(oi ObjectInfo) {
 	}
 }
 
-var globalReplicationState *replicationState
+var (
+	globalReplicationState *replicationState
+	// TODO: currently keeping it conservative
+	// but eventually can be tuned in future,
+	// take only half the CPUs for replication
+	// conservatively.
+	globalReplicationConcurrent = runtime.GOMAXPROCS(0) / 2
+)
 
 func newReplicationState() *replicationState {
-	return &replicationState{
-		// TODO: currently keeping it conservative
-		// but eventually can be tuned in future
-		replicaCh: make(chan ObjectInfo, 100),
+
+	// fix minimum concurrent replication to 1 for single CPU setup
+	if globalReplicationConcurrent == 0 {
+		globalReplicationConcurrent = 1
 	}
+	rs := &replicationState{
+		replicaCh: make(chan ObjectInfo, 10000),
+	}
+	go func() {
+		<-GlobalContext.Done()
+		close(rs.replicaCh)
+	}()
+	return rs
+}
+
+// addWorker creates a new worker to process tasks
+func (r *replicationState) addWorker(ctx context.Context, objectAPI ObjectLayer) {
+	// Add a new worker.
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case oi, ok := <-r.replicaCh:
+				if !ok {
+					return
+				}
+				replicateObject(ctx, oi, objectAPI)
+			}
+		}
+	}()
 }
 
 func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
 	if globalReplicationState == nil {
 		return
 	}
-	go func() {
-		defer close(globalReplicationState.replicaCh)
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case oi := <-globalReplicationState.replicaCh:
-				replicateObject(ctx, oi, objectAPI)
-			}
-		}
-	}()
+
+	// Start with globalReplicationConcurrent.
+	for i := 0; i < globalReplicationConcurrent; i++ {
+		globalReplicationState.addWorker(ctx, objectAPI)
+	}
 }

cmd/bucket-targets.go

@@ -98,10 +98,9 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 			return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
 		}
 		if vcfg.Status != string(versioning.Enabled) {
-			return BucketReplicationTargetNotVersioned{Bucket: tgt.TargetBucket}
+			return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
 		}
 	}
-
 	sys.Lock()
 	defer sys.Unlock()
 
@@ -113,6 +112,9 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 			if t.Arn == tgt.Arn {
 				return BucketRemoteAlreadyExists{Bucket: t.TargetBucket}
 			}
+			if t.Label == tgt.Label {
+				return BucketRemoteLabelInUse{Bucket: t.TargetBucket}
+			}
 			newtgts[idx] = *tgt
 			found = true
 			continue
@@ -173,8 +175,8 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 	return nil
 }
 
-// GetReplicationTargetClient returns minio-go client for replication target instance
-func (sys *BucketTargetSys) GetReplicationTargetClient(ctx context.Context, arn string) *miniogo.Core {
+// GetRemoteTargetClient returns minio-go client for replication target instance
+func (sys *BucketTargetSys) GetRemoteTargetClient(ctx context.Context, arn string) *miniogo.Core {
 	sys.RLock()
 	defer sys.RUnlock()
 	return sys.arnRemotesMap[arn]

cmd/bucket-versioning-handler.go

@@ -70,6 +70,14 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		}, r.URL, guessIsBrowserReq(r))
 		return
 	}
+	if _, err := getReplicationConfig(ctx, bucket); err == nil && v.Suspended() {
+		writeErrorResponse(ctx, w, APIError{
+			Code:           "InvalidBucketState",
+			Description:    "A replication configuration is present on this bucket, so the versioning state cannot be changed.",
+			HTTPStatusCode: http.StatusConflict,
+		}, r.URL, guessIsBrowserReq(r))
+		return
+	}
 
 	configData, err := xml.Marshal(v)
 	if err != nil {

cmd/common-main.go

@@ -334,14 +334,23 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 		return nil, nil, false, err
 	}
 	for _, file := range files {
-		// We exclude any regular file and the "CAs/" directory.
-		// The "CAs/" directory contains (root) CA certificates
-		// that MinIO adds to its list of trusted roots (tls.Config.RootCAs).
-		// Therefore, "CAs/" does not contain X.509 certificates that
-		// are meant to be served by MinIO.
-		if !file.IsDir() || file.Name() == "CAs" {
+		// Ignore all
+		// - regular files
+		// - "CAs" directory
+		// - any directory which starts with ".."
+		if file.Mode().IsRegular() || file.Name() == "CAs" || strings.HasPrefix(file.Name(), "..") {
 			continue
 		}
+		if file.Mode()&os.ModeSymlink == os.ModeSymlink {
+			file, err = os.Stat(filepath.Join(root.Name(), file.Name()))
+			if err != nil {
+				// not accessible ignore
+				continue
+			}
+			if !file.IsDir() {
+				continue
+			}
+		}
 
 		var (
 			certFile = filepath.Join(root.Name(), file.Name(), publicCertFile)
@@ -350,8 +359,8 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 		if !isFile(certFile) || !isFile(keyFile) {
 			continue
 		}
-		if err := manager.AddCertificate(certFile, keyFile); err != nil {
-			err = fmt.Errorf("Failed to load TLS certificate '%s': %v", certFile, err)
+		if err = manager.AddCertificate(certFile, keyFile); err != nil {
+			err = fmt.Errorf("Unable to load TLS certificate '%s,%s': %w", certFile, keyFile, err)
 			logger.LogIf(GlobalContext, err, logger.Minio)
 		}
 	}

cmd/config-current.go

@@ -493,11 +493,13 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize logger: %w", err))
 	}
 
-	for _, l := range loggerCfg.HTTP {
+	for k, l := range loggerCfg.HTTP {
 		if l.Enabled {
 			// Enable http logging
 			if err = logger.AddTarget(
-				http.New(http.WithEndpoint(l.Endpoint),
+				http.New(
+					http.WithTargetName(k),
+					http.WithEndpoint(l.Endpoint),
 					http.WithAuthToken(l.AuthToken),
 					http.WithUserAgent(loggerUserAgent),
 					http.WithLogKind(string(logger.All)),
@@ -509,11 +511,13 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 		}
 	}
 
-	for _, l := range loggerCfg.Audit {
+	for k, l := range loggerCfg.Audit {
 		if l.Enabled {
 			// Enable http audit logging
 			if err = logger.AddAuditTarget(
-				http.New(http.WithEndpoint(l.Endpoint),
+				http.New(
+					http.WithTargetName(k),
+					http.WithEndpoint(l.Endpoint),
 					http.WithAuthToken(l.AuthToken),
 					http.WithUserAgent(loggerUserAgent),
 					http.WithLogKind(string(logger.All)),

cmd/config/api/api.go

@@ -31,17 +31,23 @@ import (
 const (
 	apiRequestsMax             = "requests_max"
 	apiRequestsDeadline        = "requests_deadline"
-	apiReadyDeadline           = "ready_deadline"
+	apiClusterDeadline         = "cluster_deadline"
 	apiCorsAllowOrigin         = "cors_allow_origin"
 	apiRemoteTransportDeadline = "remote_transport_deadline"
 
 	EnvAPIRequestsMax             = "MINIO_API_REQUESTS_MAX"
 	EnvAPIRequestsDeadline        = "MINIO_API_REQUESTS_DEADLINE"
-	EnvAPIReadyDeadline           = "MINIO_API_READY_DEADLINE"
+	EnvAPIClusterDeadline         = "MINIO_API_CLUSTER_DEADLINE"
 	EnvAPICorsAllowOrigin         = "MINIO_API_CORS_ALLOW_ORIGIN"
 	EnvAPIRemoteTransportDeadline = "MINIO_API_REMOTE_TRANSPORT_DEADLINE"
 )
 
+// Deprecated key and ENVs
+const (
+	apiReadyDeadline    = "ready_deadline"
+	EnvAPIReadyDeadline = "MINIO_API_READY_DEADLINE"
+)
+
 // DefaultKVS - default storage class config
 var (
 	DefaultKVS = config.KVS{
@@ -54,7 +60,7 @@ var (
 			Value: "10s",
 		},
 		config.KV{
-			Key:   apiReadyDeadline,
+			Key:   apiClusterDeadline,
 			Value: "10s",
 		},
 		config.KV{
@@ -72,7 +78,7 @@ var (
 type Config struct {
 	RequestsMax             int           `json:"requests_max"`
 	RequestsDeadline        time.Duration `json:"requests_deadline"`
-	ReadyDeadline           time.Duration `json:"ready_deadline"`
+	ClusterDeadline         time.Duration `json:"cluster_deadline"`
 	CorsAllowOrigin         []string      `json:"cors_allow_origin"`
 	RemoteTransportDeadline time.Duration `json:"remote_transport_deadline"`
 }
@@ -90,6 +96,9 @@ func (sCfg *Config) UnmarshalJSON(data []byte) error {
 
 // LookupConfig - lookup api config and override with valid environment settings if any.
 func LookupConfig(kvs config.KVS) (cfg Config, err error) {
+	// remove this since we have removed this already.
+	kvs.Delete(apiReadyDeadline)
+
 	if err = config.CheckValidKeys(config.APISubSys, kvs, DefaultKVS); err != nil {
 		return cfg, err
 	}
@@ -109,7 +118,7 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, err
 	}
 
-	readyDeadline, err := time.ParseDuration(env.Get(EnvAPIReadyDeadline, kvs.Get(apiReadyDeadline)))
+	clusterDeadline, err := time.ParseDuration(env.Get(EnvAPIClusterDeadline, kvs.Get(apiClusterDeadline)))
 	if err != nil {
 		return cfg, err
 	}
@@ -124,7 +133,7 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	return Config{
 		RequestsMax:             requestsMax,
 		RequestsDeadline:        requestsDeadline,
-		ReadyDeadline:           readyDeadline,
+		ClusterDeadline:         clusterDeadline,
 		CorsAllowOrigin:         corsAllowOrigin,
 		RemoteTransportDeadline: remoteTransportDeadline,
 	}, nil

cmd/config/config.go

@@ -267,6 +267,16 @@ func (kvs KVS) Get(key string) string {
 	return ""
 }
 
+// Delete - deletes the key if present from the KV list.
+func (kvs *KVS) Delete(key string) {
+	for i, kv := range *kvs {
+		if kv.Key == key {
+			*kvs = append((*kvs)[:i], (*kvs)[i+1:]...)
+			return
+		}
+	}
+}
+
 // Lookup - lookup a key in a list of KVS
 func (kvs KVS) Lookup(key string) (string, bool) {
 	for _, kv := range kvs {

cmd/config/dns/operator_dns.go

@@ -22,10 +22,12 @@ import (
 	"crypto/x509"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"net/url"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/dgrijalva/jwt-go"
@@ -78,28 +80,39 @@ func (c *OperatorDNS) Put(bucket string) error {
 	defer cancel()
 	e, err := c.endpoint(bucket, false)
 	if err != nil {
-		return err
+		return newError(bucket, err)
 	}
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, e, nil)
 	if err != nil {
-		return err
+		return newError(bucket, err)
 	}
 	if err = c.addAuthHeader(req); err != nil {
-		return err
+		return newError(bucket, err)
 	}
 	resp, err := c.httpClient.Do(req)
 	if err != nil {
 		if derr := c.Delete(bucket); derr != nil {
-			return derr
+			return newError(bucket, derr)
 		}
 	}
+	var errorStringBuilder strings.Builder
+	io.Copy(&errorStringBuilder, io.LimitReader(resp.Body, resp.ContentLength))
 	xhttp.DrainBody(resp.Body)
 	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("request to create the service for bucket %s, failed with status %s", bucket, resp.Status)
+		errorString := errorStringBuilder.String()
+		return newError(bucket, fmt.Errorf("service create for bucket %s, failed with status %s, error %s", bucket, resp.Status, errorString))
 	}
 	return nil
 }
 
+func newError(bucket string, err error) error {
+	e := Error{bucket, err}
+	if strings.Contains(err.Error(), "invalid bucket name") {
+		return ErrInvalidBucketName(e)
+	}
+	return e
+}
+
 // Delete - Removes DNS entries added in Put().
 func (c *OperatorDNS) Delete(bucket string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultOperatorContextTimeout)

cmd/config/dns/store.go

@@ -16,6 +16,22 @@
 
 package dns
 
+// Error - DNS related errors error.
+type Error struct {
+	Bucket string
+	Err    error
+}
+
+// ErrInvalidBucketName for buckets with invalid name
+type ErrInvalidBucketName Error
+
+func (e ErrInvalidBucketName) Error() string {
+	return "invalid bucket name error: " + e.Err.Error()
+}
+func (e Error) Error() string {
+	return "dns related error: " + e.Err.Error()
+}
+
 // Store dns record store
 type Store interface {
 	Put(bucket string) error

cmd/config/errors-utils.go

@@ -19,7 +19,6 @@ package config
 import (
 	"errors"
 	"fmt"
-	"io"
 	"net"
 	"syscall"
 
@@ -111,18 +110,16 @@ func ErrorToErr(err error) Err {
 		case *net.OpError:
 			return ErrPortAccess(err).Msg("Insufficient permissions to use specified port")
 		}
-		return ErrNoPermissionsToAccessDirFiles(err).Msg("Insufficient permissions to access path")
-	} else if errors.Is(err, io.ErrUnexpectedEOF) {
-		return ErrUnexpectedDataContent(err)
-	} else {
-		// Failed to identify what type of error this, return a simple UI error
-		return Err{msg: err.Error()}
 	}
+
+	// Failed to identify what type of error this, return a simple UI error
+	return Err{msg: err.Error()}
 }
 
 // FmtError converts a fatal error message to a more clear error
 // using some colors
 func FmtError(introMsg string, err error, jsonFlag bool) string {
+
 	renderedTxt := ""
 	uiErr := ErrorToErr(err)
 	// JSON print

cmd/config/errors.go

@@ -205,12 +205,6 @@ Example 1:
 		`Use 'sudo setcap cap_net_bind_service=+ep /path/to/minio' to provide sufficient permissions`,
 	)
 
-	ErrNoPermissionsToAccessDirFiles = newErrFn(
-		"Missing permissions to access the specified path",
-		"Please ensure the specified path can be accessed",
-		"",
-	)
-
 	ErrSSLUnexpectedError = newErrFn(
 		"Invalid TLS certificate",
 		"Please check the content of your certificate data",
@@ -247,12 +241,6 @@ Example 1:
 		"",
 	)
 
-	ErrUnexpectedDataContent = newErrFn(
-		"Unexpected data content",
-		"Please contact MinIO at https://slack.min.io",
-		"",
-	)
-
 	ErrUnexpectedError = newErrFn(
 		"Unexpected error",
 		"Please contact MinIO at https://slack.min.io",

cmd/config/notify/help.go

@@ -340,6 +340,12 @@ var (
 			Optional:    true,
 			Type:        "sentence",
 		},
+		config.HelpKV{
+			Key:         target.PostgresMaxOpenConnections,
+			Description: "To set the maximum number of open connections to the database. The value is set to `2` by default.",
+			Optional:    true,
+			Type:        "number",
+		},
 	}
 
 	HelpMySQL = config.HelpKVS{
@@ -377,6 +383,12 @@ var (
 			Optional:    true,
 			Type:        "sentence",
 		},
+		config.HelpKV{
+			Key:         target.MySQLMaxOpenConnections,
+			Description: "To set the maximum number of open connections to the database. The value is set to `2` by default.",
+			Optional:    true,
+			Type:        "number",
+		},
 	}
 
 	HelpNATS = config.HelpKVS{

cmd/config/notify/legacy.go

@@ -357,6 +357,10 @@ func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArg
 			Key:   target.PostgresQueueLimit,
 			Value: strconv.Itoa(int(cfg.QueueLimit)),
 		},
+		config.KV{
+			Key:   target.PostgresMaxOpenConnections,
+			Value: strconv.Itoa(cfg.MaxOpenConnections),
+		},
 	}
 
 	return nil
@@ -554,6 +558,10 @@ func SetNotifyMySQL(s config.Config, sqlName string, cfg target.MySQLArgs) error
 			Key:   target.MySQLQueueLimit,
 			Value: strconv.Itoa(int(cfg.QueueLimit)),
 		},
+		config.KV{
+			Key:   target.MySQLMaxOpenConnections,
+			Value: strconv.Itoa(cfg.MaxOpenConnections),
+		},
 	}
 
 	return nil

cmd/config/notify/parse.go

@@ -807,6 +807,10 @@ var (
 			Key:   target.MySQLQueueLimit,
 			Value: "0",
 		},
+		config.KV{
+			Key:   target.MySQLMaxOpenConnections,
+			Value: "2",
+		},
 	}
 )
 
@@ -855,13 +859,25 @@ func GetNotifyMySQL(mysqlKVS map[string]config.KVS) (map[string]target.MySQLArgs
 		if k != config.Default {
 			queueDirEnv = queueDirEnv + config.Default + k
 		}
+
+		maxOpenConnectionsEnv := target.EnvMySQLMaxOpenConnections
+		if k != config.Default {
+			maxOpenConnectionsEnv = maxOpenConnectionsEnv + config.Default + k
+		}
+
+		maxOpenConnections, cErr := strconv.Atoi(env.Get(maxOpenConnectionsEnv, kv.Get(target.MySQLMaxOpenConnections)))
+		if cErr != nil {
+			return nil, cErr
+		}
+
 		mysqlArgs := target.MySQLArgs{
-			Enable:     enabled,
-			Format:     env.Get(formatEnv, kv.Get(target.MySQLFormat)),
-			DSN:        env.Get(dsnStringEnv, kv.Get(target.MySQLDSNString)),
-			Table:      env.Get(tableEnv, kv.Get(target.MySQLTable)),
-			QueueDir:   env.Get(queueDirEnv, kv.Get(target.MySQLQueueDir)),
-			QueueLimit: queueLimit,
+			Enable:             enabled,
+			Format:             env.Get(formatEnv, kv.Get(target.MySQLFormat)),
+			DSN:                env.Get(dsnStringEnv, kv.Get(target.MySQLDSNString)),
+			Table:              env.Get(tableEnv, kv.Get(target.MySQLTable)),
+			QueueDir:           env.Get(queueDirEnv, kv.Get(target.MySQLQueueDir)),
+			QueueLimit:         queueLimit,
+			MaxOpenConnections: maxOpenConnections,
 		}
 		if err = mysqlArgs.Validate(); err != nil {
 			return nil, err
@@ -1235,6 +1251,10 @@ var (
 			Key:   target.PostgresQueueLimit,
 			Value: "0",
 		},
+		config.KV{
+			Key:   target.PostgresMaxOpenConnections,
+			Value: "2",
+		},
 	}
 )
 
@@ -1285,13 +1305,24 @@ func GetNotifyPostgres(postgresKVS map[string]config.KVS) (map[string]target.Pos
 			queueDirEnv = queueDirEnv + config.Default + k
 		}
 
+		maxOpenConnectionsEnv := target.EnvPostgresMaxOpenConnections
+		if k != config.Default {
+			maxOpenConnectionsEnv = maxOpenConnectionsEnv + config.Default + k
+		}
+
+		maxOpenConnections, cErr := strconv.Atoi(env.Get(maxOpenConnectionsEnv, kv.Get(target.PostgresMaxOpenConnections)))
+		if cErr != nil {
+			return nil, cErr
+		}
+
 		psqlArgs := target.PostgreSQLArgs{
-			Enable:           enabled,
-			Format:           env.Get(formatEnv, kv.Get(target.PostgresFormat)),
-			ConnectionString: env.Get(connectionStringEnv, kv.Get(target.PostgresConnectionString)),
-			Table:            env.Get(tableEnv, kv.Get(target.PostgresTable)),
-			QueueDir:         env.Get(queueDirEnv, kv.Get(target.PostgresQueueDir)),
-			QueueLimit:       uint64(queueLimit),
+			Enable:             enabled,
+			Format:             env.Get(formatEnv, kv.Get(target.PostgresFormat)),
+			ConnectionString:   env.Get(connectionStringEnv, kv.Get(target.PostgresConnectionString)),
+			Table:              env.Get(tableEnv, kv.Get(target.PostgresTable)),
+			QueueDir:           env.Get(queueDirEnv, kv.Get(target.PostgresQueueDir)),
+			QueueLimit:         uint64(queueLimit),
+			MaxOpenConnections: maxOpenConnections,
 		}
 		if err = psqlArgs.Validate(); err != nil {
 			return nil, err

cmd/consolelogger.go

@@ -122,6 +122,16 @@ func (sys *HTTPConsoleLoggerSys) Validate() error {
 	return nil
 }
 
+// Endpoint - dummy function for interface compatibility
+func (sys *HTTPConsoleLoggerSys) Endpoint() string {
+	return sys.console.Endpoint()
+}
+
+// String - stringer function for interface compatibility
+func (sys *HTTPConsoleLoggerSys) String() string {
+	return "console+http"
+}
+
 // Content returns the console stdout log
 func (sys *HTTPConsoleLoggerSys) Content() (logs []log.Entry) {
 	sys.RLock()

cmd/data-crawler.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
+	"math/rand"
 	"os"
 	"path"
 	"strconv"
@@ -49,11 +50,11 @@ const (
 	healDeleteDangling    = true
 	healFolderIncludeProb = 32  // Include a clean folder one in n cycles.
 	healObjectSelectProb  = 512 // Overall probability of a file being scanned; one in n.
-
 )
 
 var (
-	globalCrawlerConfig crawler.Config
+	globalCrawlerConfig          crawler.Config
+	dataCrawlerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 )
 
 // initDataCrawler will start the crawler unless disabled.
@@ -67,6 +68,19 @@ func initDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 // The function will block until the context is canceled.
 // There should only ever be one crawler running per cluster.
 func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
+	// Make sure only 1 crawler is running on the cluster.
+	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "runDataCrawler.lock")
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	for {
+		err := locker.GetLock(dataCrawlerLeaderLockTimeout)
+		if err != nil {
+			time.Sleep(time.Duration(r.Float64() * float64(dataCrawlStartDelay)))
+			continue
+		}
+		break
+		// No unlock for "leader" lock.
+	}
+
 	// Load current bloom cycle
 	nextBloomCycle := intDataUpdateTracker.current() + 1
 	var buf bytes.Buffer
@@ -334,7 +348,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 		// If there are lifecycle rules for the prefix, remove the filter.
 		filter := f.withFilter
 		var activeLifeCycle *lifecycle.Lifecycle
-		if f.oldCache.Info.lifeCycle != nil && filter != nil {
+		if f.oldCache.Info.lifeCycle != nil {
 			_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
 			if f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
 				if f.dataUsageCrawlDebug {
@@ -448,7 +462,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			continue
 		}
 
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			continue
 		}

cmd/data-update-tracker.go

@@ -46,7 +46,7 @@ const (
 	dataUpdateTrackerQueueSize = 10000
 
 	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
-	dataUpdateTrackerVersion      = 2
+	dataUpdateTrackerVersion      = 3
 	dataUpdateTrackerSaveInterval = 5 * time.Minute
 )
 
@@ -236,7 +236,10 @@ func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Durati
 		d.mu.Lock()
 		if !d.dirty {
 			d.mu.Unlock()
-			return
+			if exit {
+				return
+			}
+			continue
 		}
 		d.Saved = UTCNow()
 		err := d.serialize(&buf)
@@ -363,7 +366,7 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 		return err
 	}
 	switch tmp[0] {
-	case 1:
+	case 1, 2:
 		logger.Info(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
 		return nil
 	case dataUpdateTrackerVersion:

cmd/data-update-tracker_test.go

@@ -42,6 +42,14 @@ type testingLogger struct {
 	t  testLoggerI
 }
 
+func (t *testingLogger) Endpoint() string {
+	return ""
+}
+
+func (t *testingLogger) String() string {
+	return ""
+}
+
 func (t *testingLogger) Validate() error {
 	return nil
 }

cmd/disk-cache-backend.go

@@ -194,7 +194,7 @@ func (c *diskCache) diskUsageLow() bool {
 		logger.LogIf(ctx, err)
 		return false
 	}
-	usedPercent := (di.Total - di.Free) * 100 / di.Total
+	usedPercent := (di.Used / di.Total) * 100
 	low := int(usedPercent) < gcStopPct
 	atomic.StoreUint64(&c.stats.UsagePercent, usedPercent)
 	if low {
@@ -218,7 +218,7 @@ func (c *diskCache) diskSpaceAvailable(size int64) bool {
 		logger.Info("diskCache: Received 0 total disk size")
 		return false
 	}
-	usedPercent := float64(di.Total-di.Free) * 100 / float64(di.Total)
+	usedPercent := float64(di.Used) * 100 / float64(di.Total)
 	if usedPercent >= float64(gcTriggerPct) {
 		atomic.StoreInt32(&c.stats.UsageState, 1)
 		c.queueGC()
@@ -226,7 +226,7 @@ func (c *diskCache) diskSpaceAvailable(size int64) bool {
 	atomic.StoreUint64(&c.stats.UsagePercent, uint64(usedPercent))
 
 	// Recalculate percentage with provided size added.
-	usedPercent = float64(di.Total-di.Free+uint64(size)) * 100 / float64(di.Total)
+	usedPercent = float64(di.Used+uint64(size)) * 100 / float64(di.Total)
 
 	return usedPercent < float64(c.quotaPct)
 }

cmd/dynamic-timeouts.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"math"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -26,7 +27,8 @@ const (
 	dynamicTimeoutIncreaseThresholdPct = 0.33 // Upper threshold for failures in order to increase timeout
 	dynamicTimeoutDecreaseThresholdPct = 0.10 // Lower threshold for failures in order to decrease timeout
 	dynamicTimeoutLogSize              = 16
-	maxDuration                        = 1<<63 - 1
+	maxDuration                        = math.MaxInt64
+	maxDynamicTimeout                  = 24 * time.Hour // Never set timeout bigger than this.
 )
 
 // timeouts that are dynamically adapted based on actual usage results
@@ -40,6 +42,12 @@ type dynamicTimeout struct {
 
 // newDynamicTimeout returns a new dynamic timeout initialized with timeout value
 func newDynamicTimeout(timeout, minimum time.Duration) *dynamicTimeout {
+	if timeout <= 0 || minimum <= 0 {
+		panic("newDynamicTimeout: negative or zero timeout")
+	}
+	if minimum > timeout {
+		minimum = timeout
+	}
 	return &dynamicTimeout{timeout: int64(timeout), minimum: int64(minimum)}
 }
 
@@ -61,60 +69,73 @@ func (dt *dynamicTimeout) LogFailure() {
 
 // logEntry stores a log entry
 func (dt *dynamicTimeout) logEntry(duration time.Duration) {
+	if duration < 0 {
+		return
+	}
 	entries := int(atomic.AddInt64(&dt.entries, 1))
 	index := entries - 1
 	if index < dynamicTimeoutLogSize {
 		dt.mutex.Lock()
 		dt.log[index] = duration
+
+		// We leak entries while we copy
+		if entries == dynamicTimeoutLogSize {
+
+			// Make copy on stack in order to call adjust()
+			logCopy := [dynamicTimeoutLogSize]time.Duration{}
+			copy(logCopy[:], dt.log[:])
+
+			// reset log entries
+			atomic.StoreInt64(&dt.entries, 0)
+			dt.mutex.Unlock()
+
+			dt.adjust(logCopy)
+			return
+		}
 		dt.mutex.Unlock()
 	}
-	if entries == dynamicTimeoutLogSize {
-		dt.mutex.Lock()
-
-		// Make copy on stack in order to call adjust()
-		logCopy := [dynamicTimeoutLogSize]time.Duration{}
-		copy(logCopy[:], dt.log[:])
-
-		// reset log entries
-		atomic.StoreInt64(&dt.entries, 0)
-
-		dt.mutex.Unlock()
-
-		dt.adjust(logCopy)
-	}
 }
 
 // adjust changes the value of the dynamic timeout based on the
 // previous results
 func (dt *dynamicTimeout) adjust(entries [dynamicTimeoutLogSize]time.Duration) {
-
-	failures, average := 0, int64(0)
-	for i := 0; i < len(entries); i++ {
-		if entries[i] == maxDuration {
+	failures, max := 0, time.Duration(0)
+	for _, dur := range entries[:] {
+		if dur == maxDuration {
 			failures++
-		} else {
-			average += int64(entries[i])
+		} else if dur > max {
+			max = dur
 		}
 	}
-	if failures < len(entries) {
-		average /= int64(len(entries) - failures)
-	}
 
-	timeOutHitPct := float64(failures) / float64(len(entries))
+	failPct := float64(failures) / float64(len(entries))
 
-	if timeOutHitPct > dynamicTimeoutIncreaseThresholdPct {
+	if failPct > dynamicTimeoutIncreaseThresholdPct {
 		// We are hitting the timeout too often, so increase the timeout by 25%
 		timeout := atomic.LoadInt64(&dt.timeout) * 125 / 100
-		atomic.StoreInt64(&dt.timeout, timeout)
-	} else if timeOutHitPct < dynamicTimeoutDecreaseThresholdPct {
-		// We are hitting the timeout relatively few times, so decrease the timeout
-		average = average * 125 / 100 // Add buffer of 25% on top of average
 
-		timeout := (atomic.LoadInt64(&dt.timeout) + average) / 2 // Middle between current timeout and average success
+		// Set upper cap.
+		if timeout > int64(maxDynamicTimeout) {
+			timeout = int64(maxDynamicTimeout)
+		}
+		// Safety, shouldn't happen
+		if timeout < dt.minimum {
+			timeout = dt.minimum
+		}
+		atomic.StoreInt64(&dt.timeout, timeout)
+	} else if failPct < dynamicTimeoutDecreaseThresholdPct {
+		// We are hitting the timeout relatively few times,
+		// so decrease the timeout towards 25 % of maximum time spent.
+		max = max * 125 / 100
+
+		timeout := atomic.LoadInt64(&dt.timeout)
+		if max < time.Duration(timeout) {
+			// Move 50% toward the max.
+			timeout = (int64(max) + timeout) / 2
+		}
 		if timeout < dt.minimum {
 			timeout = dt.minimum
 		}
 		atomic.StoreInt64(&dt.timeout, timeout)
 	}
-
 }

cmd/dynamic-timeouts_test.go

@@ -18,6 +18,8 @@ package cmd
 
 import (
 	"math/rand"
+	"runtime"
+	"sync"
 	"testing"
 	"time"
 )
@@ -98,7 +100,7 @@ func TestDynamicTimeoutDualDecrease(t *testing.T) {
 	adjustedAgain := timeout.Timeout()
 
 	if initial <= adjusted || adjusted <= adjustedAgain {
-		t.Errorf("Failure to decrease timeout multiple times")
+		t.Errorf("Failure to decrease timeout multiple times, initial: %v, adjusted: %v, again: %v", initial, adjusted, adjustedAgain)
 	}
 }
 
@@ -123,6 +125,30 @@ func TestDynamicTimeoutManyDecreases(t *testing.T) {
 	}
 }
 
+func TestDynamicTimeoutConcurrent(t *testing.T) {
+	// Race test.
+	timeout := newDynamicTimeout(time.Second, time.Millisecond)
+	var wg sync.WaitGroup
+	for i := 0; i < runtime.GOMAXPROCS(0); i++ {
+		wg.Add(1)
+		rng := rand.New(rand.NewSource(int64(i)))
+		go func() {
+			defer wg.Done()
+			for i := 0; i < 100; i++ {
+				timeout.LogFailure()
+				for j := 0; j < 100; j++ {
+					timeout.LogSuccess(time.Duration(float64(time.Second) * rng.Float64()))
+				}
+				to := timeout.Timeout()
+				if to < time.Millisecond || to > time.Second {
+					panic(to)
+				}
+			}
+		}()
+	}
+	wg.Wait()
+}
+
 func TestDynamicTimeoutHitMinimum(t *testing.T) {
 
 	const minimum = 30 * time.Second
@@ -168,7 +194,7 @@ func TestDynamicTimeoutAdjustExponential(t *testing.T) {
 
 	timeout := newDynamicTimeout(time.Minute, time.Second)
 
-	rand.Seed(time.Now().UTC().UnixNano())
+	rand.Seed(0)
 
 	initial := timeout.Timeout()
 
@@ -188,7 +214,7 @@ func TestDynamicTimeoutAdjustNormalized(t *testing.T) {
 
 	timeout := newDynamicTimeout(time.Minute, time.Second)
 
-	rand.Seed(time.Now().UTC().UnixNano())
+	rand.Seed(0)
 
 	initial := timeout.Timeout()
 

cmd/endpoint.go

@@ -770,10 +770,11 @@ func GetProxyEndpoints(endpointZones EndpointZones) ([]ProxyEndpoint, error) {
 				}
 			}
 
-			tr := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)()
+			// allow transport to be HTTP/1.1 for proxying.
+			tr := newCustomHTTP11Transport(tlsConfig, rest.DefaultTimeout)()
+
 			// Allow more requests to be in flight with higher response header timeout.
 			tr.ResponseHeaderTimeout = 30 * time.Minute
-			tr.MaxIdleConns = 64
 			tr.MaxIdleConnsPerHost = 64
 
 			proxyEps = append(proxyEps, ProxyEndpoint{

cmd/erasure-bucket.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"sort"
 
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
@@ -86,31 +85,40 @@ func undoDeleteBucket(storageDisks []StorageAPI, bucket string) {
 
 // getBucketInfo - returns the BucketInfo from one of the load balanced disks.
 func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (bucketInfo BucketInfo, err error) {
-	var bucketErrs []error
-	for _, disk := range er.getLoadBalancedDisks() {
-		if disk == nil {
-			bucketErrs = append(bucketErrs, errDiskNotFound)
-			continue
-		}
-		volInfo, serr := disk.StatVol(ctx, bucketName)
-		if serr == nil {
-			return BucketInfo(volInfo), nil
-		}
-		err = serr
-		// For any reason disk went offline continue and pick the next one.
-		if IsErrIgnored(err, bucketMetadataOpIgnoredErrs...) {
-			bucketErrs = append(bucketErrs, err)
-			continue
-		}
-		// Any error which cannot be ignored, we return quickly.
-		return BucketInfo{}, err
+	storageDisks := er.getDisks()
+
+	g := errgroup.WithNErrs(len(storageDisks))
+	var bucketsInfo = make([]BucketInfo, len(storageDisks))
+	// Undo previous make bucket entry on all underlying storage disks.
+	for index := range storageDisks {
+		index := index
+		g.Go(func() error {
+			if storageDisks[index] == nil {
+				return errDiskNotFound
+			}
+			volInfo, err := storageDisks[index].StatVol(ctx, bucketName)
+			if err != nil {
+				return err
+			}
+			bucketsInfo[index] = BucketInfo(volInfo)
+			return nil
+		}, index)
 	}
+
+	errs := g.Wait()
+
+	for i, err := range errs {
+		if err == nil {
+			return bucketsInfo[i], nil
+		}
+	}
+
 	// If all our errors were ignored, then we try to
 	// reduce to one error based on read quorum.
 	// `nil` is deliberately passed for ignoredErrs
 	// because these errors were already ignored.
-	readQuorum := getReadQuorum(len(er.getDisks()))
-	return BucketInfo{}, reduceReadQuorumErrs(ctx, bucketErrs, nil, readQuorum)
+	readQuorum := getReadQuorum(len(storageDisks))
+	return BucketInfo{}, reduceReadQuorumErrs(ctx, errs, nil, readQuorum)
 }
 
 // GetBucketInfo - returns BucketInfo for a bucket.
@@ -122,53 +130,6 @@ func (er erasureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi B
 	return bucketInfo, nil
 }
 
-// listBuckets - returns list of all buckets from a disk picked at random.
-func (er erasureObjects) listBuckets(ctx context.Context) (bucketsInfo []BucketInfo, err error) {
-	for _, disk := range er.getLoadBalancedDisks() {
-		if disk == nil {
-			continue
-		}
-		var volsInfo []VolInfo
-		volsInfo, err = disk.ListVols(ctx)
-		if err == nil {
-			// NOTE: The assumption here is that volumes across all disks in
-			// readQuorum have consistent view i.e they all have same number
-			// of buckets.
-			var bucketsInfo []BucketInfo
-			for _, volInfo := range volsInfo {
-				if isReservedOrInvalidBucket(volInfo.Name, true) {
-					continue
-				}
-				bucketsInfo = append(bucketsInfo, BucketInfo(volInfo))
-			}
-			// For buckets info empty, loop once again to check
-			// if we have, can happen if disks were down.
-			if len(bucketsInfo) == 0 {
-				continue
-			}
-			return bucketsInfo, nil
-		}
-		logger.LogIf(ctx, err)
-		// Ignore any disks not found.
-		if IsErrIgnored(err, bucketMetadataOpIgnoredErrs...) {
-			continue
-		}
-		break
-	}
-	return nil, err
-}
-
-// ListBuckets - lists all the buckets, sorted by its name.
-func (er erasureObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
-	bucketInfos, err := er.listBuckets(ctx)
-	if err != nil {
-		return nil, toObjectErr(err)
-	}
-	// Sort by bucket name before returning.
-	sort.Sort(byBucketName(bucketInfos))
-	return bucketInfos, nil
-}
-
 // Dangling buckets should be handled appropriately, in this following situation
 // we actually have quorum error to be `nil` but we have some disks where
 // the bucket delete returned `errVolumeNotEmpty` but this is not correct

cmd/erasure-common.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"path"
+	"sync"
 
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
@@ -28,7 +29,9 @@ func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 	// Based on the random shuffling return back randomized disks.
 	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
 		if disks[i-1] != nil && disks[i-1].IsLocal() {
-			newDisks = append(newDisks, disks[i-1])
+			if !disks[i-1].Healing() && disks[i-1].IsOnline() {
+				newDisks = append(newDisks, disks[i-1])
+			}
 		}
 	}
 	return newDisks
@@ -38,11 +41,8 @@ func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 // with N disks online. If ndisks is zero or negative, then it will returns all disks,
 // same if ndisks is greater than the number of all disks.
 func (er erasureObjects) getLoadBalancedNDisks(ndisks int) (newDisks []StorageAPI) {
-	disks := er.getLoadBalancedDisks()
+	disks := er.getLoadBalancedDisks(ndisks != -1)
 	for _, disk := range disks {
-		if disk == nil {
-			continue
-		}
 		newDisks = append(newDisks, disk)
 		ndisks--
 		if ndisks == 0 {
@@ -53,13 +53,58 @@ func (er erasureObjects) getLoadBalancedNDisks(ndisks int) (newDisks []StorageAP
 }
 
 // getLoadBalancedDisks - fetches load balanced (sufficiently randomized) disk slice.
-func (er erasureObjects) getLoadBalancedDisks() (newDisks []StorageAPI) {
+// ensures to skip disks if they are not healing and online.
+func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
 	disks := er.getDisks()
+
+	if !optimized {
+		var newDisks []StorageAPI
+		for _, i := range hashOrder(UTCNow().String(), len(disks)) {
+			newDisks = append(newDisks, disks[i-1])
+		}
+		return newDisks
+	}
+
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	var newDisks = map[uint64][]StorageAPI{}
 	// Based on the random shuffling return back randomized disks.
 	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
-		newDisks = append(newDisks, disks[i-1])
+		i := i
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if disks[i-1] == nil {
+				return
+			}
+			di, err := disks[i-1].DiskInfo(context.Background())
+			if err != nil || di.Healing {
+				// - Do not consume disks which are not reachable
+				//   unformatted or simply not accessible for some reason.
+				//
+				// - Do not consume disks which are being healed
+				//
+				// - Future: skip busy disks
+				return
+			}
+
+			mu.Lock()
+			// Capture disks usage wise
+			newDisks[di.Used] = append(newDisks[di.Used], disks[i-1])
+			mu.Unlock()
+		}()
 	}
-	return newDisks
+	wg.Wait()
+
+	var max uint64
+	for k := range newDisks {
+		if k > max {
+			max = k
+		}
+	}
+
+	// Return disks which have maximum disk usage common.
+	return newDisks[max]
 }
 
 // This function does the following check, suppose

cmd/erasure-healing.go

@@ -155,34 +155,37 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
 
 // listAllBuckets lists all buckets from all disks. It also
 // returns the occurrence of each buckets in all disks
-func listAllBuckets(storageDisks []StorageAPI, healBuckets map[string]VolInfo) (err error) {
-	for _, disk := range storageDisks {
-		if disk == nil {
-			continue
-		}
-		var volsInfo []VolInfo
-		volsInfo, err = disk.ListVols(context.TODO())
-		if err != nil {
-			if IsErrIgnored(err, bucketMetadataOpIgnoredErrs...) {
-				continue
+func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets map[string]VolInfo) error {
+	g := errgroup.WithNErrs(len(storageDisks))
+	var mu sync.Mutex
+	for index := range storageDisks {
+		index := index
+		g.Go(func() error {
+			if storageDisks[index] == nil {
+				// we ignore disk not found errors
+				return nil
 			}
-			return err
-		}
-		for _, volInfo := range volsInfo {
-			// StorageAPI can send volume names which are
-			// incompatible with buckets - these are
-			// skipped, like the meta-bucket.
-			if isReservedOrInvalidBucket(volInfo.Name, false) {
-				continue
+			volsInfo, err := storageDisks[index].ListVols(ctx)
+			if err != nil {
+				return err
 			}
-			// always save unique buckets across drives.
-			if _, ok := healBuckets[volInfo.Name]; !ok {
-				healBuckets[volInfo.Name] = volInfo
+			for _, volInfo := range volsInfo {
+				// StorageAPI can send volume names which are
+				// incompatible with buckets - these are
+				// skipped, like the meta-bucket.
+				if isReservedOrInvalidBucket(volInfo.Name, false) {
+					continue
+				}
+				mu.Lock()
+				if _, ok := healBuckets[volInfo.Name]; !ok {
+					healBuckets[volInfo.Name] = volInfo
+				}
+				mu.Unlock()
 			}
-
-		}
+			return nil
+		}, index)
 	}
-	return nil
+	return reduceReadQuorumErrs(ctx, g.Wait(), bucketMetadataOpIgnoredErrs, len(storageDisks)/2)
 }
 
 // Only heal on disks where we are sure that healing is needed. We can expand
@@ -265,7 +268,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			}
 		case errs[i] == errDiskNotFound, dataErrs[i] == errDiskNotFound:
 			driveState = madmin.DriveStateOffline
-		case errs[i] == errFileNotFound, errs[i] == errVolumeNotFound:
+		case errs[i] == errFileNotFound, errs[i] == errFileVersionNotFound, errs[i] == errVolumeNotFound:
 			fallthrough
 		case dataErrs[i] == errFileNotFound, dataErrs[i] == errFileVersionNotFound, dataErrs[i] == errVolumeNotFound:
 			driveState = madmin.DriveStateMissing

cmd/erasure-healing_test.go

@@ -422,7 +422,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
 	z := obj.(*erasureZones)
 	er := z.zones[0].sets[0]
 	firstDisk := er.getDisks()[0]
-	err = firstDisk.DeleteFile(context.Background(), bucket, object)
+	err = firstDisk.DeleteVol(context.Background(), pathJoin(bucket, encodeDirObject(object)), true)
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -434,7 +434,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
 	}
 
 	// Check if the empty directory is restored in the first disk
-	_, err = firstDisk.StatVol(context.Background(), pathJoin(bucket, object))
+	_, err = firstDisk.StatVol(context.Background(), pathJoin(bucket, encodeDirObject(object)))
 	if err != nil {
 		t.Fatalf("Expected object to be present but stat failed - %v", err)
 	}

cmd/erasure-metadata.go

@@ -96,21 +96,14 @@ func (fi FileInfo) IsValid() bool {
 
 // ToObjectInfo - Converts metadata to object info.
 func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo {
-	if HasSuffix(object, SlashSeparator) {
-		return ObjectInfo{
-			Bucket: bucket,
-			Name:   object,
-			IsDir:  true,
-		}
-	}
-
+	object = decodeDirObject(object)
 	versionID := fi.VersionID
 	if globalBucketVersioningSys.Enabled(bucket) && versionID == "" {
 		versionID = nullVersionID
 	}
 
 	objInfo := ObjectInfo{
-		IsDir:           false,
+		IsDir:           HasSuffix(object, SlashSeparator),
 		Bucket:          bucket,
 		Name:            object,
 		VersionID:       versionID,

cmd/erasure-multipart.go

@@ -148,10 +148,8 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 	result.Delimiter = delimiter
 
 	var uploadIDs []string
-	for _, disk := range er.getLoadBalancedDisks() {
-		if disk == nil {
-			continue
-		}
+	var disk StorageAPI
+	for _, disk = range er.getLoadBalancedDisks(true) {
 		uploadIDs, err = disk.ListDir(ctx, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
 		if err != nil {
 			if err == errDiskNotFound {
@@ -176,30 +174,20 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 
 	populatedUploadIds := set.NewStringSet()
 
-retry:
-	for _, disk := range er.getLoadBalancedDisks() {
-		if disk == nil {
+	for _, uploadID := range uploadIDs {
+		if populatedUploadIds.Contains(uploadID) {
 			continue
 		}
-		for _, uploadID := range uploadIDs {
-			if populatedUploadIds.Contains(uploadID) {
-				continue
-			}
-			fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "")
-			if err != nil {
-				if err == errDiskNotFound || err == errFileNotFound {
-					goto retry
-				}
-				return result, toObjectErr(err, bucket, object)
-			}
-			populatedUploadIds.Add(uploadID)
-			uploads = append(uploads, MultipartInfo{
-				Object:    object,
-				UploadID:  uploadID,
-				Initiated: fi.ModTime,
-			})
+		fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "")
+		if err != nil {
+			return result, toObjectErr(err, bucket, object)
 		}
-		break
+		populatedUploadIds.Add(uploadID)
+		uploads = append(uploads, MultipartInfo{
+			Object:    object,
+			UploadID:  uploadID,
+			Initiated: fi.ModTime,
+		})
 	}
 
 	sort.Slice(uploads, func(i int, j int) bool {

cmd/erasure-object.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -37,30 +36,6 @@ import (
 // list all errors which can be ignored in object operations.
 var objectOpIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errUnformattedDisk)
 
-// putObjectDir hints the bottom layer to create a new directory.
-func (er erasureObjects) putObjectDir(ctx context.Context, bucket, object string, writeQuorum int) error {
-	storageDisks := er.getDisks()
-
-	g := errgroup.WithNErrs(len(storageDisks))
-
-	// Prepare object creation in all disks
-	for index := range storageDisks {
-		if storageDisks[index] == nil {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			err := storageDisks[index].MakeVol(ctx, pathJoin(bucket, object))
-			if err != nil && err != errVolumeExists {
-				return err
-			}
-			return nil
-		}, index)
-	}
-
-	return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, writeQuorum)
-}
-
 /// Object Operations
 
 // CopyObject - copy object source object to destination object.
@@ -147,10 +122,6 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 // GetObjectNInfo - returns object info and an object
 // Read(Closer). When err != nil, the returned reader is always nil.
 func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	if err = checkGetObjArgs(ctx, bucket, object); err != nil {
-		return nil, err
-	}
-
 	var unlockOnDefer bool
 	var nsUnlocker = func() {}
 	defer func() {
@@ -177,17 +148,6 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		unlockOnDefer = true
 	}
 
-	// Handler directory request by returning a reader that
-	// returns no bytes.
-	if HasSuffix(object, SlashSeparator) {
-		var objInfo ObjectInfo
-		if objInfo, err = er.getObjectInfoDir(ctx, bucket, object); err != nil {
-			return nil, toObjectErr(err, bucket, object)
-		}
-		unlockOnDefer = false
-		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts, nsUnlocker)
-	}
-
 	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts)
 	if err != nil {
 		return nil, toObjectErr(err, bucket, object)
@@ -232,10 +192,6 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
 func (er erasureObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
-	if err := checkGetObjArgs(ctx, bucket, object); err != nil {
-		return err
-	}
-
 	// Lock the object before reading.
 	lk := er.NewNSLock(ctx, bucket, object)
 	if err := lk.GetRLock(globalOperationTimeout); err != nil {
@@ -255,13 +211,6 @@ func (er erasureObjects) GetObject(ctx context.Context, bucket, object string, s
 		return errUnexpected
 	}
 
-	// If its a directory request, we return an empty body.
-	if HasSuffix(object, SlashSeparator) {
-		_, err := writer.Write([]byte(""))
-		logger.LogIf(ctx, err)
-		return toObjectErr(err, bucket, object)
-	}
-
 	return er.getObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
 }
 
@@ -390,43 +339,8 @@ func (er erasureObjects) getObject(ctx context.Context, bucket, object string, s
 	return er.getObjectWithFileInfo(ctx, bucket, object, startOffset, length, writer, etag, opts, fi, metaArr, onlineDisks)
 }
 
-// getObjectInfoDir - This getObjectInfo is specific to object directory lookup.
-func (er erasureObjects) getObjectInfoDir(ctx context.Context, bucket, object string) (ObjectInfo, error) {
-	storageDisks := er.getDisks()
-
-	g := errgroup.WithNErrs(len(storageDisks))
-
-	// Prepare object creation in a all disks
-	for index, disk := range storageDisks {
-		if disk == nil {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			// Check if 'prefix' is an object on this 'disk'.
-			entries, err := storageDisks[index].ListDir(ctx, bucket, object, 1)
-			if err != nil {
-				return err
-			}
-			if len(entries) > 0 {
-				// Not a directory if not empty.
-				return errFileNotFound
-			}
-			return nil
-		}, index)
-	}
-
-	readQuorum := getReadQuorum(len(storageDisks))
-	err := reduceReadQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, readQuorum)
-	return dirObjectInfo(bucket, object, 0, map[string]string{}), err
-}
-
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
 func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) {
-	if err = checkGetObjArgs(ctx, bucket, object); err != nil {
-		return info, err
-	}
-
 	// Lock the object before reading.
 	lk := er.NewNSLock(ctx, bucket, object)
 	if err := lk.GetRLock(globalOperationTimeout); err != nil {
@@ -434,14 +348,6 @@ func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object strin
 	}
 	defer lk.RUnlock()
 
-	if HasSuffix(object, SlashSeparator) {
-		info, err = er.getObjectInfoDir(ctx, bucket, object)
-		if err != nil {
-			return info, toObjectErr(err, bucket, object)
-		}
-		return info, nil
-	}
-
 	return er.getObjectInfo(ctx, bucket, object, opts)
 }
 
@@ -608,11 +514,6 @@ func rename(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBuc
 // writes `xl.meta` which carries the necessary metadata for future
 // object operations.
 func (er erasureObjects) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	// Validate put object input args.
-	if err = checkPutObjectArgs(ctx, bucket, object, er, data.Size()); err != nil {
-		return ObjectInfo{}, err
-	}
-
 	return er.putObject(ctx, bucket, object, data, opts)
 }
 
@@ -650,24 +551,6 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	// object to delete.
 	defer er.deleteObject(context.Background(), minioMetaTmpBucket, tempObj, writeQuorum)
 
-	// This is a special case with size as '0' and object ends with
-	// a slash separator, we treat it like a valid operation and
-	// return success.
-	if isObjectDir(object, data.Size()) {
-		// Check if an object is present as one of the parent dir.
-		// -- FIXME. (needs a new kind of lock).
-		// -- FIXME (this also causes performance issue when disks are down).
-		if er.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-			return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
-		}
-
-		if err = er.putObjectDir(ctx, bucket, object, writeQuorum); err != nil {
-			return ObjectInfo{}, toObjectErr(err, bucket, object)
-		}
-
-		return dirObjectInfo(bucket, object, data.Size(), opts.UserDefined), nil
-	}
-
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < -1 {
 		logger.LogIf(ctx, errInvalidArgument, logger.Application)
@@ -830,11 +713,7 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			err := disks[index].DeleteVersion(ctx, bucket, object, fi)
-			if err != nil && err != errVolumeNotFound {
-				return err
-			}
-			return nil
+			return disks[index].DeleteVersion(ctx, bucket, object, fi)
 		}, index)
 	}
 
@@ -895,16 +774,9 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 	dobjects := make([]DeletedObject, len(objects))
 	writeQuorums := make([]int, len(objects))
 
-	for i, object := range objects {
-		errs[i] = checkDelObjArgs(ctx, bucket, object.ObjectName)
-	}
-
 	storageDisks := er.getDisks()
 
 	for i := range objects {
-		if errs[i] != nil {
-			continue
-		}
 		// Assume (N/2 + 1) quorums for all objects
 		// this is a theoretical assumption such that
 		// for delete's we do not need to honor storage
@@ -918,21 +790,19 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 	for i := range objects {
 		if objects[i].VersionID == "" {
 			if opts.Versioned || opts.VersionSuspended {
-				if !HasSuffix(objects[i].ObjectName, SlashSeparator) {
-					fi := FileInfo{
-						Name:    objects[i].ObjectName,
-						ModTime: UTCNow(),
-						Deleted: true, // delete marker
-					}
-					if opts.Versioned {
-						fi.VersionID = mustGetUUID()
-					}
-					// versioning suspended means we add `null`
-					// version as delete marker
-
-					versions[i] = fi
-					continue
+				fi := FileInfo{
+					Name:    objects[i].ObjectName,
+					ModTime: UTCNow(),
+					Deleted: true, // delete marker
 				}
+				if opts.Versioned {
+					fi.VersionID = mustGetUUID()
+				}
+				// versioning suspended means we add `null`
+				// version as delete marker
+
+				versions[i] = fi
+				continue
 			}
 		}
 		versions[i] = FileInfo{
@@ -985,11 +855,11 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 				dobjects[objIndex] = DeletedObject{
 					DeleteMarker:          versions[objIndex].Deleted,
 					DeleteMarkerVersionID: versions[objIndex].VersionID,
-					ObjectName:            versions[objIndex].Name,
+					ObjectName:            decodeDirObject(versions[objIndex].Name),
 				}
 			} else {
 				dobjects[objIndex] = DeletedObject{
-					ObjectName: versions[objIndex].Name,
+					ObjectName: decodeDirObject(versions[objIndex].Name),
 					VersionID:  versions[objIndex].VersionID,
 				}
 			}
@@ -1020,13 +890,18 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 // any error as it is not necessary for the handler to reply back a
 // response to the client request.
 func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	if err = checkDelObjArgs(ctx, bucket, object); err != nil {
-		return objInfo, err
+	goi, gerr := er.GetObjectInfo(ctx, bucket, object, opts)
+	if gerr != nil && goi.Name == "" {
+		switch gerr.(type) {
+		case InsufficientReadQuorum:
+			return objInfo, InsufficientWriteQuorum{}
+		}
+		return objInfo, gerr
 	}
 
 	// Acquire a write lock before deleting the object.
 	lk := er.NewNSLock(ctx, bucket, object)
-	if err = lk.GetLock(globalOperationTimeout); err != nil {
+	if err = lk.GetLock(globalDeleteOperationTimeout); err != nil {
 		return ObjectInfo{}, err
 	}
 	defer lk.Unlock()
@@ -1036,26 +911,24 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 
 	if opts.VersionID == "" {
 		if opts.Versioned || opts.VersionSuspended {
-			if !HasSuffix(object, SlashSeparator) {
-				fi := FileInfo{
-					Name:    object,
-					Deleted: true,
-					ModTime: UTCNow(),
-				}
-
-				if opts.Versioned {
-					fi.VersionID = mustGetUUID()
-				}
-
-				// versioning suspended means we add `null`
-				// version as delete marker
-
-				// Add delete marker, since we don't have any version specified explicitly.
-				if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi); err != nil {
-					return objInfo, toObjectErr(err, bucket, object)
-				}
-				return fi.ToObjectInfo(bucket, object), nil
+			fi := FileInfo{
+				Name:    object,
+				Deleted: true,
+				ModTime: UTCNow(),
 			}
+
+			if opts.Versioned {
+				fi.VersionID = mustGetUUID()
+			}
+
+			// versioning suspended means we add `null`
+			// version as delete marker
+
+			// Add delete marker, since we don't have any version specified explicitly.
+			if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi); err != nil {
+				return objInfo, toObjectErr(err, bucket, object)
+			}
+			return fi.ToObjectInfo(bucket, object), nil
 		}
 	}
 
@@ -1074,7 +947,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		}
 	}
 
-	return ObjectInfo{Bucket: bucket, Name: object, VersionID: opts.VersionID}, nil
+	return ObjectInfo{Bucket: bucket, Name: decodeDirObject(object), VersionID: opts.VersionID}, nil
 }
 
 // Send the successful but partial upload/delete, however ignore

cmd/erasure-sets.go

@@ -30,10 +30,12 @@ import (
 	"github.com/dchest/siphash"
 	"github.com/google/uuid"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bpool"
 	"github.com/minio/minio/pkg/dsync"
+	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
@@ -66,6 +68,9 @@ type erasureSets struct {
 	// Distributed locker clients.
 	erasureLockers setsDsyncLockers
 
+	// Distributed lock owner (constant per running instance).
+	erasureLockOwner string
+
 	// List of endpoints provided on the command line.
 	endpoints Endpoints
 
@@ -228,6 +233,11 @@ func (s *erasureSets) connectDisks() {
 				return
 			}
 			disk.SetDiskID(format.Erasure.This)
+			if endpoint.IsLocal && disk.Healing() {
+				globalBackgroundHealState.pushHealLocalDisks(disk.Endpoint())
+				logger.Info(fmt.Sprintf("Found the drive %s which needs healing, attempting to heal...", disk))
+			}
+
 			s.erasureDisksMu.Lock()
 			if s.erasureDisks[setIndex][diskIndex] != nil {
 				s.erasureDisks[setIndex][diskIndex].Close()
@@ -261,11 +271,18 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 	}
 }
 
-func (s *erasureSets) GetLockers(setIndex int) func() []dsync.NetLocker {
-	return func() []dsync.NetLocker {
+func (s *erasureSets) GetLockers(setIndex int) func() ([]dsync.NetLocker, string) {
+	return func() ([]dsync.NetLocker, string) {
 		lockers := make([]dsync.NetLocker, s.setDriveCount)
 		copy(lockers, s.erasureLockers[setIndex])
-		return lockers
+		sort.Slice(lockers, func(i, j int) bool {
+			// re-order lockers with affinity for
+			// - non-local lockers
+			// - online lockers
+			// are used first
+			return !lockers[i].IsLocal() && lockers[i].IsOnline()
+		})
+		return lockers, s.erasureLockOwner
 	}
 }
 
@@ -303,16 +320,24 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 	setDriveCount := len(format.Erasure.Sets[0])
 
 	endpointStrings := make([]string, len(endpoints))
+
+	listTolerancePerSet := 3
+	// By default this is off
+	if env.Get("MINIO_API_LIST_STRICT_QUORUM", config.EnableOff) == config.EnableOn {
+		listTolerancePerSet = -1
+	}
+
 	// Initialize the erasure sets instance.
 	s := &erasureSets{
 		sets:                make([]*erasureObjects, setCount),
 		erasureDisks:        make([][]StorageAPI, setCount),
 		erasureLockers:      make([][]dsync.NetLocker, setCount),
+		erasureLockOwner:    mustGetUUID(),
 		endpoints:           endpoints,
 		endpointStrings:     endpointStrings,
 		setCount:            setCount,
 		setDriveCount:       setDriveCount,
-		listTolerancePerSet: setDriveCount / 2,
+		listTolerancePerSet: listTolerancePerSet,
 		format:              format,
 		disksConnectEvent:   make(chan diskConnectInfo),
 		distributionAlgo:    format.Erasure.DistributionAlgo,
@@ -665,7 +690,7 @@ func undoDeleteBucketSets(ctx context.Context, bucket string, sets []*erasureObj
 // that all buckets are present on all sets.
 func (s *erasureSets) ListBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
 	// Always lists from the same set signified by the empty string.
-	return s.getHashedSet("").ListBuckets(ctx)
+	return s.ListBucketsHeal(ctx)
 }
 
 // --- Object Operations ---
@@ -1381,7 +1406,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 		}
 
 		// Save formats `format.json` across all disks.
-		if err = saveFormatErasureAll(ctx, storageDisks, tmpNewFormats); err != nil {
+		if err = saveFormatErasureAllWithErrs(ctx, storageDisks, sErrs, tmpNewFormats); err != nil {
 			return madmin.HealResultItem{}, err
 		}
 
@@ -1465,7 +1490,7 @@ func (s *erasureSets) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error)
 	var healBuckets = map[string]VolInfo{}
 	for _, set := range s.sets {
 		// lists all unique buckets across drives.
-		if err := listAllBuckets(set.getDisks(), healBuckets); err != nil {
+		if err := listAllBuckets(ctx, set.getDisks(), healBuckets); err != nil {
 			return nil, err
 		}
 	}

cmd/erasure-zones.go

@@ -298,19 +298,19 @@ func (z *erasureZones) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter
 
 	// Collect for each set in zones.
 	for _, z := range z.zones {
+		buckets, err := z.ListBuckets(ctx)
+		if err != nil {
+			return err
+		}
+		// Add new buckets.
+		for _, b := range buckets {
+			if _, ok := knownBuckets[b.Name]; ok {
+				continue
+			}
+			allBuckets = append(allBuckets, b)
+			knownBuckets[b.Name] = struct{}{}
+		}
 		for _, erObj := range z.sets {
-			// Add new buckets.
-			buckets, err := erObj.ListBuckets(ctx)
-			if err != nil {
-				return err
-			}
-			for _, b := range buckets {
-				if _, ok := knownBuckets[b.Name]; ok {
-					continue
-				}
-				allBuckets = append(allBuckets, b)
-				knownBuckets[b.Name] = struct{}{}
-			}
 			wg.Add(1)
 			results = append(results, dataUsageCache{})
 			go func(i int, erObj *erasureObjects) {
@@ -460,6 +460,12 @@ func (z *erasureZones) MakeBucketWithLocation(ctx context.Context, bucket string
 }
 
 func (z *erasureZones) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+	if err = checkGetObjArgs(ctx, bucket, object); err != nil {
+		return nil, err
+	}
+
+	object = encodeDirObject(object)
+
 	for _, zone := range z.zones {
 		gr, err = zone.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 		if err != nil {
@@ -477,6 +483,12 @@ func (z *erasureZones) GetObjectNInfo(ctx context.Context, bucket, object string
 }
 
 func (z *erasureZones) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
+	if err := checkGetObjArgs(ctx, bucket, object); err != nil {
+		return err
+	}
+
+	object = encodeDirObject(object)
+
 	for _, zone := range z.zones {
 		if err := zone.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts); err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
@@ -493,6 +505,11 @@ func (z *erasureZones) GetObject(ctx context.Context, bucket, object string, sta
 }
 
 func (z *erasureZones) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+	if err = checkGetObjArgs(ctx, bucket, object); err != nil {
+		return objInfo, err
+	}
+
+	object = encodeDirObject(object)
 	for _, zone := range z.zones {
 		objInfo, err = zone.GetObjectInfo(ctx, bucket, object, opts)
 		if err != nil {
@@ -503,6 +520,7 @@ func (z *erasureZones) GetObjectInfo(ctx context.Context, bucket, object string,
 		}
 		return objInfo, nil
 	}
+	object = decodeDirObject(object)
 	if opts.VersionID != "" {
 		return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
 	}
@@ -511,6 +529,13 @@ func (z *erasureZones) GetObjectInfo(ctx context.Context, bucket, object string,
 
 // PutObject - writes an object to least used erasure zone.
 func (z *erasureZones) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
+	// Validate put object input args.
+	if err := checkPutObjectArgs(ctx, bucket, object, z); err != nil {
+		return ObjectInfo{}, err
+	}
+
+	object = encodeDirObject(object)
+
 	if z.SingleZone() {
 		return z.zones[0].PutObject(ctx, bucket, object, data, opts)
 	}
@@ -525,6 +550,12 @@ func (z *erasureZones) PutObject(ctx context.Context, bucket string, object stri
 }
 
 func (z *erasureZones) DeleteObject(ctx context.Context, bucket string, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+	if err = checkDelObjArgs(ctx, bucket, object); err != nil {
+		return objInfo, err
+	}
+
+	object = encodeDirObject(object)
+
 	if z.SingleZone() {
 		return z.zones[0].DeleteObject(ctx, bucket, object, opts)
 	}
@@ -537,6 +568,7 @@ func (z *erasureZones) DeleteObject(ctx context.Context, bucket string, object s
 			break
 		}
 	}
+
 	return objInfo, err
 }
 
@@ -545,6 +577,8 @@ func (z *erasureZones) DeleteObjects(ctx context.Context, bucket string, objects
 	dobjects := make([]DeletedObject, len(objects))
 	objSets := set.NewStringSet()
 	for i := range derrs {
+		objects[i].ObjectName = encodeDirObject(objects[i].ObjectName)
+
 		derrs[i] = checkDelObjArgs(ctx, bucket, objects[i].ObjectName)
 		objSets.Add(objects[i].ObjectName)
 	}
@@ -576,6 +610,9 @@ func (z *erasureZones) DeleteObjects(ctx context.Context, bucket string, objects
 }
 
 func (z *erasureZones) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
+	srcObject = encodeDirObject(srcObject)
+	dstObject = encodeDirObject(dstObject)
+
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
 
 	zoneIdx, err := z.getZoneIdx(ctx, dstBucket, dstObject, dstOpts, srcInfo.Size)
@@ -646,8 +683,12 @@ func (z *erasureZones) listObjectsNonSlash(ctx context.Context, bucket, prefix,
 
 	for _, zone := range z.zones {
 		zonesEntryChs = append(zonesEntryChs,
-			zone.startMergeWalksN(ctx, bucket, prefix, "", true, endWalkCh, zone.setDriveCount, false))
-		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet)
+			zone.startMergeWalksN(ctx, bucket, prefix, "", true, endWalkCh, zone.listTolerancePerSet, false))
+		if zone.listTolerancePerSet == -1 {
+			zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.setDriveCount/2)
+		} else {
+			zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-2)
+		}
 	}
 
 	var objInfos []ObjectInfo
@@ -673,7 +714,7 @@ func (z *erasureZones) listObjectsNonSlash(ctx context.Context, bucket, prefix,
 		}
 
 		if quorumCount < zonesListTolerancePerSet[zoneIndex] {
-			// Skip entries which are not found on upto ndisks/2.
+			// Skip entries which are not found on upto expected tolerance
 			continue
 		}
 
@@ -765,11 +806,15 @@ func (z *erasureZones) listObjectsSplunk(ctx context.Context, bucket, prefix, ma
 		entryChs, endWalkCh := zone.poolSplunk.Release(listParams{bucket, recursive, marker, prefix})
 		if entryChs == nil {
 			endWalkCh = make(chan struct{})
-			entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.setDriveCount, true)
+			entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.listTolerancePerSet, true)
 		}
 		zonesEntryChs = append(zonesEntryChs, entryChs)
 		zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh)
-		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet)
+		if zone.listTolerancePerSet == -1 {
+			zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.setDriveCount/2)
+		} else {
+			zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-2)
+		}
 	}
 
 	entries := mergeZonesEntriesCh(zonesEntryChs, maxKeys, zonesListTolerancePerSet)
@@ -857,11 +902,15 @@ func (z *erasureZones) listObjects(ctx context.Context, bucket, prefix, marker,
 		entryChs, endWalkCh := zone.pool.Release(listParams{bucket, recursive, marker, prefix})
 		if entryChs == nil {
 			endWalkCh = make(chan struct{})
-			entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.setDriveCount, false)
+			entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.listTolerancePerSet, false)
 		}
 		zonesEntryChs = append(zonesEntryChs, entryChs)
 		zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh)
-		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet)
+		if zone.listTolerancePerSet == -1 {
+			zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.setDriveCount/2)
+		} else {
+			zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-2)
+		}
 	}
 
 	entries := mergeZonesEntriesCh(zonesEntryChs, maxKeys, zonesListTolerancePerSet)
@@ -935,7 +984,16 @@ func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileI
 				zoneIndex = i
 				continue
 			}
-			if zoneEntries[i][j].Name < lentry.Name {
+			str1 := zoneEntries[i][j].Name
+			str2 := lentry.Name
+			if HasSuffix(str1, globalDirSuffix) {
+				str1 = strings.TrimSuffix(str1, globalDirSuffix) + slashSeparator
+			}
+			if HasSuffix(str2, globalDirSuffix) {
+				str2 = strings.TrimSuffix(str2, globalDirSuffix) + slashSeparator
+			}
+
+			if str1 < str2 {
 				lentry = zoneEntries[i][j]
 				zoneIndex = i
 			}
@@ -968,6 +1026,10 @@ func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileI
 		}
 	}
 
+	if HasSuffix(lentry.Name, globalDirSuffix) {
+		lentry.Name = strings.TrimSuffix(lentry.Name, globalDirSuffix) + slashSeparator
+	}
+
 	return lentry, lexicallySortedEntryCount, zoneIndex, isTruncated
 }
 
@@ -1013,7 +1075,16 @@ func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneE
 				zoneIndex = i
 				continue
 			}
-			if zoneEntries[i][j].Name < lentry.Name {
+			str1 := zoneEntries[i][j].Name
+			str2 := lentry.Name
+			if HasSuffix(str1, globalDirSuffix) {
+				str1 = strings.TrimSuffix(str1, globalDirSuffix) + slashSeparator
+			}
+			if HasSuffix(str2, globalDirSuffix) {
+				str2 = strings.TrimSuffix(str2, globalDirSuffix) + slashSeparator
+			}
+
+			if str1 < str2 {
 				lentry = zoneEntries[i][j]
 				zoneIndex = i
 			}
@@ -1046,6 +1117,10 @@ func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneE
 		}
 	}
 
+	if HasSuffix(lentry.Name, globalDirSuffix) {
+		lentry.Name = strings.TrimSuffix(lentry.Name, globalDirSuffix) + slashSeparator
+	}
+
 	return lentry, lexicallySortedEntryCount, zoneIndex, isTruncated
 }
 
@@ -1058,6 +1133,7 @@ func mergeZonesEntriesVersionsCh(zonesEntryChs [][]FileInfoVersionsCh, maxKeys i
 		zonesEntriesInfos = append(zonesEntriesInfos, make([]FileInfoVersions, len(entryChs)))
 		zonesEntriesValid = append(zonesEntriesValid, make([]bool, len(entryChs)))
 	}
+
 	for {
 		fi, quorumCount, zoneIndex, ok := lexicallySortedEntryZoneVersions(zonesEntryChs, zonesEntriesInfos, zonesEntriesValid)
 		if !ok {
@@ -1089,6 +1165,7 @@ func mergeZonesEntriesCh(zonesEntryChs [][]FileInfoCh, maxKeys int, zonesListTol
 		zonesEntriesInfos = append(zonesEntriesInfos, make([]FileInfo, len(entryChs)))
 		zonesEntriesValid = append(zonesEntriesValid, make([]bool, len(entryChs)))
 	}
+	var prevEntry string
 	for {
 		fi, quorumCount, zoneIndex, ok := lexicallySortedEntryZone(zonesEntryChs, zonesEntriesInfos, zonesEntriesValid)
 		if !ok {
@@ -1101,12 +1178,17 @@ func mergeZonesEntriesCh(zonesEntryChs [][]FileInfoCh, maxKeys int, zonesListTol
 			continue
 		}
 
+		if HasSuffix(fi.Name, slashSeparator) && fi.Name == prevEntry {
+			continue
+		}
+
 		entries.Files = append(entries.Files, fi)
 		i++
 		if i == maxKeys {
 			entries.IsTruncated = isTruncatedZones(zonesEntryChs, zonesEntriesInfos, zonesEntriesValid)
 			break
 		}
+		prevEntry = fi.Name
 	}
 	return entries
 }
@@ -1226,11 +1308,15 @@ func (z *erasureZones) listObjectVersions(ctx context.Context, bucket, prefix, m
 		entryChs, endWalkCh := zone.poolVersions.Release(listParams{bucket, recursive, marker, prefix})
 		if entryChs == nil {
 			endWalkCh = make(chan struct{})
-			entryChs = zone.startMergeWalksVersionsN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.setDriveCount)
+			entryChs = zone.startMergeWalksVersionsN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.listTolerancePerSet)
 		}
 		zonesEntryChs = append(zonesEntryChs, entryChs)
 		zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh)
-		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet)
+		if zone.listTolerancePerSet == -1 {
+			zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.setDriveCount/2)
+		} else {
+			zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-2)
+		}
 	}
 
 	entries := mergeZonesEntriesVersionsCh(zonesEntryChs, maxKeys, zonesListTolerancePerSet)
@@ -1702,17 +1788,21 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results
 		return err
 	}
 
+	zonesListTolerancePerSet := make([]int, 0, len(z.zones))
+	for _, zone := range z.zones {
+		if zone.listTolerancePerSet == -1 {
+			zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.setDriveCount/2)
+		} else {
+			zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet-2)
+		}
+	}
+
 	if opts.WalkVersions {
 		var zonesEntryChs [][]FileInfoVersionsCh
 		for _, zone := range z.zones {
 			zonesEntryChs = append(zonesEntryChs, zone.startMergeWalksVersions(ctx, bucket, prefix, "", true, ctx.Done()))
 		}
 
-		var zoneDrivesPerSet []int
-		for _, zone := range z.zones {
-			zoneDrivesPerSet = append(zoneDrivesPerSet, zone.setDriveCount)
-		}
-
 		var zonesEntriesInfos [][]FileInfoVersions
 		var zonesEntriesValid [][]bool
 		for _, entryChs := range zonesEntryChs {
@@ -1724,20 +1814,17 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results
 			defer close(results)
 
 			for {
-				entry, quorumCount, zoneIndex, ok := lexicallySortedEntryZoneVersions(zonesEntryChs, zonesEntriesInfos, zonesEntriesValid)
+				entry, quorumCount, zoneIdx, ok := lexicallySortedEntryZoneVersions(zonesEntryChs, zonesEntriesInfos, zonesEntriesValid)
 				if !ok {
 					// We have reached EOF across all entryChs, break the loop.
 					return
 				}
 
-				if quorumCount >= zoneDrivesPerSet[zoneIndex]/2 {
-					// Read quorum exists proceed
+				if quorumCount >= zonesListTolerancePerSet[zoneIdx] {
 					for _, version := range entry.Versions {
 						results <- version.ToObjectInfo(bucket, version.Name)
 					}
 				}
-
-				// skip entries which do not have quorum
 			}
 		}()
 
@@ -1745,10 +1832,8 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results
 	}
 
 	zonesEntryChs := make([][]FileInfoCh, 0, len(z.zones))
-	zoneDrivesPerSet := make([]int, 0, len(z.zones))
 	for _, zone := range z.zones {
 		zonesEntryChs = append(zonesEntryChs, zone.startMergeWalks(ctx, bucket, prefix, "", true, ctx.Done()))
-		zoneDrivesPerSet = append(zoneDrivesPerSet, zone.setDriveCount)
 	}
 
 	zonesEntriesInfos := make([][]FileInfo, 0, len(zonesEntryChs))
@@ -1762,17 +1847,15 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results
 		defer close(results)
 
 		for {
-			entry, quorumCount, zoneIndex, ok := lexicallySortedEntryZone(zonesEntryChs, zonesEntriesInfos, zonesEntriesValid)
+			entry, quorumCount, zoneIdx, ok := lexicallySortedEntryZone(zonesEntryChs, zonesEntriesInfos, zonesEntriesValid)
 			if !ok {
 				// We have reached EOF across all entryChs, break the loop.
 				return
 			}
 
-			if quorumCount >= zoneDrivesPerSet[zoneIndex]/2 {
-				// Read quorum exists proceed
+			if quorumCount >= zonesListTolerancePerSet[zoneIdx] {
 				results <- entry.ToObjectInfo(bucket, entry.Name)
 			}
-			// skip entries which do not have quorum
 		}
 	}()
 
@@ -1811,6 +1894,7 @@ func (z *erasureZones) HealObjects(ctx context.Context, bucket, prefix string, o
 		if !ok {
 			break
 		}
+
 		// Indicate that first attempt was a success and subsequent loop
 		// knows that its not our first attempt at 'prefix'
 		err = nil
@@ -1818,6 +1902,7 @@ func (z *erasureZones) HealObjects(ctx context.Context, bucket, prefix string, o
 		if zoneIndex >= len(zoneDrivesPerSet) || zoneIndex < 0 {
 			return fmt.Errorf("invalid zone index returned: %d", zoneIndex)
 		}
+
 		if quorumCount == zoneDrivesPerSet[zoneIndex] && opts.ScanMode == madmin.HealNormalScan {
 			// Skip good entries.
 			continue
@@ -1836,6 +1921,8 @@ func (z *erasureZones) HealObjects(ctx context.Context, bucket, prefix string, o
 }
 
 func (z *erasureZones) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
+	object = encodeDirObject(object)
+
 	lk := z.NewNSLock(ctx, bucket, object)
 	if bucket == minioMetaBucket {
 		// For .minio.sys bucket heals we should hold write locks.
@@ -1956,6 +2043,7 @@ func (z *erasureZones) Health(ctx context.Context, opts HealthOptions) HealthRes
 
 	parityDrives := globalStorageClass.GetParityForSC(storageclass.STANDARD)
 	diskCount := z.SetDriveCount()
+
 	if parityDrives == 0 {
 		parityDrives = getDefaultParityBlocks(diskCount)
 	}
@@ -2019,6 +2107,7 @@ func (z *erasureZones) Health(ctx context.Context, opts HealthOptions) HealthRes
 
 // PutObjectTags - replace or add tags to an existing object
 func (z *erasureZones) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) error {
+	object = encodeDirObject(object)
 	if z.SingleZone() {
 		return z.zones[0].PutObjectTags(ctx, bucket, object, tags, opts)
 	}
@@ -2048,6 +2137,7 @@ func (z *erasureZones) PutObjectTags(ctx context.Context, bucket, object string,
 
 // DeleteObjectTags - delete object tags from an existing object
 func (z *erasureZones) DeleteObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) error {
+	object = encodeDirObject(object)
 	if z.SingleZone() {
 		return z.zones[0].DeleteObjectTags(ctx, bucket, object, opts)
 	}
@@ -2076,6 +2166,7 @@ func (z *erasureZones) DeleteObjectTags(ctx context.Context, bucket, object stri
 
 // GetObjectTags - get object tags from an existing object
 func (z *erasureZones) GetObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (*tags.Tags, error) {
+	object = encodeDirObject(object)
 	if z.SingleZone() {
 		return z.zones[0].GetObjectTags(ctx, bucket, object, opts)
 	}

cmd/erasure.go

@@ -52,7 +52,7 @@ type erasureObjects struct {
 	getDisks func() []StorageAPI
 
 	// getLockers returns list of remote and local lockers.
-	getLockers func() []dsync.NetLocker
+	getLockers func() ([]dsync.NetLocker, string)
 
 	// getEndpoints returns list of endpoint strings belonging this set.
 	// some may be local and some remote.
@@ -149,16 +149,6 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 				return errDiskNotFound
 			}
 			info, err := disks[index].DiskInfo(context.TODO())
-			if err != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("disk", disks[index].String())
-				ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-				logger.LogIf(ctx, err)
-				disksInfo[index] = madmin.Disk{
-					State:    diskErrToDriveState(err),
-					Endpoint: endpoints[index],
-				}
-				return err
-			}
 			di := madmin.Disk{
 				Endpoint:       endpoints[index],
 				DrivePath:      info.MountPath,
@@ -167,6 +157,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 				AvailableSpace: info.Free,
 				UUID:           info.ID,
 				RootDisk:       info.RootDisk,
+				Healing:        info.Healing,
 				State:          diskErrToDriveState(err),
 			}
 			if info.Total > 0 {
@@ -181,7 +172,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 	// Wait for the routines.
 	for i, diskInfoErr := range errs {
 		ep := disksInfo[i].Endpoint
-		if diskInfoErr != nil {
+		if diskInfoErr != nil && !errors.Is(diskInfoErr, errUnformattedDisk) {
 			offlineDisks[ep]++
 			continue
 		}
@@ -256,46 +247,20 @@ func (er erasureObjects) StorageInfo(ctx context.Context, local bool) (StorageIn
 // Updates are sent on a regular basis and the caller *must* consume them.
 func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, updates chan<- dataUsageCache) error {
 	if len(buckets) == 0 {
+		logger.Info(color.Green("data-crawl:") + " No buckets found, skipping crawl")
 		return nil
 	}
 
-	// Collect any disk healing.
-	healing, err := getAggregatedBackgroundHealState(ctx)
-	if err != nil {
-		return err
-	}
-
-	healDisks := make(map[string]struct{}, len(healing.HealDisks))
-	for _, disk := range healing.HealDisks {
-		healDisks[disk] = struct{}{}
-	}
-
 	// Collect disks we can use.
-	var disks []StorageAPI
-	for _, d := range er.getLoadBalancedDisks() {
-		if d == nil || !d.IsOnline() {
-			continue
-		}
-		di, err := d.DiskInfo(ctx)
-		if err != nil {
-			logger.LogIf(ctx, err)
-			continue
-		}
-		if _, ok := healDisks[di.Endpoint]; ok {
-			logger.Info(color.Green("data-crawl:")+" Disk %q is Healing, skipping disk.", di.Endpoint)
-			continue
-		}
-		disks = append(disks, d)
-	}
+	disks := er.getLoadBalancedDisks(true)
 	if len(disks) == 0 {
-		logger.Info(color.Green("data-crawl:") + " No disks found, skipping crawl")
+		logger.Info(color.Green("data-crawl:") + " all disks are offline or being healed, skipping crawl")
 		return nil
 	}
 
 	// Load bucket totals
 	oldCache := dataUsageCache{}
-	err = oldCache.load(ctx, er, dataUsageCacheName)
-	if err != nil {
+	if err := oldCache.load(ctx, er, dataUsageCacheName); err != nil {
 		return err
 	}
 
@@ -403,6 +368,7 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
 
 				// Calc usage
 				before := cache.Info.LastUpdate
+				var err error
 				cache, err = disk.CrawlAndGetDataUsage(ctx, cache)
 				cache.Info.BloomFilter = nil
 				if err != nil {

cmd/format-erasure.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"reflect"
@@ -335,7 +336,7 @@ func loadFormatErasureAll(storageDisks []StorageAPI, heal bool) ([]*formatErasur
 	return formats, g.Wait()
 }
 
-func saveFormatErasure(disk StorageAPI, format *formatErasureV3) error {
+func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) error {
 	if disk == nil || format == nil {
 		return errDiskNotFound
 	}
@@ -368,6 +369,18 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3) error {
 	}
 
 	disk.SetDiskID(diskID)
+	if heal {
+		htracker := healingTracker{
+			ID: diskID,
+		}
+		htrackerBytes, err := htracker.MarshalMsg(nil)
+		if err != nil {
+			return err
+		}
+		return disk.WriteAll(context.TODO(), minioMetaBucket,
+			pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename),
+			bytes.NewReader(htrackerBytes))
+	}
 	return nil
 }
 
@@ -551,7 +564,8 @@ func formatErasureFixLocalDeploymentID(endpoints Endpoints, storageDisks []Stora
 					return nil
 				}
 				format.ID = refFormat.ID
-				if err := saveFormatErasure(storageDisks[index], format); err != nil {
+				// Heal the drive if we fixed its deployment ID.
+				if err := saveFormatErasure(storageDisks[index], format, true); err != nil {
 					logger.LogIf(GlobalContext, err)
 					return fmt.Errorf("Unable to save format.json, %w", err)
 				}
@@ -686,6 +700,27 @@ func initErasureMetaVolumesInLocalDisks(storageDisks []StorageAPI, formats []*fo
 	return nil
 }
 
+// saveFormatErasureAllWithErrs - populates `format.json` on disks in its order.
+// also adds `.healing.bin` on the disks which are being actively healed.
+func saveFormatErasureAllWithErrs(ctx context.Context, storageDisks []StorageAPI, fErrs []error, formats []*formatErasureV3) error {
+	g := errgroup.WithNErrs(len(storageDisks))
+
+	// Write `format.json` to all disks.
+	for index := range storageDisks {
+		index := index
+		g.Go(func() error {
+			if formats[index] == nil {
+				return errDiskNotFound
+			}
+			return saveFormatErasure(storageDisks[index], formats[index], errors.Is(fErrs[index], errUnformattedDisk))
+		}, index)
+	}
+
+	writeQuorum := getWriteQuorum(len(storageDisks))
+	// Wait for the routines to finish.
+	return reduceWriteQuorumErrs(ctx, g.Wait(), nil, writeQuorum)
+}
+
 // saveFormatErasureAll - populates `format.json` on disks in its order.
 func saveFormatErasureAll(ctx context.Context, storageDisks []StorageAPI, formats []*formatErasureV3) error {
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -697,7 +732,7 @@ func saveFormatErasureAll(ctx context.Context, storageDisks []StorageAPI, format
 			if formats[index] == nil {
 				return errDiskNotFound
 			}
-			return saveFormatErasure(storageDisks[index], formats[index])
+			return saveFormatErasure(storageDisks[index], formats[index], false)
 		}, index)
 	}
 
@@ -771,7 +806,8 @@ func fixFormatErasureV3(storageDisks []StorageAPI, endpoints Endpoints, formats
 			}
 			if formats[i].Erasure.This == "" {
 				formats[i].Erasure.This = formats[i].Erasure.Sets[0][i]
-				if err := saveFormatErasure(storageDisks[i], formats[i]); err != nil {
+				// Heal the drive if drive has .This empty.
+				if err := saveFormatErasure(storageDisks[i], formats[i], true); err != nil {
 					return err
 				}
 			}

cmd/fs-v1-multipart.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -31,6 +32,7 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/logger"
 	mioutil "github.com/minio/minio/pkg/ioutil"
+	"github.com/minio/minio/pkg/trie"
 )
 
 // Returns EXPORT/.minio.sys/multipart/SHA256/UPLOADID
@@ -446,52 +448,51 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 		return result, toObjectErr(err, bucket)
 	}
 
-	partsMap := make(map[int]string)
+	partsMap := make(map[int]PartInfo)
 	for _, entry := range entries {
 		if entry == fs.metaJSONFile {
 			continue
 		}
-		partNumber, etag1, _, derr := fs.decodePartFile(entry)
+
+		partNumber, currentEtag, actualSize, derr := fs.decodePartFile(entry)
 		if derr != nil {
 			// Skip part files whose name don't match expected format. These could be backend filesystem specific files.
 			continue
 		}
-		etag2, ok := partsMap[partNumber]
-		if !ok {
-			partsMap[partNumber] = etag1
+
+		entryStat, err := fsStatFile(ctx, pathJoin(uploadIDDir, entry))
+		if err != nil {
 			continue
 		}
-		stat1, serr := fsStatFile(ctx, pathJoin(uploadIDDir, getPartFile(entries, partNumber, etag1)))
-		if serr != nil {
-			return result, toObjectErr(serr)
+
+		currentMeta := PartInfo{
+			PartNumber:   partNumber,
+			ETag:         currentEtag,
+			ActualSize:   actualSize,
+			Size:         entryStat.Size(),
+			LastModified: entryStat.ModTime(),
 		}
-		stat2, serr := fsStatFile(ctx, pathJoin(uploadIDDir, getPartFile(entries, partNumber, etag2)))
-		if serr != nil {
-			return result, toObjectErr(serr)
+
+		cachedMeta, ok := partsMap[partNumber]
+		if !ok {
+			partsMap[partNumber] = currentMeta
+			continue
 		}
-		if stat1.ModTime().After(stat2.ModTime()) {
-			partsMap[partNumber] = etag1
+
+		if currentMeta.LastModified.After(cachedMeta.LastModified) {
+			partsMap[partNumber] = currentMeta
 		}
 	}
 
 	var parts []PartInfo
-	var actualSize int64
-	for partNumber, etag := range partsMap {
-		partFile := getPartFile(entries, partNumber, etag)
-		if partFile == "" {
-			return result, InvalidPart{}
-		}
-		// Read the actualSize from the pathFileName.
-		subParts := strings.Split(partFile, ".")
-		actualSize, err = strconv.ParseInt(subParts[len(subParts)-1], 10, 64)
-		if err != nil {
-			return result, InvalidPart{}
-		}
-		parts = append(parts, PartInfo{PartNumber: partNumber, ETag: etag, ActualSize: actualSize})
+	for _, partInfo := range partsMap {
+		parts = append(parts, partInfo)
 	}
+
 	sort.Slice(parts, func(i int, j int) bool {
 		return parts[i].PartNumber < parts[j].PartNumber
 	})
+
 	i := 0
 	if partNumberMarker != 0 {
 		// If the marker was set, skip the entries till the marker.
@@ -515,21 +516,19 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 			result.NextPartNumberMarker = result.Parts[partsCount-1].PartNumber
 		}
 	}
-	for i, part := range result.Parts {
-		var stat os.FileInfo
-		stat, err = fsStatFile(ctx, pathJoin(uploadIDDir,
-			fs.encodePartFile(part.PartNumber, part.ETag, part.ActualSize)))
-		if err != nil {
-			return result, toObjectErr(err)
-		}
-		result.Parts[i].LastModified = stat.ModTime()
-		result.Parts[i].Size = part.ActualSize
-	}
 
-	fsMetaBytes, err := ioutil.ReadFile(pathJoin(uploadIDDir, fs.metaJSONFile))
+	rc, _, err := fsOpenFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile), 0)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return result, err
+		if err == errFileNotFound || err == errFileAccessDenied {
+			return result, InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID}
+		}
+		return result, toObjectErr(err, bucket, object)
+	}
+	defer rc.Close()
+
+	fsMetaBytes, err := ioutil.ReadAll(rc)
+	if err != nil {
+		return result, toObjectErr(err, bucket, object)
 	}
 
 	var fsMeta fsMetaV1
@@ -579,7 +578,10 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	// Calculate s3 compatible md5sum for complete multipart.
 	s3MD5 := getCompleteMultipartMD5(parts)
 
-	partSize := int64(-1) // Used later to ensure that all parts sizes are same.
+	// ensure that part ETag is canonicalized to strip off extraneous quotes
+	for i := range parts {
+		parts[i].ETag = canonicalizeETag(parts[i].ETag)
+	}
 
 	fsMeta := fsMetaV1{}
 
@@ -593,16 +595,17 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		return oi, err
 	}
 
-	// ensure that part ETag is canonicalized to strip off extraneous quotes
-	for i := range parts {
-		parts[i].ETag = canonicalizeETag(parts[i].ETag)
+	// Create entries trie structure for prefix match
+	entriesTrie := trie.NewTrie()
+	for _, entry := range entries {
+		entriesTrie.Insert(entry)
 	}
 
 	// Save consolidated actual size.
 	var objectActualSize int64
 	// Validate all parts and then commit to disk.
 	for i, part := range parts {
-		partFile := getPartFile(entries, part.PartNumber, part.ETag)
+		partFile := getPartFile(entriesTrie, part.PartNumber, part.ETag)
 		if partFile == "" {
 			return oi, InvalidPart{
 				PartNumber: part.PartNumber,
@@ -630,9 +633,6 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 			}
 			return oi, err
 		}
-		if partSize == -1 {
-			partSize = actualSize
-		}
 
 		fsMeta.Parts[i] = ObjectPartInfo{
 			Number:     part.PartNumber,
@@ -697,8 +697,16 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 			fsRemoveFile(ctx, file.filePath)
 		}
 		for _, part := range parts {
-			partPath := getPartFile(entries, part.PartNumber, part.ETag)
-			if err = mioutil.AppendFile(appendFilePath, pathJoin(uploadIDDir, partPath), globalFSOSync); err != nil {
+			partFile := getPartFile(entriesTrie, part.PartNumber, part.ETag)
+			if partFile == "" {
+				logger.LogIf(ctx, fmt.Errorf("%.5d.%s missing will not proceed",
+					part.PartNumber, part.ETag))
+				return oi, InvalidPart{
+					PartNumber: part.PartNumber,
+					GotETag:    part.ETag,
+				}
+			}
+			if err = mioutil.AppendFile(appendFilePath, pathJoin(uploadIDDir, partFile), globalFSOSync); err != nil {
 				logger.LogIf(ctx, err)
 				return oi, toObjectErr(err)
 			}
@@ -711,13 +719,35 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		return oi, err
 	}
 	defer destLock.Unlock()
-	fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)
-	metaFile, err := fs.rwPool.Create(fsMetaPath)
+
+	bucketMetaDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix)
+	fsMetaPath := pathJoin(bucketMetaDir, bucket, object, fs.metaJSONFile)
+	metaFile, err := fs.rwPool.Write(fsMetaPath)
+	var freshFile bool
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return oi, toObjectErr(err, bucket, object)
+		if !errors.Is(err, errFileNotFound) {
+			logger.LogIf(ctx, err)
+			return oi, toObjectErr(err, bucket, object)
+		}
+		metaFile, err = fs.rwPool.Create(fsMetaPath)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return oi, toObjectErr(err, bucket, object)
+		}
+		freshFile = true
 	}
 	defer metaFile.Close()
+	defer func() {
+		// Remove meta file when CompleteMultipart encounters
+		// any error and it is a fresh file.
+		//
+		// We should preserve the `fs.json` of any
+		// existing object
+		if e != nil && freshFile {
+			tmpDir := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID)
+			fsRemoveMeta(ctx, bucketMetaDir, fsMetaPath, tmpDir)
+		}
+	}()
 
 	// Read saved fs metadata for ongoing multipart.
 	fsMetaBuf, err := ioutil.ReadFile(pathJoin(uploadIDDir, fs.metaJSONFile))

cmd/fs-v1-rwpool.go

@@ -157,6 +157,9 @@ func (fsi *fsIOPool) Write(path string) (wlk *lock.LockedFile, err error) {
 		case isSysErrIsDir(err):
 			return nil, errIsNotRegular
 		default:
+			if isSysErrPathNotFound(err) {
+				return nil, errFileNotFound
+			}
 			return nil, err
 		}
 	}

cmd/fs-v1.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -217,12 +216,11 @@ func (fs *FSObjects) StorageInfo(ctx context.Context, _ bool) (StorageInfo, []er
 	if err != nil {
 		return StorageInfo{}, []error{err}
 	}
-	used := di.Total - di.Free
 	storageInfo := StorageInfo{
 		Disks: []madmin.Disk{
 			{
 				TotalSpace:     di.Total,
-				UsedSpace:      used,
+				UsedSpace:      di.Used,
 				AvailableSpace: di.Free,
 				DrivePath:      fs.fsPath,
 			},
@@ -623,10 +621,6 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 		fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, srcBucket, srcObject, fs.metaJSONFile)
 		wlk, err := fs.rwPool.Write(fsMetaPath)
 		if err != nil {
-			if !errors.Is(err, errFileNotFound) {
-				logger.LogIf(ctx, err)
-				return oi, toObjectErr(err, srcBucket, srcObject)
-			}
 			wlk, err = fs.rwPool.Create(fsMetaPath)
 			if err != nil {
 				logger.LogIf(ctx, err)
@@ -659,7 +653,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 		return fsMeta.ToObjectInfo(srcBucket, srcObject, fi), nil
 	}
 
-	if err := checkPutObjectArgs(ctx, dstBucket, dstObject, fs, srcInfo.PutObjReader.Size()); err != nil {
+	if err := checkPutObjectArgs(ctx, dstBucket, dstObject, fs); err != nil {
 		return ObjectInfo{}, err
 	}
 
@@ -1103,7 +1097,7 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
 		return objInfo, NotImplemented{}
 	}
 
-	if err := checkPutObjectArgs(ctx, bucket, object, fs, r.Size()); err != nil {
+	if err := checkPutObjectArgs(ctx, bucket, object, fs); err != nil {
 		return ObjectInfo{}, err
 	}
 
@@ -1173,18 +1167,26 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 	var wlk *lock.LockedFile
 	if bucket != minioMetaBucket {
 		bucketMetaDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix)
-
 		fsMetaPath := pathJoin(bucketMetaDir, bucket, object, fs.metaJSONFile)
-		wlk, err = fs.rwPool.Create(fsMetaPath)
+		wlk, err = fs.rwPool.Write(fsMetaPath)
+		var freshFile bool
 		if err != nil {
-			logger.LogIf(ctx, err)
-			return ObjectInfo{}, toObjectErr(err, bucket, object)
+			wlk, err = fs.rwPool.Create(fsMetaPath)
+			if err != nil {
+				logger.LogIf(ctx, err)
+				return ObjectInfo{}, toObjectErr(err, bucket, object)
+			}
+			freshFile = true
 		}
 		// This close will allow for locks to be synchronized on `fs.json`.
 		defer wlk.Close()
 		defer func() {
-			// Remove meta file when PutObject encounters any error
-			if retErr != nil {
+			// Remove meta file when PutObject encounters
+			// any error and it is a fresh file.
+			//
+			// We should preserve the `fs.json` of any
+			// existing object
+			if retErr != nil && freshFile {
 				tmpDir := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID)
 				fsRemoveMeta(ctx, bucketMetaDir, fsMetaPath, tmpDir)
 			}
@@ -1499,10 +1501,6 @@ func (fs *FSObjects) PutObjectTags(ctx context.Context, bucket, object string, t
 	fsMeta := fsMetaV1{}
 	wlk, err := fs.rwPool.Write(fsMetaPath)
 	if err != nil {
-		if !errors.Is(err, errFileNotFound) {
-			logger.LogIf(ctx, err)
-			return toObjectErr(err, bucket, object)
-		}
 		wlk, err = fs.rwPool.Create(fsMetaPath)
 		if err != nil {
 			logger.LogIf(ctx, err)

cmd/gateway-main.go

@@ -300,7 +300,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	// Once endpoints are finalized, initialize the new object api in safe mode.
 	globalObjLayerMutex.Lock()
-	globalSafeMode = true
 	globalObjectAPI = newObject
 	globalObjLayerMutex.Unlock()
 
@@ -325,7 +324,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	if enableIAMOps {
 		// Initialize IAM sys.
-		startBackgroundIAMLoad(GlobalContext)
+		startBackgroundIAMLoad(GlobalContext, newObject)
 	}
 
 	if globalCacheConfig.Enabled {
@@ -353,11 +352,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// - compression
 	verifyObjectLayerFeatures("gateway "+gatewayName, newObject)
 
-	// Disable safe mode operation, after all initialization is over.
-	globalObjLayerMutex.Lock()
-	globalSafeMode = false
-	globalObjLayerMutex.Unlock()
-
 	// Prints the formatted startup message once object layer is initialized.
 	if !globalCLIContext.Quiet {
 		mode := globalMinioModeGatewayPrefix + gatewayName

cmd/global-heal.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"sync"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
@@ -26,17 +27,11 @@ import (
 
 const (
 	bgHealingUUID = "0000-0000-0000-0000"
-	// sleep for an hour after a lock timeout
-	// before retrying to acquire lock again.
-	leaderLockTimeoutSleepInterval = time.Hour
 )
 
-var leaderLockTimeout = newDynamicTimeout(30*time.Second, time.Minute)
-
 // NewBgHealSequence creates a background healing sequence
 // operation which crawls all objects and heal them.
 func newBgHealSequence() *healSequence {
-
 	reqInfo := &logger.ReqInfo{API: "BackgroundHeal"}
 	ctx, cancelCtx := context.WithCancel(logger.SetReqInfo(GlobalContext, reqInfo))
 
@@ -71,19 +66,26 @@ func getLocalBackgroundHealStatus() (madmin.BgHealState, bool) {
 	if globalBackgroundHealState == nil {
 		return madmin.BgHealState{}, false
 	}
+
 	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
 	if !ok {
 		return madmin.BgHealState{}, false
 	}
 
-	objAPI := newObjectLayerWithoutSafeModeFn()
-	if objAPI == nil {
-		return madmin.BgHealState{}, false
+	var healDisksMap = map[string]struct{}{}
+	for _, ep := range getLocalDisksToHeal() {
+		healDisksMap[ep.String()] = struct{}{}
+	}
+
+	for _, ep := range globalBackgroundHealState.getHealLocalDisks() {
+		if _, ok := healDisksMap[ep.String()]; !ok {
+			healDisksMap[ep.String()] = struct{}{}
+		}
 	}
 
 	var healDisks []string
-	for _, ep := range getLocalDisksToHeal() {
-		healDisks = append(healDisks, ep.String())
+	for disk := range healDisksMap {
+		healDisks = append(healDisks, disk)
 	}
 
 	return madmin.BgHealState{
@@ -95,12 +97,7 @@ func getLocalBackgroundHealStatus() (madmin.BgHealState, bool) {
 }
 
 // healErasureSet lists and heals all objects in a specific erasure set
-func healErasureSet(ctx context.Context, setIndex int, xlObj *erasureObjects, setDriveCount int) error {
-	buckets, err := xlObj.ListBuckets(ctx)
-	if err != nil {
-		return err
-	}
-
+func healErasureSet(ctx context.Context, setIndex int, buckets []BucketInfo, disks []StorageAPI) error {
 	// Get background heal sequence to send elements to heal
 	var bgSeq *healSequence
 	var ok bool
@@ -131,37 +128,36 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *erasureObjects, se
 		}
 
 		var entryChs []FileInfoVersionsCh
-		for _, disk := range xlObj.getLoadBalancedDisks() {
-			if disk == nil {
-				// Disk can be offline
-				continue
-			}
-
-			entryCh, err := disk.WalkVersions(ctx, bucket.Name, "", "", true, ctx.Done())
-			if err != nil {
-				// Disk walk returned error, ignore it.
-				continue
-			}
-
-			entryChs = append(entryChs, FileInfoVersionsCh{
-				Ch: entryCh,
-			})
+		var mu sync.Mutex
+		var wg sync.WaitGroup
+		for _, disk := range disks {
+			disk := disk
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				entryCh, err := disk.WalkVersions(ctx, bucket.Name, "", "", true, ctx.Done())
+				if err != nil {
+					// Disk walk returned error, ignore it.
+					return
+				}
+				mu.Lock()
+				entryChs = append(entryChs, FileInfoVersionsCh{
+					Ch: entryCh,
+				})
+				mu.Unlock()
+			}()
 		}
+		wg.Wait()
 
 		entriesValid := make([]bool, len(entryChs))
 		entries := make([]FileInfoVersions, len(entryChs))
 
 		for {
-			entry, quorumCount, ok := lexicallySortedEntryVersions(entryChs, entries, entriesValid)
+			entry, _, ok := lexicallySortedEntryVersions(entryChs, entries, entriesValid)
 			if !ok {
 				break
 			}
 
-			if quorumCount == setDriveCount {
-				// Skip good entries.
-				continue
-			}
-
 			for _, version := range entry.Versions {
 				bgSeq.sourceCh <- healSource{
 					bucket:    bucket.Name,

cmd/globals.go

@@ -63,6 +63,8 @@ const (
 	globalMinioModeErasure         = "mode-server-xl"
 	globalMinioModeDistErasure     = "mode-server-distributed-xl"
 	globalMinioModeGatewayPrefix   = "mode-gateway-"
+	globalDirSuffix                = "__XLDIR__"
+	globalDirSuffixWithSlash       = globalDirSuffix + slashSeparator
 
 	// Add new global values here.
 )
@@ -208,7 +210,8 @@ var (
 	globalDomainNames []string      // Root domains for virtual host style requests
 	globalDomainIPs   set.StringSet // Root domain IP address(s) for a distributed MinIO deployment
 
-	globalOperationTimeout = newDynamicTimeout(10*time.Minute, 5*time.Minute) // default timeout for general ops
+	globalOperationTimeout       = newDynamicTimeout(10*time.Minute, 5*time.Minute) // default timeout for general ops
+	globalDeleteOperationTimeout = newDynamicTimeout(5*time.Minute, 1*time.Minute)  // default time for delete ops
 
 	globalBucketObjectLockSys *BucketObjectLockSys
 	globalBucketQuotaSys      *BucketQuotaSys
@@ -265,11 +268,6 @@ var (
 	globalBackgroundHealRoutine *healRoutine
 	globalBackgroundHealState   *allHealState
 
-	// Only enabled when one of the sub-systems fail
-	// to initialize, this allows for administrators to
-	// fix the system.
-	globalSafeMode bool
-
 	// If writes to FS backend should be O_SYNC.
 	globalFSOSync bool
 

cmd/handler-api.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/config/api"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/sys"
 )
 
@@ -30,7 +31,7 @@ type apiConfig struct {
 
 	requestsDeadline time.Duration
 	requestsPool     chan struct{}
-	readyDeadline    time.Duration
+	clusterDeadline  time.Duration
 	corsAllowOrigins []string
 }
 
@@ -38,14 +39,16 @@ func (t *apiConfig) init(cfg api.Config, setDriveCount int) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
-	t.readyDeadline = cfg.ReadyDeadline
+	t.clusterDeadline = cfg.ClusterDeadline
 	t.corsAllowOrigins = cfg.CorsAllowOrigin
 
 	var apiRequestsMaxPerNode int
 	if cfg.RequestsMax <= 0 {
 		stats, err := sys.GetStats()
 		if err != nil {
-			return
+			logger.LogIf(GlobalContext, err)
+			// Default to 16 GiB, not critical.
+			stats.TotalRAM = 16 << 30
 		}
 		// max requests per node is calculated as
 		// total_ram / ram_per_request
@@ -71,15 +74,15 @@ func (t *apiConfig) getCorsAllowOrigins() []string {
 	return corsAllowOrigins
 }
 
-func (t *apiConfig) getReadyDeadline() time.Duration {
+func (t *apiConfig) getClusterDeadline() time.Duration {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 
-	if t.readyDeadline == 0 {
+	if t.clusterDeadline == 0 {
 		return 10 * time.Second
 	}
 
-	return t.readyDeadline
+	return t.clusterDeadline
 }
 
 func (t *apiConfig) getRequestsPool() (chan struct{}, <-chan time.Time) {

cmd/handler-utils.go

@@ -238,7 +238,9 @@ func extractReqParams(r *http.Request) map[string]string {
 
 // Extract response elements to be sent with event notifiation.
 func extractRespElements(w http.ResponseWriter) map[string]string {
-
+	if w == nil {
+		return map[string]string{}
+	}
 	return map[string]string{
 		"requestId":      w.Header().Get(xhttp.AmzRequestID),
 		"content-length": w.Header().Get(xhttp.ContentLength),

cmd/healthcheck-handler.go

@@ -33,7 +33,7 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getReadyDeadline())
+	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getClusterDeadline())
 	defer cancel()
 
 	opts := HealthOptions{Maintenance: r.URL.Query().Get("maintenance") == "true"}

cmd/http-tracer.go

@@ -121,11 +121,6 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 		t.NodeName = host
 	}
 
-	rw := logger.NewResponseWriter(w)
-	rw.LogErrBody = true
-	rw.LogAllBody = logBody
-	f(rw, r)
-
 	rq := trace.RequestInfo{
 		Time:     time.Now().UTC(),
 		Proto:    r.Proto,
@@ -134,8 +129,15 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 		RawQuery: r.URL.RawQuery,
 		Client:   handlers.GetSourceIP(r),
 		Headers:  reqHeaders,
-		Body:     reqBodyRecorder.Data(),
 	}
+
+	rw := logger.NewResponseWriter(w)
+	rw.LogErrBody = true
+	rw.LogAllBody = logBody
+
+	// Execute call.
+	f(rw, r)
+
 	rs := trace.ResponseInfo{
 		Time:       time.Now().UTC(),
 		Headers:    rw.Header().Clone(),
@@ -143,6 +145,9 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 		Body:       rw.Body(),
 	}
 
+	// Transfer request body
+	rq.Body = reqBodyRecorder.Data()
+
 	if rs.StatusCode == 0 {
 		rs.StatusCode = http.StatusOK
 	}

cmd/http/server.go

@@ -130,6 +130,9 @@ func (srv *Server) Shutdown() error {
 	srv.listenerMutex.Lock()
 	err := srv.listener.Close()
 	srv.listenerMutex.Unlock()
+	if err != nil {
+		return err
+	}
 
 	// Wait for opened connection to be closed up to Shutdown timeout.
 	shutdownTimeout := srv.ShutdownTimeout
@@ -144,12 +147,12 @@ func (srv *Server) Shutdown() error {
 			if err == nil {
 				_ = pprof.Lookup("goroutine").WriteTo(tmp, 1)
 				tmp.Close()
-				return errors.New("timed out. some connections are still active. doing abnormal shutdown. goroutines written to " + tmp.Name())
+				return errors.New("timed out. some connections are still active. goroutines written to " + tmp.Name())
 			}
-			return errors.New("timed out. some connections are still active. doing abnormal shutdown")
+			return errors.New("timed out. some connections are still active")
 		case <-ticker.C:
 			if atomic.LoadInt32(&srv.requestCount) <= 0 {
-				return err
+				return nil
 			}
 		}
 	}

cmd/iam.go

@@ -411,8 +411,8 @@ func (sys *IAMSys) doIAMConfigMigration(ctx context.Context) error {
 // Loads IAM users and policies in background, any un-handled
 // error means this code can potentially crash the server
 // in such a situation manual intervention is necessary.
-func startBackgroundIAMLoad(ctx context.Context) {
-	go globalIAMSys.Init(ctx, newObjectLayerWithoutSafeModeFn())
+func startBackgroundIAMLoad(ctx context.Context, objAPI ObjectLayer) {
+	go globalIAMSys.Init(ctx, objAPI)
 }
 
 // Init - initializes config system by reading entries from config/iam
@@ -450,7 +450,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 	wquorum := InsufficientWriteQuorum{}
 
 	// allocate dynamic timeout once before the loop
-	iamLockTimeout := newDynamicTimeout(3*time.Second, 5*time.Second)
+	iamLockTimeout := newDynamicTimeout(5*time.Second, 3*time.Second)
 
 	for range retry.NewTimerWithJitter(retryCtx, time.Second, 5*time.Second, retry.MaxJitter) {
 		// let one of the server acquire the lock, if not let them timeout.
@@ -514,7 +514,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 
 // DeletePolicy - deletes a canned policy from backend or etcd.
 func (sys *IAMSys) DeletePolicy(policyName string) error {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
@@ -567,7 +567,7 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 
 // InfoPolicy - expands the canned policy into its JSON structure.
 func (sys *IAMSys) InfoPolicy(policyName string) (iampolicy.Policy, error) {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return iampolicy.Policy{}, errServerNotInitialized
 	}
@@ -585,7 +585,7 @@ func (sys *IAMSys) InfoPolicy(policyName string) (iampolicy.Policy, error) {
 
 // ListPolicies - lists all canned policies.
 func (sys *IAMSys) ListPolicies() (map[string]iampolicy.Policy, error) {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return nil, errServerNotInitialized
 	}
@@ -607,7 +607,7 @@ func (sys *IAMSys) ListPolicies() (map[string]iampolicy.Policy, error) {
 
 // SetPolicy - sets a new name policy.
 func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
@@ -629,7 +629,7 @@ func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error {
 
 // DeleteUser - delete user (only for long-term users not STS users).
 func (sys *IAMSys) DeleteUser(accessKey string) error {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
@@ -699,7 +699,7 @@ func (sys *IAMSys) currentPolicies(policyName string) string {
 
 // SetTempUser - set temporary user credentials, these credentials have an expiry.
 func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyName string) error {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
@@ -749,7 +749,7 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
 
 // ListUsers - list all users.
 func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return nil, errServerNotInitialized
 	}
@@ -786,7 +786,7 @@ func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
 
 // IsTempUser - returns if given key is a temporary user.
 func (sys *IAMSys) IsTempUser(name string) (bool, error) {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return false, errServerNotInitialized
 	}
@@ -804,7 +804,7 @@ func (sys *IAMSys) IsTempUser(name string) (bool, error) {
 
 // IsServiceAccount - returns if given key is a service account
 func (sys *IAMSys) IsServiceAccount(name string) (bool, string, error) {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return false, "", errServerNotInitialized
 	}
@@ -826,7 +826,7 @@ func (sys *IAMSys) IsServiceAccount(name string) (bool, string, error) {
 
 // GetUserInfo - get info on a user.
 func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return u, errServerNotInitialized
 	}
@@ -872,7 +872,7 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 
 // SetUserStatus - sets current user status, supports disabled or enabled.
 func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus) error {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
@@ -918,7 +918,7 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 
 // NewServiceAccount - create a new service account
 func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, sessionPolicy *iampolicy.Policy) (auth.Credentials, error) {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return auth.Credentials{}, errServerNotInitialized
 	}
@@ -985,7 +985,7 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, ses
 
 // ListServiceAccounts - lists all services accounts associated to a specific user
 func (sys *IAMSys) ListServiceAccounts(ctx context.Context, accessKey string) ([]string, error) {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return nil, errServerNotInitialized
 	}
@@ -1010,7 +1010,7 @@ func (sys *IAMSys) ListServiceAccounts(ctx context.Context, accessKey string) ([
 
 // GetServiceAccountParent - gets information about a service account
 func (sys *IAMSys) GetServiceAccountParent(ctx context.Context, accessKey string) (string, error) {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return "", errServerNotInitialized
 	}
@@ -1027,7 +1027,7 @@ func (sys *IAMSys) GetServiceAccountParent(ctx context.Context, accessKey string
 
 // DeleteServiceAccount - delete a service account
 func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) error {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
@@ -1056,7 +1056,7 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) e
 
 // SetUser - set user credentials and policy.
 func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
@@ -1094,7 +1094,7 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 
 // SetUserSecretKey - sets user secret key
 func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
@@ -1123,7 +1123,7 @@ func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
 
 // GetUser - get user credentials
 func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return cred, false
 	}
@@ -1187,7 +1187,7 @@ func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 // AddUsersToGroup - adds users to a group, creating the group if
 // needed. No error if user(s) already are in the group.
 func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
@@ -1248,7 +1248,7 @@ func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
 // RemoveUsersFromGroup - remove users from group. If no users are
 // given, and the group is empty, deletes the group as well.
 func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
@@ -1329,7 +1329,7 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 
 // SetGroupStatus - enable/disabled a group
 func (sys *IAMSys) SetGroupStatus(group string, enabled bool) error {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
@@ -1365,7 +1365,7 @@ func (sys *IAMSys) SetGroupStatus(group string, enabled bool) error {
 
 // GetGroupDescription - builds up group description
 func (sys *IAMSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err error) {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return gd, errServerNotInitialized
 	}
@@ -1406,7 +1406,7 @@ func (sys *IAMSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err e
 
 // ListGroups - lists groups.
 func (sys *IAMSys) ListGroups() (r []string, err error) {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return r, errServerNotInitialized
 	}
@@ -1430,7 +1430,7 @@ func (sys *IAMSys) ListGroups() (r []string, err error) {
 
 // PolicyDBSet - sets a policy for a user or group in the PolicyDB.
 func (sys *IAMSys) PolicyDBSet(name, policy string, isGroup bool) error {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return errServerNotInitialized
 	}
@@ -1497,7 +1497,7 @@ func (sys *IAMSys) policyDBSet(name, policyName string, userType IAMUserType, is
 // be a member of multiple groups, this function returns an array of
 // applicable policies (each group is mapped to at most one policy).
 func (sys *IAMSys) PolicyDBGet(name string, isGroup bool) ([]string, error) {
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || sys == nil || sys.store == nil {
 		return nil, errServerNotInitialized
 	}

cmd/local-locker.go

@@ -32,6 +32,9 @@ type lockRequesterInfo struct {
 	Timestamp     time.Time // Timestamp set at the time of initialization.
 	TimeLastCheck time.Time // Timestamp for last check of validity of lock.
 	Source        string    // Contains line, function and filename reqesting the lock.
+	// Owner represents the UUID of the owner who originally requested the lock
+	// useful in expiry.
+	Owner string
 }
 
 // isWriteLock returns whether the lock is a write or read lock.
@@ -73,34 +76,30 @@ func (l *localLocker) canTakeLock(resources ...string) bool {
 }
 
 func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
-	select {
-	case <-ctx.Done():
-		return false, ctx.Err()
-	default:
-		l.mutex.Lock()
-		defer l.mutex.Unlock()
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
 
-		if !l.canTakeLock(args.Resources...) {
-			// Not all locks can be taken on resources,
-			// reject it completely.
-			return false, nil
-		}
-
-		// No locks held on the all resources, so claim write
-		// lock on all resources at once.
-		for _, resource := range args.Resources {
-			l.lockMap[resource] = []lockRequesterInfo{
-				{
-					Writer:        true,
-					Source:        args.Source,
-					UID:           args.UID,
-					Timestamp:     UTCNow(),
-					TimeLastCheck: UTCNow(),
-				},
-			}
-		}
-		return true, nil
+	if !l.canTakeLock(args.Resources...) {
+		// Not all locks can be taken on resources,
+		// reject it completely.
+		return false, nil
 	}
+
+	// No locks held on the all resources, so claim write
+	// lock on all resources at once.
+	for _, resource := range args.Resources {
+		l.lockMap[resource] = []lockRequesterInfo{
+			{
+				Writer:        true,
+				Source:        args.Source,
+				Owner:         args.Owner,
+				UID:           args.UID,
+				Timestamp:     UTCNow(),
+				TimeLastCheck: UTCNow(),
+			},
+		}
+	}
+	return true, nil
 }
 
 func (l *localLocker) Unlock(args dsync.LockArgs) (reply bool, err error) {
@@ -113,7 +112,7 @@ func (l *localLocker) Unlock(args dsync.LockArgs) (reply bool, err error) {
 	}
 	for _, resource := range args.Resources {
 		lri := l.lockMap[resource]
-		if !l.removeEntry(resource, args.UID, &lri) {
+		if !l.removeEntry(resource, args, &lri) {
 			return false, fmt.Errorf("Unlock unable to find corresponding lock for uid: %s on resource %s", args.UID, resource)
 		}
 	}
@@ -124,10 +123,10 @@ func (l *localLocker) Unlock(args dsync.LockArgs) (reply bool, err error) {
 // removeEntry based on the uid of the lock message, removes a single entry from the
 // lockRequesterInfo array or the whole array from the map (in case of a write lock
 // or last read lock)
-func (l *localLocker) removeEntry(name, uid string, lri *[]lockRequesterInfo) bool {
+func (l *localLocker) removeEntry(name string, args dsync.LockArgs, lri *[]lockRequesterInfo) bool {
 	// Find correct entry to remove based on uid.
 	for index, entry := range *lri {
-		if entry.UID == uid {
+		if entry.UID == args.UID && entry.Owner == args.Owner {
 			if len(*lri) == 1 {
 				// Remove the write lock.
 				delete(l.lockMap, name)
@@ -145,32 +144,28 @@ func (l *localLocker) removeEntry(name, uid string, lri *[]lockRequesterInfo) bo
 }
 
 func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) {
-	select {
-	case <-ctx.Done():
-		return false, ctx.Err()
-	default:
-		l.mutex.Lock()
-		defer l.mutex.Unlock()
-		lrInfo := lockRequesterInfo{
-			Writer:        false,
-			Source:        args.Source,
-			UID:           args.UID,
-			Timestamp:     UTCNow(),
-			TimeLastCheck: UTCNow(),
-		}
-		resource := args.Resources[0]
-		if lri, ok := l.lockMap[resource]; ok {
-			if reply = !isWriteLock(lri); reply {
-				// Unless there is a write lock
-				l.lockMap[resource] = append(l.lockMap[resource], lrInfo)
-			}
-		} else {
-			// No locks held on the given name, so claim (first) read lock
-			l.lockMap[resource] = []lockRequesterInfo{lrInfo}
-			reply = true
-		}
-		return reply, nil
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+	lrInfo := lockRequesterInfo{
+		Writer:        false,
+		Source:        args.Source,
+		Owner:         args.Owner,
+		UID:           args.UID,
+		Timestamp:     UTCNow(),
+		TimeLastCheck: UTCNow(),
 	}
+	resource := args.Resources[0]
+	if lri, ok := l.lockMap[resource]; ok {
+		if reply = !isWriteLock(lri); reply {
+			// Unless there is a write lock
+			l.lockMap[resource] = append(l.lockMap[resource], lrInfo)
+		}
+	} else {
+		// No locks held on the given name, so claim (first) read lock
+		l.lockMap[resource] = []lockRequesterInfo{lrInfo}
+		reply = true
+	}
+	return reply, nil
 }
 
 func (l *localLocker) RUnlock(args dsync.LockArgs) (reply bool, err error) {
@@ -187,7 +182,7 @@ func (l *localLocker) RUnlock(args dsync.LockArgs) (reply bool, err error) {
 		// A write-lock is held, cannot release a read lock
 		return reply, fmt.Errorf("RUnlock attempted on a write locked entity: %s", resource)
 	}
-	if !l.removeEntry(resource, args.UID, &lri) {
+	if !l.removeEntry(resource, args, &lri) {
 		return false, fmt.Errorf("RUnlock unable to find corresponding read lock for uid: %s", args.UID)
 	}
 	return reply, nil
@@ -208,11 +203,16 @@ func (l *localLocker) Close() error {
 	return nil
 }
 
-// Local locker is always online.
+// IsOnline - local locker is always online.
 func (l *localLocker) IsOnline() bool {
 	return true
 }
 
+// IsLocal - local locker returns true.
+func (l *localLocker) IsLocal() bool {
+	return true
+}
+
 func (l *localLocker) Expired(ctx context.Context, args dsync.LockArgs) (expired bool, err error) {
 	select {
 	case <-ctx.Done():
@@ -226,7 +226,7 @@ func (l *localLocker) Expired(ctx context.Context, args dsync.LockArgs) (expired
 			if lri, ok := l.lockMap[resource]; ok {
 				// Check whether uid is still active
 				for _, entry := range lri {
-					if entry.UID == args.UID {
+					if entry.UID == args.UID && entry.Owner == args.Owner {
 						return false, nil
 					}
 				}
@@ -244,7 +244,7 @@ func (l *localLocker) removeEntryIfExists(nlrip nameLockRequesterInfoPair) {
 		// Even if the entry exists, it may not be the same entry which was
 		// considered as expired, so we simply an attempt to remove it if its
 		// not possible there is nothing we need to do.
-		l.removeEntry(nlrip.name, nlrip.lri.UID, &lri)
+		l.removeEntry(nlrip.name, dsync.LockArgs{Owner: nlrip.lri.Owner, UID: nlrip.lri.UID}, &lri)
 	}
 }
 

cmd/lock-rest-client.go

@@ -76,6 +76,11 @@ func (client *lockRESTClient) IsOnline() bool {
 	return client.restClient.IsOnline()
 }
 
+// Not a local locker
+func (client *lockRESTClient) IsLocal() bool {
+	return false
+}
+
 // Close - marks the client as closed.
 func (client *lockRESTClient) Close() error {
 	client.restClient.Close()
@@ -86,6 +91,7 @@ func (client *lockRESTClient) Close() error {
 func (client *lockRESTClient) restCall(ctx context.Context, call string, args dsync.LockArgs) (reply bool, err error) {
 	values := url.Values{}
 	values.Set(lockRESTUID, args.UID)
+	values.Set(lockRESTOwner, args.Owner)
 	values.Set(lockRESTSource, args.Source)
 	var buffer bytes.Buffer
 	for _, resource := range args.Resources {
@@ -152,7 +158,7 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
 		}
 	}
 
-	trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
+	trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultTimeout)
 	restClient := rest.NewClient(serverURL, trFn, newAuthToken)
 	restClient.HealthCheckFn = func() bool {
 		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)

cmd/lock-rest-server-common.go

@@ -34,14 +34,19 @@ const (
 	lockRESTMethodRUnlock = "/runlock"
 	lockRESTMethodExpired = "/expired"
 
+	// lockRESTOwner represents owner UUID
+	lockRESTOwner = "owner"
+
 	// Unique ID of lock/unlock request.
 	lockRESTUID = "uid"
+
 	// Source contains the line number, function and file name of the code
 	// on the client node that requested the lock.
 	lockRESTSource = "source"
 )
 
 var (
-	errLockConflict   = errors.New("lock conflict")
-	errLockNotExpired = errors.New("lock not expired")
+	errLockConflict       = errors.New("lock conflict")
+	errLockNotExpired     = errors.New("lock not expired")
+	errLockNotInitialized = errors.New("lock not initialized")
 )

cmd/lock-rest-server-common_test.go

@@ -21,6 +21,8 @@ import (
 	"reflect"
 	"sync"
 	"testing"
+
+	"github.com/minio/minio/pkg/dsync"
 )
 
 // Helper function to create a lock server for testing
@@ -53,12 +55,14 @@ func TestLockRpcServerRemoveEntry(t *testing.T) {
 	defer os.RemoveAll(testPath)
 
 	lockRequesterInfo1 := lockRequesterInfo{
+		Owner:         "owner",
 		Writer:        true,
 		UID:           "0123-4567",
 		Timestamp:     UTCNow(),
 		TimeLastCheck: UTCNow(),
 	}
 	lockRequesterInfo2 := lockRequesterInfo{
+		Owner:         "owner",
 		Writer:        true,
 		UID:           "89ab-cdef",
 		Timestamp:     UTCNow(),
@@ -73,11 +77,17 @@ func TestLockRpcServerRemoveEntry(t *testing.T) {
 	lri := locker.ll.lockMap["name"]
 
 	// test unknown uid
-	if locker.ll.removeEntry("name", "unknown-uid", &lri) {
+	if locker.ll.removeEntry("name", dsync.LockArgs{
+		Owner: "owner",
+		UID:   "unknown-uid",
+	}, &lri) {
 		t.Errorf("Expected %#v, got %#v", false, true)
 	}
 
-	if !locker.ll.removeEntry("name", "0123-4567", &lri) {
+	if !locker.ll.removeEntry("name", dsync.LockArgs{
+		Owner: "owner",
+		UID:   "0123-4567",
+	}, &lri) {
 		t.Errorf("Expected %#v, got %#v", true, false)
 	} else {
 		gotLri := locker.ll.lockMap["name"]
@@ -87,7 +97,10 @@ func TestLockRpcServerRemoveEntry(t *testing.T) {
 		}
 	}
 
-	if !locker.ll.removeEntry("name", "89ab-cdef", &lri) {
+	if !locker.ll.removeEntry("name", dsync.LockArgs{
+		Owner: "owner",
+		UID:   "89ab-cdef",
+	}, &lri) {
 		t.Errorf("Expected %#v, got %#v", true, false)
 	} else {
 		gotLri := locker.ll.lockMap["name"]

cmd/lock-rest-server.go

@@ -32,10 +32,10 @@ import (
 
 const (
 	// Lock maintenance interval.
-	lockMaintenanceInterval = 1 * time.Minute
+	lockMaintenanceInterval = 30 * time.Second
 
 	// Lock validity check interval.
-	lockValidityCheckInterval = 2 * time.Minute
+	lockValidityCheckInterval = 30 * time.Second
 )
 
 // To abstract a node over network.
@@ -50,6 +50,11 @@ func (l *lockRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
 
 // IsValid - To authenticate and verify the time difference.
 func (l *lockRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
+	if l.ll == nil {
+		l.writeErrorResponse(w, errLockNotInitialized)
+		return false
+	}
+
 	if err := storageServerRequestValidate(r); err != nil {
 		l.writeErrorResponse(w, err)
 		return false
@@ -59,6 +64,7 @@ func (l *lockRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
 
 func getLockArgs(r *http.Request) (args dsync.LockArgs, err error) {
 	args = dsync.LockArgs{
+		Owner:  r.URL.Query().Get(lockRESTOwner),
 		UID:    r.URL.Query().Get(lockRESTUID),
 		Source: r.URL.Query().Get(lockRESTSource),
 	}
@@ -236,15 +242,33 @@ func getLongLivedLocks(interval time.Duration) map[Endpoint][]nameLockRequesterI
 //
 // We will ignore the error, and we will retry later to get a resolve on this lock
 func lockMaintenance(ctx context.Context, interval time.Duration) error {
-	objAPI := newObjectLayerWithoutSafeModeFn()
+	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return nil
 	}
 
+	type nlock struct {
+		locks  int
+		writer bool
+	}
+
+	updateNlocks := func(nlripsMap map[string]nlock, name string, writer bool) {
+		nlk, ok := nlripsMap[name]
+		if !ok {
+			nlripsMap[name] = nlock{
+				locks:  1,
+				writer: writer,
+			}
+		} else {
+			nlk.locks++
+			nlripsMap[name] = nlk
+		}
+	}
+
 	// Validate if long lived locks are indeed clean.
 	// Get list of long lived locks to check for staleness.
 	for lendpoint, nlrips := range getLongLivedLocks(interval) {
-		nlripsMap := make(map[string]int, len(nlrips))
+		nlripsMap := make(map[string]nlock, len(nlrips))
 		for _, nlrip := range nlrips {
 			// Locks are only held on first zone, make sure that
 			// we only look for ownership of locks from endpoints
@@ -252,7 +276,7 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error {
 			for _, endpoint := range globalEndpoints[0].Endpoints {
 				c := newLockAPI(endpoint)
 				if !c.IsOnline() {
-					nlripsMap[nlrip.name]++
+					updateNlocks(nlripsMap, nlrip.name, nlrip.lri.Writer)
 					continue
 				}
 
@@ -261,18 +285,19 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error {
 				// Call back to original server verify whether the lock is
 				// still active (based on name & uid)
 				expired, err := c.Expired(ctx, dsync.LockArgs{
+					Owner:     nlrip.lri.Owner,
 					UID:       nlrip.lri.UID,
 					Resources: []string{nlrip.name},
 				})
 				cancel()
 				if err != nil {
-					nlripsMap[nlrip.name]++
+					updateNlocks(nlripsMap, nlrip.name, nlrip.lri.Writer)
 					c.Close()
 					continue
 				}
 
 				if !expired {
-					nlripsMap[nlrip.name]++
+					updateNlocks(nlripsMap, nlrip.name, nlrip.lri.Writer)
 				}
 
 				// Close the connection regardless of the call response.
@@ -280,14 +305,13 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error {
 			}
 
 			// Read locks we assume quorum for be N/2 success
-			quorum := objAPI.SetDriveCount() / 2
+			quorum := getReadQuorum(objAPI.SetDriveCount())
 			if nlrip.lri.Writer {
-				// For write locks we need N/2+1 success
-				quorum = objAPI.SetDriveCount()/2 + 1
+				quorum = getWriteQuorum(objAPI.SetDriveCount())
 			}
 
 			// less than the quorum, we have locks expired.
-			if nlripsMap[nlrip.name] < quorum {
+			if nlripsMap[nlrip.name].locks < quorum {
 				// The lock is no longer active at server that originated
 				// the lock, attempt to remove the lock.
 				globalLockServers[lendpoint].mutex.Lock()
@@ -308,7 +332,7 @@ func startLockMaintenance(ctx context.Context) {
 	// no need to start the lock maintenance
 	// if ObjectAPI is not initialized.
 	for {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			time.Sleep(time.Second)
 			continue
@@ -343,7 +367,6 @@ func startLockMaintenance(ctx context.Context) {
 
 // registerLockRESTHandlers - register lock rest router.
 func registerLockRESTHandlers(router *mux.Router, endpointZones EndpointZones) {
-	queries := restQueries(lockRESTUID, lockRESTSource)
 	for _, ep := range endpointZones {
 		for _, endpoint := range ep.Endpoints {
 			if !endpoint.IsLocal {
@@ -356,11 +379,11 @@ func registerLockRESTHandlers(router *mux.Router, endpointZones EndpointZones) {
 
 			subrouter := router.PathPrefix(path.Join(lockRESTPrefix, endpoint.Path)).Subrouter()
 			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodHealth).HandlerFunc(httpTraceHdrs(lockServer.HealthHandler))
-			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodLock).HandlerFunc(httpTraceHdrs(lockServer.LockHandler)).Queries(queries...)
-			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRLock).HandlerFunc(httpTraceHdrs(lockServer.RLockHandler)).Queries(queries...)
-			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodUnlock).HandlerFunc(httpTraceHdrs(lockServer.UnlockHandler)).Queries(queries...)
-			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRUnlock).HandlerFunc(httpTraceHdrs(lockServer.RUnlockHandler)).Queries(queries...)
-			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodExpired).HandlerFunc(httpTraceAll(lockServer.ExpiredHandler)).Queries(queries...)
+			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodLock).HandlerFunc(httpTraceHdrs(lockServer.LockHandler))
+			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRLock).HandlerFunc(httpTraceHdrs(lockServer.RLockHandler))
+			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodUnlock).HandlerFunc(httpTraceHdrs(lockServer.UnlockHandler))
+			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRUnlock).HandlerFunc(httpTraceHdrs(lockServer.RUnlockHandler))
+			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodExpired).HandlerFunc(httpTraceAll(lockServer.ExpiredHandler))
 
 			globalLockServers[endpoint] = lockServer.ll
 		}

cmd/logger/audit.go

@@ -60,17 +60,16 @@ func NewResponseWriter(w http.ResponseWriter) *ResponseWriter {
 }
 
 func (lrw *ResponseWriter) Write(p []byte) (int, error) {
+	if !lrw.headersLogged {
+		// We assume the response code to be '200 OK' when WriteHeader() is not called,
+		// that way following Golang HTTP response behavior.
+		lrw.WriteHeader(http.StatusOK)
+	}
 	n, err := lrw.ResponseWriter.Write(p)
 	lrw.bytesWritten += n
 	if lrw.TimeToFirstByte == 0 {
 		lrw.TimeToFirstByte = time.Now().UTC().Sub(lrw.StartTime)
 	}
-	if !lrw.headersLogged {
-		// We assume the response code to be '200 OK' when WriteHeader() is not called,
-		// that way following Golang HTTP response behavior.
-		lrw.writeHeaders(&lrw.headers, http.StatusOK, lrw.Header())
-		lrw.headersLogged = true
-	}
 	if (lrw.LogErrBody && lrw.StatusCode >= http.StatusBadRequest) || lrw.LogAllBody {
 		// Always logging error responses.
 		lrw.body.Write(p)
@@ -107,12 +106,12 @@ func (lrw *ResponseWriter) Body() []byte {
 
 // WriteHeader - writes http status code
 func (lrw *ResponseWriter) WriteHeader(code int) {
-	lrw.StatusCode = code
 	if !lrw.headersLogged {
+		lrw.StatusCode = code
 		lrw.writeHeaders(&lrw.headers, code, lrw.ResponseWriter.Header())
 		lrw.headersLogged = true
+		lrw.ResponseWriter.WriteHeader(code)
 	}
-	lrw.ResponseWriter.WriteHeader(code)
 }
 
 // Flush - Calls the underlying Flush.

cmd/logger/target/console/console.go

@@ -37,6 +37,15 @@ func (c *Target) Validate() error {
 	return nil
 }
 
+// Endpoint returns the backend endpoint
+func (c *Target) Endpoint() string {
+	return ""
+}
+
+func (c *Target) String() string {
+	return "console"
+}
+
 // Send log message 'e' to console
 func (c *Target) Send(e interface{}, logKind string) error {
 	entry, ok := e.(log.Entry)

cmd/logger/target/http/http.go

@@ -39,6 +39,7 @@ type Target struct {
 	// Channel of log entries
 	logCh chan interface{}
 
+	name string
 	// HTTP(s) endpoint
 	endpoint string
 	// Authorization token for `endpoint`
@@ -49,6 +50,15 @@ type Target struct {
 	client    http.Client
 }
 
+// Endpoint returns the backend endpoint
+func (h *Target) Endpoint() string {
+	return h.endpoint
+}
+
+func (h *Target) String() string {
+	return h.name
+}
+
 // Validate validate the http target
 func (h *Target) Validate() error {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
@@ -145,6 +155,13 @@ func (h *Target) startHTTPLogger() {
 // Option is a function type that accepts a pointer Target
 type Option func(*Target)
 
+// WithTargetName target name
+func WithTargetName(name string) Option {
+	return func(t *Target) {
+		t.name = name
+	}
+}
+
 // WithEndpoint adds a new endpoint
 func WithEndpoint(endpoint string) Option {
 	return func(t *Target) {

cmd/logger/targets.go

@@ -20,6 +20,8 @@ package logger
 // a single log entry and Send it to the log target
 //   e.g. Send the log to a http server
 type Target interface {
+	String() string
+	Endpoint() string
 	Validate() error
 	Send(entry interface{}, errKind string) error
 }

cmd/main.go

@@ -102,21 +102,19 @@ func newApp(name string) *cli.App {
 
 	findClosestCommands := func(command string) []string {
 		var closestCommands []string
-		for _, value := range commandsTree.PrefixMatch(command) {
-			closestCommands = append(closestCommands, value.(string))
-		}
+		closestCommands = append(closestCommands, commandsTree.PrefixMatch(command)...)
 
 		sort.Strings(closestCommands)
 		// Suggest other close commands - allow missed, wrongly added and
 		// even transposed characters
 		for _, value := range commandsTree.Walk(commandsTree.Root()) {
-			if sort.SearchStrings(closestCommands, value.(string)) < len(closestCommands) {
+			if sort.SearchStrings(closestCommands, value) < len(closestCommands) {
 				continue
 			}
 			// 2 is arbitrary and represents the max
 			// allowed number of typed errors
-			if words.DamerauLevenshteinDistance(command, value.(string)) < 2 {
-				closestCommands = append(closestCommands, value.(string))
+			if words.DamerauLevenshteinDistance(command, value) < 2 {
+				closestCommands = append(closestCommands, value)
 			}
 		}
 

cmd/metrics.go

@@ -161,7 +161,7 @@ func gatewayMetricsPrometheus(ch chan<- prometheus.Metric) {
 		return
 	}
 
-	objLayer := newObjectLayerWithoutSafeModeFn()
+	objLayer := newObjectLayerFn()
 	// Service not initialized yet
 	if objLayer == nil {
 		return
@@ -372,7 +372,7 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 // Populates prometheus with bucket usage metrics, this metrics
 // is only enabled if crawler is enabled.
 func bucketUsageMetricsPrometheus(ch chan<- prometheus.Metric) {
-	objLayer := newObjectLayerWithoutSafeModeFn()
+	objLayer := newObjectLayerFn()
 	// Service not initialized yet
 	if objLayer == nil {
 		return
@@ -435,7 +435,7 @@ func bucketUsageMetricsPrometheus(ch chan<- prometheus.Metric) {
 // collects storage metrics for MinIO server in Prometheus specific format
 // and sends to given channel
 func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
-	objLayer := newObjectLayerWithoutSafeModeFn()
+	objLayer := newObjectLayerFn()
 	// Service not initialized yet
 	if objLayer == nil {
 		return