Added 'close' to results channel in Walk() (#9956)

Readiness as no reasoning to be cluster scope
because that is not how the k8s networking works
for pods, all the pods to a deployment are not
sharing the network in a singleton. Instead they
are run as local scopes to themselves, with
readiness failures the pod is potentially taken
out of the network to be resolvable - this
affects the distributed setup in myriad of
different ways.

Instead readiness should behave like liveness
with local scope alone, and should be a dummy
implementation.

This PR all the startup times and overal k8s
startup time dramatically improves.

Added another handler called as `/minio/health/cluster`
to understand the cluster scope health.

.github/workflows/codeql.yml

@@ -0,0 +1,51 @@
+name: "Code scanning - action"
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '0 19 * * 0'
+
+jobs:
+  CodeQL-Build:
+
+    # CodeQL runs on ubuntu-latest and windows-latest
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+      
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+         languages: go, javascript
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

Makefile

@@ -8,8 +8,6 @@ GOOS := $(shell go env GOOS)
 VERSION ?= $(shell git describe --tags)
 TAG ?= "minio/minio:$(VERSION)"
 
-BUILD_LDFLAGS := '$(LDFLAGS)'
-
 all: build
 
 checks:
@@ -48,19 +46,19 @@ test-race: verifiers build
 # Verify minio binary
 verify:
 	@echo "Verifying build with race"
-	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 # Verify healing of disks with minio binary
 verify-healing:
 	@echo "Verify healing build with race"
-	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 
 # Builds minio locally.
 build: checks
 	@echo "Building minio binary to './minio'"
-	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath  --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
 docker: build
 	@docker build -t $(TAG) . -f Dockerfile.dev

browser/app/js/browser/StorageInfo.js

@@ -26,13 +26,10 @@ export class StorageInfo extends React.Component {
   }
   render() {
     const { used } = this.props.storageInfo
-
-    if (!used) {
+    if (!used || used == 0) {
       return <noscript />
     }
 
-    const totalUsed = used.reduce((v1, v2) => v1 + v2, 0)
-
     return (
       <div className="feh-used">
         <div className="fehu-chart">
@@ -41,7 +38,7 @@ export class StorageInfo extends React.Component {
         <ul>
           <li>
             <span>Used: </span>
-            {humanize.filesize(totalUsed)}
+            {humanize.filesize(used)}
           </li>
         </ul>
       </div>

browser/app/js/browser/__tests__/StorageInfo.test.js

@@ -21,7 +21,7 @@ import { StorageInfo } from "../StorageInfo"
 describe("StorageInfo", () => {
   it("should render without crashing", () => {
     shallow(
-      <StorageInfo storageInfo={{ used: [60] }} fetchStorageInfo={jest.fn()} />
+      <StorageInfo storageInfo={ {used: 60} } fetchStorageInfo={jest.fn()} />
     )
   })
 
@@ -29,7 +29,7 @@ describe("StorageInfo", () => {
     const fetchStorageInfo = jest.fn()
     shallow(
       <StorageInfo
-        storageInfo={{ used: [60] }}
+        storageInfo={ {used: 60} }
         fetchStorageInfo={fetchStorageInfo}
       />
     )
@@ -40,7 +40,7 @@ describe("StorageInfo", () => {
     const fetchStorageInfo = jest.fn()
     const wrapper = shallow(
       <StorageInfo
-        storageInfo={{ used: null }}
+      storageInfo={ {used: 0} }
         fetchStorageInfo={fetchStorageInfo}
       />
     )

browser/app/js/browser/__tests__/actions.test.js

@@ -20,7 +20,9 @@ import * as actionsCommon from "../actions"
 
 jest.mock("../../web", () => ({
   StorageInfo: jest.fn(() => {
-    return Promise.resolve({ storageInfo: { Used: [60] } })
+    return Promise.resolve({
+      used: 60
+    })
   }),
   ServerInfo: jest.fn(() => {
     return Promise.resolve({
@@ -39,7 +41,7 @@ describe("Common actions", () => {
   it("creates common/SET_STORAGE_INFO after fetching the storage details ", () => {
     const store = mockStore()
     const expectedActions = [
-      { type: "common/SET_STORAGE_INFO", storageInfo: { used: [60] } }
+      { type: "common/SET_STORAGE_INFO", storageInfo: { used: 60 } }
     ]
     return store.dispatch(actionsCommon.fetchStorageInfo()).then(() => {
       const actions = store.getActions()

browser/app/js/browser/__tests__/reducer.test.js

@@ -21,11 +21,7 @@ describe("common reducer", () => {
   it("should return the initial state", () => {
     expect(reducer(undefined, {})).toEqual({
       sidebarOpen: false,
-      storageInfo: {
-        total: [0],
-        free: [0],
-        used: [0]
-      },
+      storageInfo: {used: 0},
       serverInfo: {}
     })
   })
@@ -62,11 +58,11 @@ describe("common reducer", () => {
         {},
         {
           type: actionsCommon.SET_STORAGE_INFO,
-          storageInfo: { total: [100], free: [40] }
+          storageInfo: { }
         }
       )
     ).toEqual({
-      storageInfo: { total: [100], free: [40] }
+      storageInfo: { }
     })
   })
 

browser/app/js/browser/actions.js

@@ -33,8 +33,7 @@ export const fetchStorageInfo = () => {
   return function(dispatch) {
     return web.StorageInfo().then(res => {
       const storageInfo = {
-        total: res.storageInfo.Total,
-        used: res.storageInfo.Used
+        used: res.used
       }
       dispatch(setStorageInfo(storageInfo))
     })

browser/app/js/browser/reducer.js

@@ -19,7 +19,7 @@ import * as actionsCommon from "./actions"
 export default (
   state = {
     sidebarOpen: false,
-    storageInfo: { total: [0], free: [0], used: [0] },
+    storageInfo: {used: 0},
     serverInfo: {}
   },
   action

browser/app/js/buckets/actions.js

@@ -31,19 +31,38 @@ export const SET_POLICIES = "buckets/SET_POLICIES"
 
 export const fetchBuckets = () => {
   return function(dispatch) {
+    const { bucket, prefix } = pathSlice(history.location.pathname)
     return web.ListBuckets().then(res => {
       const buckets = res.buckets ? res.buckets.map(bucket => bucket.name) : []
-      dispatch(setList(buckets))
       if (buckets.length > 0) {
-        const { bucket, prefix } = pathSlice(history.location.pathname)
+        dispatch(setList(buckets))
         if (bucket && buckets.indexOf(bucket) > -1) {
           dispatch(selectBucket(bucket, prefix))
         } else {
           dispatch(selectBucket(buckets[0]))
         }
       } else {
-        dispatch(selectBucket(""))
-        history.replace("/")
+        if (bucket) {
+          dispatch(setList([bucket]))
+          dispatch(selectBucket(bucket, prefix))
+        } else {
+          dispatch(selectBucket(""))
+          history.replace("/")
+        }
+      }
+    })
+    .catch(err => {
+      if (bucket && err.message === "Access Denied." || err.message.indexOf('Prefix access is denied') > -1 ) {
+        dispatch(setList([bucket]))
+        dispatch(selectBucket(bucket, prefix))
+      } else {
+        dispatch(
+          alertActions.set({
+            type: "danger",
+            message: err.message,
+            autoClear: true,
+          })
+        )
       }
     })
   }

browser/app/js/uploads/Dropzone.js

@@ -48,18 +48,29 @@ export class Dropzone extends React.Component {
     const rejectStyle = {
       backgroundColor: "#ffdddd"
     }
+    const getStyle = (isDragActive, isDragAccept, isDragReject) => ({
+      ...style,
+      ...(isDragActive ? activeStyle : {}),
+      ...(isDragReject ? rejectStyle : {})
+    })
 
     // disableClick means that it won't trigger a file upload box when
     // the user clicks on a file.
     return (
       <ReactDropzone
-        style={style}
-        activeStyle={activeStyle}
-        rejectStyle={rejectStyle}
-        disableClick={true}
         onDrop={this.onDrop.bind(this)}
       >
-        {this.props.children}
+        {({getRootProps, getInputProps, isDragActive, isDragAccept, isDragReject}) => (
+          <div
+            {...getRootProps({
+              onClick: event => event.stopPropagation()
+            })}
+            style={getStyle(isDragActive, isDragAccept, isDragReject)}
+          >
+            <input {...getInputProps()} />
+            {this.props.children}
+          </div>
+        )}
       </ReactDropzone>
     )
   }

browser/app/js/uploads/actions.js

@@ -89,11 +89,16 @@ export const uploadFile = file => {
       return
     }
     const currentPrefix = getCurrentPrefix(state)
-    const objectName = `${currentPrefix}${file.name}`
+    var _filePath = file.path || file.name
+    if (_filePath.charAt(0) == '/') {
+      _filePath = _filePath.substring(1)
+    }
+    const filePath = _filePath
+    const objectName = `${currentPrefix}${filePath}`
     const uploadUrl = `${
       window.location.origin
     }${minioBrowserPrefix}/upload/${currentBucket}/${objectName}`
-    const slug = `${currentBucket}-${currentPrefix}-${file.name}`
+    const slug = `${currentBucket}-${currentPrefix}-${filePath}`
 
     let xhr = new XMLHttpRequest()
     xhr.open("PUT", uploadUrl, true)
@@ -141,7 +146,7 @@ export const uploadFile = file => {
         dispatch(
           alertActions.set({
             type: "success",
-            message: "File '" + file.name + "' uploaded successfully."
+            message: "File '" + filePath + "' uploaded successfully."
           })
         )
         dispatch(objectsActions.selectPrefix(currentPrefix))
@@ -153,7 +158,7 @@ export const uploadFile = file => {
       dispatch(
         alertActions.set({
           type: "danger",
-          message: "Error occurred uploading '" + file.name + "'."
+          message: "Error occurred uploading '" + filePath + "'."
         })
       )
     })

browser/package-lock.json

@@ -2368,19 +2368,9 @@
       "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg=="
     },
     "attr-accept": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-1.1.3.tgz",
-      "integrity": "sha512-iT40nudw8zmCweivz6j58g+RT33I4KbaIvRUhjNmDwO2WmsQUxFEZZYZ5w3vXe5x5MX9D7mfvA/XaLOZYFR9EQ==",
-      "requires": {
-        "core-js": "^2.5.0"
-      },
-      "dependencies": {
-        "core-js": {
-          "version": "2.6.11",
-          "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.11.tgz",
-          "integrity": "sha512-5wjnpaT/3dV+XB4borEsnAYQchn00XSgTAWKDkEqv+K8KevjbzmofK6hfJ9TZIlpj2N0xQpazy7PiRQiWHqzWg=="
-        }
-      }
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.1.0.tgz",
+      "integrity": "sha512-sLzVM3zCCmmDtDNhI0i96k6PUztkotSOXqE4kDGQt/6iDi5M+H0srjeF+QC6jN581l4X/Zq3Zu/tgcErEssavg=="
     },
     "aws-sign2": {
       "version": "0.7.0",
@@ -6758,6 +6748,14 @@
       "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz",
       "integrity": "sha512-0btnI/H8f2pavGMN8w40mlSKOfTK2SVJmBfBeVIj3kNw0swwgzyRq0d5TJVOwodFmtvpPeWPN/MCcfuWF0Ezbw=="
     },
+    "file-selector": {
+      "version": "0.1.12",
+      "resolved": "https://registry.npmjs.org/file-selector/-/file-selector-0.1.12.tgz",
+      "integrity": "sha512-Kx7RTzxyQipHuiqyZGf+Nz4vY9R1XGxuQl/hLoJwq+J4avk/9wxxgZyHKtbyIPJmbD4A66DWGYfyykWNpcYutQ==",
+      "requires": {
+        "tslib": "^1.9.0"
+      }
+    },
     "filename-regex": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/filename-regex/-/filename-regex-2.0.1.tgz",
@@ -12737,12 +12735,33 @@
       }
     },
     "react-dropzone": {
-      "version": "4.3.0",
-      "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-4.3.0.tgz",
-      "integrity": "sha512-ULfrLaTSsd8BDa9KVAGCueuq1AN3L14dtMsGGqtP0UwYyjG4Vhf158f/ITSHuSPYkZXbvfcIiOlZsH+e3QWm+Q==",
+      "version": "11.0.1",
+      "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-11.0.1.tgz",
+      "integrity": "sha512-x/6wqRHaR8jsrNiu/boVMIPYuoxb83Vyfv77hO7/3ZRn8Pr+KH5onsCsB8MLBa3zdJl410C5FXPUINbu16XIzw==",
       "requires": {
-        "attr-accept": "^1.1.3",
-        "prop-types": "^15.5.7"
+        "attr-accept": "^2.0.0",
+        "file-selector": "^0.1.12",
+        "prop-types": "^15.7.2"
+      },
+      "dependencies": {
+        "loose-envify": {
+          "version": "1.4.0",
+          "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
+          "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==",
+          "requires": {
+            "js-tokens": "^3.0.0 || ^4.0.0"
+          }
+        },
+        "prop-types": {
+          "version": "15.7.2",
+          "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.7.2.tgz",
+          "integrity": "sha512-8QQikdH7//R2vurIJSutZ1smHYTcLpRWEOlHnzcWHmBYrOGUysKwSsrC89BCiFj3CbrfJ/nXFdJepOVrY1GCHQ==",
+          "requires": {
+            "loose-envify": "^1.4.0",
+            "object-assign": "^4.1.1",
+            "react-is": "^16.8.1"
+          }
+        }
       }
     },
     "react-infinite-scroller": {

browser/package.json

@@ -81,7 +81,7 @@
     "react-copy-to-clipboard": "^5.0.2",
     "react-custom-scrollbars": "^4.2.1",
     "react-dom": "^16.13.1",
-    "react-dropzone": "^4.3.0",
+    "react-dropzone": "^11.0.1",
     "react-infinite-scroller": "^1.2.4",
     "react-onclickout": "^2.0.8",
     "react-redux": "^5.1.2",

buildscripts/cross-compile.sh

@@ -9,7 +9,7 @@ function _init() {
     export CGO_ENABLED=0
 
     ## List of architectures and OS to test coss compilation.
-    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386"
+    SUPPORTED_OSARCH="linux/ppc64le linux/arm64 linux/s390x darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64"
 }
 
 function _build() {

buildscripts/verify-healing.sh

@@ -33,6 +33,7 @@ function start_minio_3_node() {
     declare -a ARGS
     export MINIO_ACCESS_KEY=minio
     export MINIO_SECRET_KEY=minio123
+    export MINIO_ERASURE_SET_DRIVE_COUNT=6
 
     start_port=$(shuf -i 10000-65000 -n 1)
     for i in $(seq 1 3); do

cmd/admin-handlers-config-kv.go

@@ -173,7 +173,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	}
 
 	cfg := globalServerConfig
-	if globalSafeMode {
+	if newObjectLayerFn() == nil {
 		var err error
 		cfg, err = getValidConfig(objectAPI)
 		if err != nil {

cmd/admin-handlers.go

@@ -708,10 +708,6 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	// find number of disks in the setup, ignore any errors here.
-	info, _ := objectAPI.StorageInfo(ctx, false)
-	numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum()
-
 	healPath := pathJoin(hip.bucket, hip.objPrefix)
 	if hip.clientToken == "" && !hip.forceStart && !hip.forceStop {
 		nh, exists := globalAllHealState.getHealSequence(healPath)
@@ -754,7 +750,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 			respCh <- hr
 		}()
 	case hip.clientToken == "":
-		nh := newHealSequence(GlobalContext, hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), numDisks, hip.hs, hip.forceStart)
+		nh := newHealSequence(GlobalContext, hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), hip.hs, hip.forceStart)
 		go func() {
 			respBytes, apiErr, errMsg := globalAllHealState.LaunchNewHealSequence(nh)
 			hr := healResp{respBytes, apiErr, errMsg}
@@ -899,13 +895,26 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 			HTTPStatusCode: e.StatusCode,
 		}
 	default:
-		if errors.Is(err, errConfigNotFound) {
+		switch {
+		case errors.Is(err, errConfigNotFound):
 			apiErr = APIError{
 				Code:           "XMinioConfigError",
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusNotFound,
 			}
-		} else {
+		case errors.Is(err, errIAMActionNotAllowed):
+			apiErr = APIError{
+				Code:           "XMinioIAMActionNotAllowed",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusForbidden,
+			}
+		case errors.Is(err, errIAMNotInitialized):
+			apiErr = APIError{
+				Code:           "XMinioIAMNotInitialized",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusServiceUnavailable,
+			}
+		default:
 			apiErr = errorCodes.ToAPIErrWithErr(toAdminAPIErrCode(ctx, err), err)
 		}
 	}
@@ -1386,10 +1395,8 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
-	mode := ""
-	if globalSafeMode {
-		mode = "safe"
-	} else {
+	mode := "safe"
+	if newObjectLayerFn() != nil {
 		mode = "online"
 	}
 

cmd/admin-handlers_test.go

@@ -70,7 +70,7 @@ func prepareAdminXLTestBed(ctx context.Context) (*adminXLTestBed, error) {
 
 	newAllSubsystems()
 
-	initAllSubsystems(objLayer)
+	initAllSubsystems(ctx, objLayer)
 
 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()

cmd/admin-heal-ops.go

@@ -77,9 +77,6 @@ type healSequenceStatus struct {
 	FailureDetail string            `json:"Detail,omitempty"`
 	StartTime     time.Time         `json:"StartTime"`
 
-	// disk information
-	NumDisks int `json:"NumDisks"`
-
 	// settings for the heal sequence
 	HealSettings madmin.HealOpts `json:"Settings"`
 
@@ -95,8 +92,8 @@ type allHealState struct {
 	healSeqMap map[string]*healSequence
 }
 
-// initHealState - initialize healing apparatus
-func initHealState() *allHealState {
+// newHealState - initialize global heal state management
+func newHealState() *allHealState {
 	healState := &allHealState{
 		healSeqMap: make(map[string]*healSequence),
 	}
@@ -296,8 +293,8 @@ func (ahs *allHealState) PopHealStatusJSON(path string,
 
 // healSource denotes single entity and heal option.
 type healSource struct {
-	path string          // entity path (format, buckets, objects) to heal
-	opts madmin.HealOpts // optional heal option overrides default setting
+	path string           // entity path (format, buckets, objects) to heal
+	opts *madmin.HealOpts // optional heal option overrides default setting
 }
 
 // healSequence - state for each heal sequence initiated on the
@@ -368,7 +365,7 @@ type healSequence struct {
 // NewHealSequence - creates healSettings, assumes bucket and
 // objPrefix are already validated.
 func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
-	numDisks int, hs madmin.HealOpts, forceStart bool) *healSequence {
+	hs madmin.HealOpts, forceStart bool) *healSequence {
 
 	reqInfo := &logger.ReqInfo{RemoteHost: clientAddr, API: "Heal", BucketName: bucket}
 	reqInfo.AppendTags("prefix", objPrefix)
@@ -388,7 +385,6 @@ func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
 		currentStatus: healSequenceStatus{
 			Summary:      healNotStartedStatus,
 			HealSettings: hs,
-			NumDisks:     numDisks,
 		},
 		traverseAndHealDoneCh: make(chan error),
 		cancelCtx:             cancel,
@@ -622,8 +618,8 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		opts:       h.settings,
 		responseCh: h.respCh,
 	}
-	if !source.opts.Equal(h.settings) {
-		task.opts = source.opts
+	if source.opts != nil {
+		task.opts = *source.opts
 	}
 	globalBackgroundHealRoutine.queueHealTask(task)
 
@@ -677,11 +673,6 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 }
 
 func (h *healSequence) healItemsFromSourceCh() error {
-	bucketsOnly := true // heal buckets only, not objects.
-	if err := h.healItems(bucketsOnly); err != nil {
-		logger.LogIf(h.ctx, err)
-	}
-
 	for {
 		select {
 		case source, ok := <-h.sourceCh:
@@ -716,7 +707,7 @@ func (h *healSequence) healFromSourceCh() {
 	h.healItemsFromSourceCh()
 }
 
-func (h *healSequence) healItems(bucketsOnly bool) error {
+func (h *healSequence) healDiskMeta() error {
 	// Start with format healing
 	if err := h.healDiskFormat(); err != nil {
 		return err
@@ -728,7 +719,11 @@ func (h *healSequence) healItems(bucketsOnly bool) error {
 	}
 
 	// Start healing the bucket config prefix.
-	if err := h.healMinioSysMeta(bucketConfigPrefix)(); err != nil {
+	return h.healMinioSysMeta(bucketConfigPrefix)()
+}
+
+func (h *healSequence) healItems(bucketsOnly bool) error {
+	if err := h.healDiskMeta(); err != nil {
 		return err
 	}
 

cmd/api-errors.go

@@ -1138,14 +1138,14 @@ var errorCodes = errorCodeMap{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrOperationTimedOut: {
-		Code:           "XMinioServerTimedOut",
-		Description:    "A timeout occurred while trying to lock a resource",
-		HTTPStatusCode: http.StatusRequestTimeout,
+		Code:           "RequestTimeout",
+		Description:    "A timeout occurred while trying to lock a resource, please reduce your request rate",
+		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
 	ErrOperationMaxedOut: {
-		Code:           "XMinioServerTimedOut",
-		Description:    "A timeout exceeded while waiting to proceed with the request",
-		HTTPStatusCode: http.StatusRequestTimeout,
+		Code:           "SlowDown",
+		Description:    "A timeout exceeded while waiting to proceed with the request, please reduce your request rate",
+		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
 	ErrUnsupportedMetadata: {
 		Code:           "InvalidArgument",

cmd/api-headers.go

@@ -68,6 +68,13 @@ func encodeResponseJSON(response interface{}) []byte {
 	return bytesBuffer.Bytes()
 }
 
+// Write parts count
+func setPartsCountHeaders(w http.ResponseWriter, objInfo ObjectInfo) {
+	if strings.Contains(objInfo.ETag, "-") && len(objInfo.Parts) > 0 {
+		w.Header()[xhttp.AmzMpPartsCount] = []string{strconv.Itoa(len(objInfo.Parts))}
+	}
+}
+
 // Write object header
 func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSpec) (err error) {
 	// set common headers
@@ -82,10 +89,6 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
 	}
 
-	if strings.Contains(objInfo.ETag, "-") && len(objInfo.Parts) > 0 {
-		w.Header()[xhttp.AmzMpPartsCount] = []string{strconv.Itoa(len(objInfo.Parts))}
-	}
-
 	if objInfo.ContentType != "" {
 		w.Header().Set(xhttp.ContentType, objInfo.ContentType)
 	}

cmd/background-heal-ops.go

@@ -100,7 +100,7 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 	}
 }
 
-func initHealRoutine() *healRoutine {
+func newHealRoutine() *healRoutine {
 	return &healRoutine{
 		tasks:  make(chan healTask),
 		doneCh: make(chan struct{}),
@@ -108,22 +108,22 @@ func initHealRoutine() *healRoutine {
 
 }
 
-func startBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
+func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 	// Run the background healer
-	globalBackgroundHealRoutine = initHealRoutine()
+	globalBackgroundHealRoutine = newHealRoutine()
 	go globalBackgroundHealRoutine.run(ctx, objAPI)
 
-	// Launch the background healer sequence to track
-	// background healing operations, ignore errors
-	// errors are handled into offline disks already.
-	info, _ := objAPI.StorageInfo(ctx, false)
-	numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum()
-	nh := newBgHealSequence(numDisks)
-	globalBackgroundHealState.LaunchNewHealSequence(nh)
-}
+	nh := newBgHealSequence()
+	// Heal any disk format and metadata early, if possible.
+	if err := nh.healDiskMeta(); err != nil {
+		if newObjectLayerFn() != nil {
+			// log only in situations, when object layer
+			// has fully initialized.
+			logger.LogIf(nh.ctx, err)
+		}
+	}
 
-func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
-	go startBackgroundHealing(ctx, objAPI)
+	globalBackgroundHealState.LaunchNewHealSequence(nh)
 }
 
 // healDiskFormat - heals format.json, return value indicates if a
@@ -140,12 +140,20 @@ func healDiskFormat(ctx context.Context, objAPI ObjectLayer, opts madmin.HealOpt
 	// Healing succeeded notify the peers to reload format and re-initialize disks.
 	// We will not notify peers if healing is not required.
 	if err == nil {
-		for _, nerr := range globalNotificationSys.ReloadFormat(opts.DryRun) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
+		// Notify servers in background and retry if needed.
+		go func() {
+		retry:
+			for _, nerr := range globalNotificationSys.ReloadFormat(opts.DryRun) {
+				if nerr.Err != nil {
+					if nerr.Err.Error() == errServerNotInitialized.Error() {
+						time.Sleep(time.Second)
+						goto retry
+					}
+					logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+					logger.LogIf(ctx, nerr.Err)
+				}
 			}
-		}
+		}()
 	}
 
 	return res, nil

cmd/background-newdisks-heal-ops.go

@@ -115,7 +115,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 			// Heal all erasure sets that need
 			for i, erasureSetToHeal := range erasureSetInZoneToHeal {
 				for _, setIndex := range erasureSetToHeal {
-					err := healErasureSet(ctx, setIndex, z.zones[i].sets[setIndex])
+					err := healErasureSet(ctx, setIndex, z.zones[i].sets[setIndex], z.zones[i].drivesPerSet)
 					if err != nil {
 						logger.LogIf(ctx, err)
 					}

cmd/bucket-encryption-handlers.go

@@ -23,7 +23,6 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bucket/policy"
 )
@@ -46,15 +45,14 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	// PutBucketEncyrption API requires Content-Md5
-	if _, ok := r.Header[xhttp.ContentMD5]; !ok {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
+	if !objAPI.IsEncryptionSupported() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketEncryptionAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return

cmd/bucket-handlers.go

@@ -47,7 +47,7 @@ import (
 const (
 	getBucketVersioningResponse = `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`
 	objectLockConfig            = "object-lock.xml"
-	bucketTaggingConfigFile     = "tagging.xml"
+	bucketTaggingConfig         = "tagging.xml"
 )
 
 // Check if there are buckets on server without corresponding entry in etcd backend and
@@ -1123,7 +1123,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if err = globalBucketMetadataSys.Update(bucket, bucketTaggingConfigFile, configData); err != nil {
+	if err = globalBucketMetadataSys.Update(bucket, bucketTaggingConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -1191,7 +1191,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err := globalBucketMetadataSys.Update(bucket, bucketTaggingConfigFile, nil); err != nil {
+	if err := globalBucketMetadataSys.Update(bucket, bucketTaggingConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}

cmd/bucket-metadata-sys.go

@@ -30,6 +30,7 @@ import (
 	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
 // BucketMetadataSys captures all bucket metadata for a given cluster.
@@ -40,7 +41,7 @@ type BucketMetadataSys struct {
 
 // Remove bucket metadata from memory.
 func (sys *BucketMetadataSys) Remove(bucket string) {
-	if globalIsGateway && globalGatewayName != "nas" {
+	if globalIsGateway {
 		return
 	}
 	sys.Lock()
@@ -73,9 +74,49 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		return errServerNotInitialized
 	}
 
-	if globalIsGateway && globalGatewayName != "nas" {
+	if globalIsGateway {
 		// This code is needed only for gateway implementations.
-		if configFile == bucketPolicyConfig {
+		switch configFile {
+		case bucketSSEConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.EncryptionConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketLifecycleConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.LifecycleConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketTaggingConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.TaggingConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketNotificationConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.NotificationConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketPolicyConfig:
+			if configData == nil {
+				return objAPI.DeleteBucketPolicy(GlobalContext, bucket)
+			}
 			config, err := policy.ParseConfig(bytes.NewReader(configData), bucket)
 			if err != nil {
 				return err
@@ -103,7 +144,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		meta.LifecycleConfigXML = configData
 	case bucketSSEConfig:
 		meta.EncryptionConfigXML = configData
-	case bucketTaggingConfigFile:
+	case bucketTaggingConfig:
 		meta.TaggingConfigXML = configData
 	case objectLockConfig:
 		meta.ObjectLockConfigXML = configData
@@ -194,6 +235,19 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
 // GetNotificationConfig returns configured notification config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
+	if globalIsGateway && globalGatewayName == "nas" {
+		// Only needed in case of NAS gateway.
+		objAPI := newObjectLayerWithoutSafeModeFn()
+		if objAPI == nil {
+			return nil, errServerNotInitialized
+		}
+		meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+		if err != nil {
+			return nil, err
+		}
+		return meta.notificationConfig, nil
+	}
+
 	meta, err := sys.GetConfig(bucket)
 	if err != nil {
 		return nil, err
@@ -220,6 +274,14 @@ func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEC
 // GetPolicyConfig returns configured bucket policy
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, error) {
+	if globalIsGateway {
+		objAPI := newObjectLayerWithoutSafeModeFn()
+		if objAPI == nil {
+			return nil, errServerNotInitialized
+		}
+		return objAPI.GetBucketPolicy(GlobalContext, bucket)
+	}
+
 	meta, err := sys.GetConfig(bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
@@ -243,7 +305,7 @@ func (sys *BucketMetadataSys) GetQuotaConfig(bucket string) (*madmin.BucketQuota
 	return meta.quotaConfig, nil
 }
 
-// GetConfig returns a specific configuration from the bucket metadata.
+// GetConfig returns the current bucket metadata
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetConfig(bucket string) (BucketMetadata, error) {
 	objAPI := newObjectLayerWithoutSafeModeFn()
@@ -251,7 +313,7 @@ func (sys *BucketMetadataSys) GetConfig(bucket string) (BucketMetadata, error) {
 		return newBucketMetadata(bucket), errServerNotInitialized
 	}
 
-	if globalIsGateway && globalGatewayName != "nas" {
+	if globalIsGateway {
 		return newBucketMetadata(bucket), NotImplemented{}
 	}
 
@@ -291,21 +353,44 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
 	return sys.load(ctx, buckets, objAPI)
 }
 
-// Loads bucket metadata for all buckets into BucketMetadataSys.
-func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
-	sys.Lock()
-	defer sys.Unlock()
-
-	for _, bucket := range buckets {
-		meta, err := loadBucketMetadata(ctx, objAPI, bucket.Name)
+// concurrently load bucket metadata to speed up loading bucket metadata.
+func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+	g := errgroup.WithNErrs(len(buckets))
+	for index := range buckets {
+		index := index
+		g.Go(func() error {
+			meta, err := loadBucketMetadata(ctx, objAPI, buckets[index].Name)
+			if err != nil {
+				return err
+			}
+			sys.Lock()
+			sys.metadataMap[buckets[index].Name] = meta
+			sys.Unlock()
+			return nil
+		}, index)
+	}
+	for _, err := range g.Wait() {
 		if err != nil {
 			return err
 		}
-		sys.metadataMap[bucket.Name] = meta
 	}
 	return nil
 }
 
+// Loads bucket metadata for all buckets into BucketMetadataSys.
+func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+	count := 100 // load 100 bucket metadata at a time.
+	for {
+		if len(buckets) < count {
+			return sys.concurrentLoad(ctx, buckets, objAPI)
+		}
+		if err := sys.concurrentLoad(ctx, buckets[:count], objAPI); err != nil {
+			return err
+		}
+		buckets = buckets[count:]
+	}
+}
+
 // NewBucketMetadataSys - creates new policy system.
 func NewBucketMetadataSys() *BucketMetadataSys {
 	return &BucketMetadataSys{

cmd/bucket-metadata.go

@@ -206,7 +206,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 		bucketLifecycleConfig,
 		bucketQuotaConfigFile,
 		bucketSSEConfig,
-		bucketTaggingConfigFile,
+		bucketTaggingConfig,
 		objectLockConfig,
 	}
 
@@ -255,7 +255,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 			b.LifecycleConfigXML = configData
 		case bucketSSEConfig:
 			b.EncryptionConfigXML = configData
-		case bucketTaggingConfigFile:
+		case bucketTaggingConfig:
 			b.TaggingConfigXML = configData
 		case objectLockConfig:
 			b.ObjectLockConfigXML = configData
@@ -303,9 +303,15 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
 // deleteBucketMetadata deletes bucket metadata
 // If config does not exist no error is returned.
 func deleteBucketMetadata(ctx context.Context, obj ObjectLayer, bucket string) error {
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketMetadataFile)
-	if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
-		return err
+	metadataFiles := []string{
+		dataUsageCacheName,
+		bucketMetadataFile,
+	}
+	for _, metaFile := range metadataFiles {
+		configFile := path.Join(bucketConfigPrefix, bucket, metaFile)
+		if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
+			return err
+		}
 	}
 	return nil
 }

cmd/bucket-policy.go

@@ -37,13 +37,6 @@ type PolicySys struct{}
 
 // Get returns stored bucket policy
 func (sys *PolicySys) Get(bucket string) (*policy.Policy, error) {
-	if globalIsGateway {
-		objAPI := newObjectLayerFn()
-		if objAPI == nil {
-			return nil, errServerNotInitialized
-		}
-		return objAPI.GetBucketPolicy(GlobalContext, bucket)
-	}
 	return globalBucketMetadataSys.GetPolicyConfig(bucket)
 }
 

cmd/config-current.go

@@ -325,13 +325,40 @@ func lookupConfigs(s config.Config) {
 
 	etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+		if globalIsGateway {
+			logger.FatalIf(err, "Unable to initialize etcd config")
+		} else {
+			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+		}
 	}
 
 	if etcdCfg.Enabled {
-		globalEtcdClient, err = etcd.New(etcdCfg)
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+		if globalEtcdClient == nil {
+			globalEtcdClient, err = etcd.New(etcdCfg)
+			if err != nil {
+				if globalIsGateway {
+					logger.FatalIf(err, "Unable to initialize etcd config")
+				} else {
+					logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+				}
+			}
+		}
+
+		if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil && globalDNSConfig == nil {
+			globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
+				dns.DomainNames(globalDomainNames),
+				dns.DomainIPs(globalDomainIPs),
+				dns.DomainPort(globalMinioPort),
+				dns.CoreDNSPath(etcdCfg.CoreDNSPath),
+			)
+			if err != nil {
+				if globalIsGateway {
+					logger.FatalIf(err, "Unable to initialize DNS config")
+				} else {
+					logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
+						globalDomainNames, err))
+				}
+			}
 		}
 	}
 
@@ -342,19 +369,6 @@ func lookupConfigs(s config.Config) {
 	// but not federation.
 	globalBucketFederation = etcdCfg.PathPrefix == "" && etcdCfg.Enabled
 
-	if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
-		globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
-			dns.DomainNames(globalDomainNames),
-			dns.DomainIPs(globalDomainIPs),
-			dns.DomainPort(globalMinioPort),
-			dns.CoreDNSPath(etcdCfg.CoreDNSPath),
-		)
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
-				globalDomainNames, err))
-		}
-	}
-
 	globalServerRegion, err = config.LookupRegion(s[config.RegionSubSys][config.Default])
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Invalid region configuration: %w", err))
@@ -365,13 +379,7 @@ func lookupConfigs(s config.Config) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
 	}
 
-	apiRequestsMax := apiConfig.APIRequestsMax
-	if len(globalEndpoints.Hosts()) > 0 {
-		apiRequestsMax /= len(globalEndpoints.Hosts())
-	}
-
-	globalAPIThrottling.init(apiRequestsMax, apiConfig.APIRequestsDeadline)
-	globalReadyDeadline = apiConfig.APIReadyDeadline
+	globalAPIConfig.init(apiConfig)
 
 	if globalIsXL {
 		globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],
@@ -383,7 +391,11 @@ func lookupConfigs(s config.Config) {
 
 	globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
+		if globalIsGateway {
+			logger.FatalIf(err, "Unable to setup cache")
+		} else {
+			logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
+		}
 	}
 
 	if globalCacheConfig.Enabled {

cmd/config.go

@@ -149,7 +149,7 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 	if err != nil {
 		// Config not found for some reason, allow things to continue
 		// by initializing a new fresh config in safe mode.
-		if err == errConfigNotFound && globalSafeMode {
+		if err == errConfigNotFound && newObjectLayerFn() == nil {
 			return newServerConfig(), nil
 		}
 		return nil, err

cmd/config/api/api.go

@@ -20,16 +20,24 @@ import (
 	"encoding/json"
 	"errors"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/env"
 )
 
+// API sub-system constants
 const (
 	apiRequestsMax      = "requests_max"
 	apiRequestsDeadline = "requests_deadline"
 	apiReadyDeadline    = "ready_deadline"
+	apiCorsAllowOrigin  = "cors_allow_origin"
+
+	EnvAPIRequestsMax      = "MINIO_API_REQUESTS_MAX"
+	EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE"
+	EnvAPIReadyDeadline    = "MINIO_API_READY_DEADLINE"
+	EnvAPICorsAllowOrigin  = "MINIO_API_CORS_ALLOW_ORIGIN"
 )
 
 // DefaultKVS - default storage class config
@@ -47,6 +55,10 @@ var (
 			Key:   apiReadyDeadline,
 			Value: "10s",
 		},
+		config.KV{
+			Key:   apiCorsAllowOrigin,
+			Value: "*",
+		},
 	}
 )
 
@@ -55,6 +67,7 @@ type Config struct {
 	APIRequestsMax      int           `json:"requests_max"`
 	APIRequestsDeadline time.Duration `json:"requests_deadline"`
 	APIReadyDeadline    time.Duration `json:"ready_deadline"`
+	APICorsAllowOrigin  []string      `json:"cors_allow_origin"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -75,7 +88,7 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	}
 
 	// Check environment variables parameters
-	requestsMax, err := strconv.Atoi(env.Get(config.EnvAPIRequestsMax, kvs.Get(apiRequestsMax)))
+	requestsMax, err := strconv.Atoi(env.Get(EnvAPIRequestsMax, kvs.Get(apiRequestsMax)))
 	if err != nil {
 		return cfg, err
 	}
@@ -84,20 +97,21 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, errors.New("invalid API max requests value")
 	}
 
-	requestsDeadline, err := time.ParseDuration(env.Get(config.EnvAPIRequestsDeadline, kvs.Get(apiRequestsDeadline)))
+	requestsDeadline, err := time.ParseDuration(env.Get(EnvAPIRequestsDeadline, kvs.Get(apiRequestsDeadline)))
 	if err != nil {
 		return cfg, err
 	}
 
-	readyDeadline, err := time.ParseDuration(env.Get(config.EnvAPIReadyDeadline, kvs.Get(apiReadyDeadline)))
+	readyDeadline, err := time.ParseDuration(env.Get(EnvAPIReadyDeadline, kvs.Get(apiReadyDeadline)))
 	if err != nil {
 		return cfg, err
 	}
 
-	cfg = Config{
+	corsAllowOrigin := strings.Split(env.Get(EnvAPICorsAllowOrigin, kvs.Get(apiCorsAllowOrigin)), ",")
+	return Config{
 		APIRequestsMax:      requestsMax,
 		APIRequestsDeadline: requestsDeadline,
 		APIReadyDeadline:    readyDeadline,
-	}
-	return cfg, nil
+		APICorsAllowOrigin:  corsAllowOrigin,
+	}, nil
 }

cmd/config/api/help.go

@@ -34,10 +34,10 @@ var (
 			Type:        "duration",
 		},
 		config.HelpKV{
-			Key:         apiReadyDeadline,
-			Description: `set the deadline for health check API /minio/health/ready e.g. "1m"`,
+			Key:         apiCorsAllowOrigin,
+			Description: `set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"`,
 			Optional:    true,
-			Type:        "duration",
+			Type:        "csv",
 		},
 	}
 )

cmd/config/cache/config.go

@@ -37,6 +37,7 @@ type Config struct {
 	After         int      `json:"after"`
 	WatermarkLow  int      `json:"watermark_low"`
 	WatermarkHigh int      `json:"watermark_high"`
+	Range         bool     `json:"range"`
 }
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling

cmd/config/cache/help.go

@@ -68,5 +68,11 @@ var (
 			Optional:    true,
 			Type:        "number",
 		},
+		config.HelpKV{
+			Key:         Range,
+			Description: `set to "on" or "off" caching of independent range requests per object, defaults to "on"`,
+			Optional:    true,
+			Type:        "string",
+		},
 	}
 )

cmd/config/cache/lookup.go

@@ -34,6 +34,7 @@ const (
 	After         = "after"
 	WatermarkLow  = "watermark_low"
 	WatermarkHigh = "watermark_high"
+	Range         = "range"
 
 	EnvCacheDrives        = "MINIO_CACHE_DRIVES"
 	EnvCacheExclude       = "MINIO_CACHE_EXCLUDE"
@@ -43,6 +44,7 @@ const (
 	EnvCacheAfter         = "MINIO_CACHE_AFTER"
 	EnvCacheWatermarkLow  = "MINIO_CACHE_WATERMARK_LOW"
 	EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
+	EnvCacheRange         = "MINIO_CACHE_RANGE"
 
 	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
 
@@ -84,6 +86,10 @@ var (
 			Key:   WatermarkHigh,
 			Value: DefaultWaterMarkHigh,
 		},
+		config.KV{
+			Key:   Range,
+			Value: config.EnableOn,
+		},
 	}
 )
 
@@ -195,5 +201,15 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		err := errors.New("config high watermark value should be greater than low watermark value")
 		return cfg, config.ErrInvalidCacheWatermarkHigh(err)
 	}
+
+	cfg.Range = true // by default range caching is enabled.
+	if rangeStr := env.Get(EnvCacheRange, kvs.Get(Range)); rangeStr != "" {
+		rng, err := config.ParseBool(rangeStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheRange(err)
+		}
+		cfg.Range = rng
+	}
+
 	return cfg, nil
 }

cmd/config/constants.go

@@ -34,11 +34,6 @@ const (
 	EnvEndpoints    = "MINIO_ENDPOINTS"
 	EnvFSOSync      = "MINIO_FS_OSYNC"
 
-	// API sub-system
-	EnvAPIRequestsMax      = "MINIO_API_REQUESTS_MAX"
-	EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE"
-	EnvAPIReadyDeadline    = "MINIO_API_READY_DEADLINE"
-
 	EnvUpdate = "MINIO_UPDATE"
 
 	EnvWorm   = "MINIO_WORM"   // legacy

cmd/config/errors.go

@@ -96,6 +96,12 @@ var (
 		"MINIO_CACHE_ENCRYPTION_MASTER_KEY: For more information, please refer to https://docs.min.io/docs/minio-disk-cache-guide",
 	)
 
+	ErrInvalidCacheRange = newErrFn(
+		"Invalid cache range value",
+		"Please check the passed value",
+		"MINIO_CACHE_RANGE: Valid expected value is `on` or `off`",
+	)
+
 	ErrInvalidRotatingCredentialsBackendEncrypted = newErrFn(
 		"Invalid rotating credentials",
 		"Please set correct rotating credentials in the environment for decryption",

cmd/config/etcd/dns/etcd_dns.go

@@ -47,6 +47,11 @@ func newCoreDNSMsg(ip string, port string, ttl uint32, t time.Time) ([]byte, err
 	})
 }
 
+// Close closes the internal etcd client and cannot be used further
+func (c *CoreDNS) Close() {
+	c.etcdClient.Close()
+}
+
 // List - Retrieves list of DNS entries for the domain.
 func (c *CoreDNS) List() (map[string][]SrvRecord, error) {
 	var srvRecords = map[string][]SrvRecord{}

cmd/config/notify/help.go

@@ -59,6 +59,18 @@ var (
 			Optional:    true,
 			Type:        "sentence",
 		},
+		config.HelpKV{
+			Key:         target.WebhookClientCert,
+			Description: "client cert for Webhook mTLS auth",
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         target.WebhookClientKey,
+			Description: "client cert key for Webhook mTLS auth",
+			Optional:    true,
+			Type:        "string",
+		},
 	}
 
 	HelpAMQP = config.HelpKVS{

cmd/config/notify/legacy.go

@@ -281,6 +281,14 @@ func SetNotifyWebhook(s config.Config, whName string, cfg target.WebhookArgs) er
 			Key:   target.WebhookQueueLimit,
 			Value: strconv.Itoa(int(cfg.QueueLimit)),
 		},
+		config.KV{
+			Key:   target.WebhookClientCert,
+			Value: cfg.ClientCert,
+		},
+		config.KV{
+			Key:   target.WebhookClientKey,
+			Value: cfg.ClientKey,
+		},
 	}
 
 	return nil

cmd/config/notify/parse.go

@@ -1428,6 +1428,14 @@ var (
 			Key:   target.WebhookQueueDir,
 			Value: "",
 		},
+		config.KV{
+			Key:   target.WebhookClientCert,
+			Value: "",
+		},
+		config.KV{
+			Key:   target.WebhookClientKey,
+			Value: "",
+		},
 	}
 )
 
@@ -1471,6 +1479,15 @@ func GetNotifyWebhook(webhookKVS map[string]config.KVS, transport *http.Transpor
 		if k != config.Default {
 			authEnv = authEnv + config.Default + k
 		}
+		clientCertEnv := target.EnvWebhookClientCert
+		if k != config.Default {
+			clientCertEnv = clientCertEnv + config.Default + k
+		}
+
+		clientKeyEnv := target.EnvWebhookClientKey
+		if k != config.Default {
+			clientKeyEnv = clientKeyEnv + config.Default + k
+		}
 
 		webhookArgs := target.WebhookArgs{
 			Enable:     enabled,
@@ -1479,6 +1496,8 @@ func GetNotifyWebhook(webhookKVS map[string]config.KVS, transport *http.Transpor
 			AuthToken:  env.Get(authEnv, kv.Get(target.WebhookAuthToken)),
 			QueueDir:   env.Get(queueDirEnv, kv.Get(target.WebhookQueueDir)),
 			QueueLimit: uint64(queueLimit),
+			ClientCert: env.Get(clientCertEnv, kv.Get(target.WebhookClientCert)),
+			ClientKey:  env.Get(clientKeyEnv, kv.Get(target.WebhookClientKey)),
 		}
 		if err = webhookArgs.Validate(); err != nil {
 			return nil, err

cmd/crypto/kes.go

@@ -34,6 +34,10 @@ import (
 	xnet "github.com/minio/minio/pkg/net"
 )
 
+// ErrKESKeyNotFound is the error returned a KES server
+// when a master key does not exist.
+var ErrKESKeyNotFound = NewKESError(http.StatusNotFound, "key does not exist")
+
 // KesConfig contains the configuration required
 // to initialize and connect to a kes server.
 type KesConfig struct {
@@ -154,6 +158,12 @@ func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 
 	var plainKey []byte
 	plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes())
+	if err == ErrKESKeyNotFound { // Try to create the key if it does not exist.
+		if err = kes.client.CreateKey(keyID); err != nil {
+			return key, nil, err
+		}
+		plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes())
+	}
 	if err != nil {
 		return key, nil, err
 	}
@@ -214,16 +224,19 @@ type kesClient struct {
 	httpClient http.Client
 }
 
-// Response KES response struct
-type response struct {
-	Plaintext  []byte `json:"plaintext"`
-	Ciphertext []byte `json:"ciphertext,omitempty"`
-}
-
-// Request KES request struct
-type request struct {
-	Ciphertext []byte `json:"ciphertext,omitempty"`
-	Context    []byte `json:"context"`
+// CreateKey tries to create a new cryptographic key with
+// the specified name.
+//
+// The key will be generated by the server. The client
+// application does not have the cryptographic key at
+// any point in time.
+func (c *kesClient) CreateKey(name string) error {
+	url := fmt.Sprintf("%s/v1/key/create/%s", c.addr, url.PathEscape(name))
+	_, err := c.postRetry(url, nil, 0) // No request body and no response expected
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
 // GenerateDataKey requests a new data key from the KES server.
@@ -235,48 +248,155 @@ type request struct {
 // such that you have to provide the same context when decrypting
 // the data key.
 func (c *kesClient) GenerateDataKey(name string, context []byte) ([]byte, []byte, error) {
-	body, err := json.Marshal(request{
+	type Request struct {
+		Context []byte `json:"context"`
+	}
+	type Response struct {
+		Plaintext  []byte `json:"plaintext"`
+		Ciphertext []byte `json:"ciphertext"`
+	}
+
+	body, err := json.Marshal(Request{
 		Context: context,
 	})
 	if err != nil {
 		return nil, nil, err
 	}
 
-	url := fmt.Sprintf("%s/v1/key/generate/%s", c.addr, url.PathEscape(name))
-
 	const limit = 1 << 20 // A plaintext/ciphertext key pair will never be larger than 1 MB
+	url := fmt.Sprintf("%s/v1/key/generate/%s", c.addr, url.PathEscape(name))
 	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	return resp.Plaintext, resp.Ciphertext, nil
+	var response Response
+	if err = json.NewDecoder(resp).Decode(&response); err != nil {
+		return nil, nil, err
+	}
+	return response.Plaintext, response.Ciphertext, nil
 }
 
-func (c *kesClient) post(url string, body io.Reader, limit int64) (*response, error) {
-	resp, err := c.httpClient.Post(url, "application/json", body)
+// GenerateDataKey decrypts an encrypted data key with the key
+// specified by name by talking to the KES server.
+// On success, the KES server will respond with the plaintext key.
+//
+// The optional context must match the value you provided when
+// generating the data key.
+func (c *kesClient) DecryptDataKey(name string, ciphertext, context []byte) ([]byte, error) {
+	type Request struct {
+		Ciphertext []byte `json:"ciphertext"`
+		Context    []byte `json:"context,omitempty"`
+	}
+	type Response struct {
+		Plaintext []byte `json:"plaintext"`
+	}
+
+	body, err := json.Marshal(Request{
+		Ciphertext: ciphertext,
+		Context:    context,
+	})
 	if err != nil {
 		return nil, err
 	}
 
+	const limit = 1 << 20 // A data key will never be larger than 1 MiB
+	url := fmt.Sprintf("%s/v1/key/decrypt/%s", c.addr, url.PathEscape(name))
+	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
+	if err != nil {
+		return nil, err
+	}
+
+	var response Response
+	if err = json.NewDecoder(resp).Decode(&response); err != nil {
+		return nil, err
+	}
+	return response.Plaintext, nil
+}
+
+// NewKESError returns a new KES API error with the given
+// HTTP status code and error message.
+//
+// Two errors with the same status code and
+// error message are equal:
+//   e1 == e2 // true.
+func NewKESError(code int, text string) error {
+	return kesError{
+		code:    code,
+		message: text,
+	}
+}
+
+type kesError struct {
+	code    int
+	message string
+}
+
+// Status returns the HTTP status code of the error.
+func (e kesError) Status() int { return e.code }
+
+// Status returns the error message of the error.
+func (e kesError) Error() string { return e.message }
+
+func parseErrorResponse(resp *http.Response) error {
+	if resp == nil || resp.StatusCode < 400 {
+		return nil
+	}
+	if resp.Body == nil {
+		return NewKESError(resp.StatusCode, "")
+	}
+	defer resp.Body.Close()
+
+	const MaxBodySize = 1 << 20
+	var size = resp.ContentLength
+	if size < 0 || size > MaxBodySize {
+		size = MaxBodySize
+	}
+
+	contentType := strings.TrimSpace(resp.Header.Get("Content-Type"))
+	if strings.HasPrefix(contentType, "application/json") {
+		type Response struct {
+			Message string `json:"message"`
+		}
+		var response Response
+		if err := json.NewDecoder(io.LimitReader(resp.Body, size)).Decode(&response); err != nil {
+			return err
+		}
+		return NewKESError(resp.StatusCode, response.Message)
+	}
+
+	var sb strings.Builder
+	if _, err := io.Copy(&sb, io.LimitReader(resp.Body, size)); err != nil {
+		return err
+	}
+	return NewKESError(resp.StatusCode, sb.String())
+}
+
+func (c *kesClient) post(url string, body io.Reader, limit int64) (io.Reader, error) {
+	resp, err := c.httpClient.Post(url, "application/json", body)
+	if err != nil {
+		return nil, err
+	}
 	// Drain the entire body to make sure we have re-use connections
 	defer xhttp.DrainBody(resp.Body)
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, c.parseErrorResponse(resp)
+		return nil, parseErrorResponse(resp)
 	}
 
-	response := &response{}
-	if err = json.NewDecoder(io.LimitReader(resp.Body, limit)).Decode(response); err != nil {
+	// We have to copy the response body due to draining.
+	var respBody bytes.Buffer
+	if _, err = io.Copy(&respBody, io.LimitReader(resp.Body, limit)); err != nil {
 		return nil, err
 	}
-	return response, nil
+	return &respBody, nil
 }
 
-func (c *kesClient) postRetry(url string, body io.ReadSeeker, limit int64) (*response, error) {
+func (c *kesClient) postRetry(url string, body io.ReadSeeker, limit int64) (io.Reader, error) {
 	for i := 0; ; i++ {
-		body.Seek(0, io.SeekStart) // seek to the beginning of the body.
-
+		if body != nil {
+			body.Seek(0, io.SeekStart) // seek to the beginning of the body.
+		}
 		response, err := c.post(url, body, limit)
 		if err == nil {
 			return response, nil
@@ -296,45 +416,6 @@ func (c *kesClient) postRetry(url string, body io.ReadSeeker, limit int64) (*res
 	}
 }
 
-// GenerateDataKey decrypts an encrypted data key with the key
-// specified by name by talking to the KES server.
-// On success, the KES server will respond with the plaintext key.
-//
-// The optional context must match the value you provided when
-// generating the data key.
-func (c *kesClient) DecryptDataKey(name string, ciphertext, context []byte) ([]byte, error) {
-	body, err := json.Marshal(request{
-		Ciphertext: ciphertext,
-		Context:    context,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	url := fmt.Sprintf("%s/v1/key/decrypt/%s", c.addr, url.PathEscape(name))
-
-	const limit = 32 * 1024 // A data key will never be larger than 32 KB
-	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
-	if err != nil {
-		return nil, err
-	}
-	return resp.Plaintext, nil
-}
-
-func (c *kesClient) parseErrorResponse(resp *http.Response) error {
-	if resp.Body == nil {
-		return Errorf("%s: no body", http.StatusText(resp.StatusCode))
-	}
-
-	const limit = 32 * 1024 // A (valid) error response will not be greater than 32 KB
-	var errMsg strings.Builder
-	if _, err := io.Copy(&errMsg, io.LimitReader(resp.Body, limit)); err != nil {
-		return Errorf("%s: %s", http.StatusText(resp.StatusCode), err)
-	}
-
-	return Errorf("%s: %s", http.StatusText(resp.StatusCode), errMsg.String())
-}
-
 // loadCACertificates returns a new CertPool
 // that contains all system root CA certificates
 // and any PEM-encoded certificate(s) found at

cmd/daily-lifecycle-ops.go

@@ -1,133 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"time"
-
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/bucket/lifecycle"
-	"github.com/minio/minio/pkg/event"
-)
-
-const (
-	bgLifecycleInterval = 24 * time.Hour
-)
-
-// initDailyLifecycle starts the routine that receives the daily
-// listing of all objects and applies any matching bucket lifecycle
-// rules.
-func initDailyLifecycle(ctx context.Context, objAPI ObjectLayer) {
-	go startDailyLifecycle(ctx, objAPI)
-}
-
-func startDailyLifecycle(ctx context.Context, objAPI ObjectLayer) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.NewTimer(bgLifecycleInterval).C:
-			// Perform one lifecycle operation
-			logger.LogIf(ctx, lifecycleRound(ctx, objAPI))
-		}
-
-	}
-}
-
-func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
-	buckets, err := objAPI.ListBuckets(ctx)
-	if err != nil {
-		return err
-	}
-
-	for _, bucket := range buckets {
-		// Check if the current bucket has a configured lifecycle policy, skip otherwise
-		l, err := globalLifecycleSys.Get(bucket.Name)
-		if err != nil {
-			if _, ok := err.(BucketLifecycleNotFound); !ok {
-				logger.LogIf(ctx, err)
-			}
-			continue
-		}
-
-		rcfg, _ := globalBucketObjectLockSys.Get(bucket.Name)
-
-		// Calculate the common prefix of all lifecycle rules
-		var prefixes []string
-		for _, rule := range l.Rules {
-			prefixes = append(prefixes, rule.Prefix())
-		}
-		commonPrefix := lcp(prefixes)
-
-		// Allocate new results channel to receive ObjectInfo.
-		objInfoCh := make(chan ObjectInfo)
-
-		// Walk through all objects
-		if err := objAPI.Walk(ctx, bucket.Name, commonPrefix, objInfoCh); err != nil {
-			return err
-		}
-
-		for {
-			var objects []string
-			for obj := range objInfoCh {
-				if len(objects) == maxDeleteList {
-					// Reached maximum delete requests, attempt a delete for now.
-					break
-				}
-				// Find the action that need to be executed
-				if l.ComputeAction(obj.Name, obj.UserTags, obj.ModTime) == lifecycle.DeleteAction {
-					if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
-						continue
-					}
-					objects = append(objects, obj.Name)
-				}
-			}
-
-			// Nothing to do.
-			if len(objects) == 0 {
-				break
-			}
-
-			waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()))
-
-			// Deletes a list of objects.
-			deleteErrs, err := objAPI.DeleteObjects(ctx, bucket.Name, objects)
-			if err != nil {
-				logger.LogIf(ctx, err)
-			} else {
-				for i := range deleteErrs {
-					if deleteErrs[i] != nil {
-						logger.LogIf(ctx, deleteErrs[i])
-						continue
-					}
-					// Notify object deleted event.
-					sendEvent(eventArgs{
-						EventName:  event.ObjectRemovedDelete,
-						BucketName: bucket.Name,
-						Object: ObjectInfo{
-							Name: objects[i],
-						},
-						Host: "Internal: [ILM-EXPIRY]",
-					})
-				}
-			}
-		}
-	}
-
-	return nil
-}

cmd/data-crawler.go

@@ -0,0 +1,600 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"errors"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/minio/minio/cmd/config"
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/hash"
+	"github.com/willf/bloom"
+)
+
+const (
+	dataCrawlSleepPerFolder  = time.Millisecond // Time to wait between folders.
+	dataCrawlSleepDefMult    = 10.0             // Default multiplier for waits between operations.
+	dataCrawlStartDelay      = 5 * time.Minute  // Time to wait on startup and between cycles.
+	dataUsageUpdateDirCycles = 16               // Visit all folders every n cycles.
+
+)
+
+// initDataCrawler will start the crawler unless disabled.
+func initDataCrawler(ctx context.Context, objAPI ObjectLayer) {
+	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
+		go runDataCrawler(ctx, objAPI)
+	}
+}
+
+// runDataCrawler will start a data crawler.
+// The function will block until the context is canceled.
+// There should only ever be one crawler running per cluster.
+func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
+	// Load current bloom cycle
+	nextBloomCycle := intDataUpdateTracker.current() + 1
+	var buf bytes.Buffer
+	err := objAPI.GetObject(ctx, dataUsageBucket, dataUsageBloomName, 0, -1, &buf, "", ObjectOptions{})
+	if err != nil {
+		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) {
+			logger.LogIf(ctx, err)
+		}
+	} else {
+		if buf.Len() == 8 {
+			nextBloomCycle = binary.LittleEndian.Uint64(buf.Bytes())
+		}
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.NewTimer(dataCrawlStartDelay).C:
+			// Wait before starting next cycle and wait on startup.
+			results := make(chan DataUsageInfo, 1)
+			go storeDataUsageInBackend(ctx, objAPI, results)
+			bf, err := globalNotificationSys.updateBloomFilter(ctx, nextBloomCycle)
+			logger.LogIf(ctx, err)
+			err = objAPI.CrawlAndGetDataUsage(ctx, bf, results)
+			close(results)
+			logger.LogIf(ctx, err)
+			if err == nil {
+				// Store new cycle...
+				nextBloomCycle++
+				if nextBloomCycle%dataUpdateTrackerResetEvery == 0 {
+					if intDataUpdateTracker.debug {
+						logger.Info(color.Green("runDataCrawler:") + " Resetting bloom filter for next runs.")
+					}
+					nextBloomCycle++
+				}
+				var tmp [8]byte
+				binary.LittleEndian.PutUint64(tmp[:], nextBloomCycle)
+				r, err := hash.NewReader(bytes.NewReader(tmp[:]), int64(len(tmp)), "", "", int64(len(tmp)), false)
+				if err != nil {
+					logger.LogIf(ctx, err)
+					continue
+				}
+
+				_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageBloomName, NewPutObjReader(r, nil, nil), ObjectOptions{})
+				if !isErrBucketNotFound(err) {
+					logger.LogIf(ctx, err)
+				}
+			}
+		}
+	}
+}
+
+type cachedFolder struct {
+	name   string
+	parent *dataUsageHash
+}
+
+type folderScanner struct {
+	root               string
+	getSize            getSizeFn
+	oldCache           dataUsageCache
+	newCache           dataUsageCache
+	withFilter         *bloomFilter
+	waitForLowActiveIO func()
+
+	dataUsageCrawlMult  float64
+	dataUsageCrawlDebug bool
+
+	newFolders      []cachedFolder
+	existingFolders []cachedFolder
+}
+
+// crawlDataFolder will crawl the basepath+cache.Info.Name and return an updated cache.
+// The returned cache will always be valid, but may not be updated from the existing.
+// Before each operation waitForLowActiveIO is called which can be used to temporarily halt the crawler.
+// If the supplied context is canceled the function will return at the first chance.
+func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache, waitForLowActiveIO func(), getSize getSizeFn) (dataUsageCache, error) {
+	t := UTCNow()
+
+	logPrefix := color.Green("data-usage: ")
+	logSuffix := color.Blue(" - %v + %v", basePath, cache.Info.Name)
+	if intDataUpdateTracker.debug {
+		defer func() {
+			logger.Info(logPrefix+" Crawl time: %v"+logSuffix, time.Since(t))
+		}()
+
+	}
+
+	switch cache.Info.Name {
+	case "", dataUsageRoot:
+		return cache, errors.New("internal error: root scan attempted")
+	}
+
+	delayMult, err := strconv.ParseFloat(env.Get(envDataUsageCrawlDelay, "10.0"), 64)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		delayMult = dataCrawlSleepDefMult
+	}
+
+	s := folderScanner{
+		root:                basePath,
+		getSize:             getSize,
+		oldCache:            cache,
+		newCache:            dataUsageCache{Info: cache.Info},
+		waitForLowActiveIO:  waitForLowActiveIO,
+		newFolders:          nil,
+		existingFolders:     nil,
+		dataUsageCrawlMult:  delayMult,
+		dataUsageCrawlDebug: intDataUpdateTracker.debug,
+	}
+
+	if len(cache.Info.BloomFilter) > 0 {
+		s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
+		_, err := s.withFilter.ReadFrom(bytes.NewBuffer(cache.Info.BloomFilter))
+		if err != nil {
+			logger.LogIf(ctx, err, logPrefix+"Error reading bloom filter")
+			s.withFilter = nil
+		}
+	}
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Start crawling. Bloom filter: %v"+logSuffix, s.withFilter != nil)
+	}
+
+	done := ctx.Done()
+	var flattenLevels = 2
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Cycle: %v, Entries: %v"+logSuffix, cache.Info.NextCycle, len(cache.Cache))
+	}
+
+	// Always scan flattenLevels deep. Cache root is level 0.
+	todo := []cachedFolder{{name: cache.Info.Name}}
+	for i := 0; i < flattenLevels; i++ {
+		if s.dataUsageCrawlDebug {
+			logger.Info(logPrefix+"Level %v, scanning %v directories."+logSuffix, i, len(todo))
+		}
+		select {
+		case <-done:
+			return cache, ctx.Err()
+		default:
+		}
+		var err error
+		todo, err = s.scanQueuedLevels(ctx, todo, i == flattenLevels-1)
+		if err != nil {
+			// No useful information...
+			return cache, err
+		}
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"New folders: %v"+logSuffix, s.newFolders)
+	}
+
+	// Add new folders first
+	for _, folder := range s.newFolders {
+		select {
+		case <-done:
+			return s.newCache, ctx.Err()
+		default:
+		}
+		du, err := s.deepScanFolder(ctx, folder.name)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		if du == nil {
+			logger.Info(logPrefix + "no disk usage provided" + logSuffix)
+			continue
+		}
+
+		s.newCache.replace(folder.name, "", *du)
+		// Add to parent manually
+		if folder.parent != nil {
+			parent := s.newCache.Cache[folder.parent.Key()]
+			parent.addChildString(folder.name)
+		}
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Existing folders: %v"+logSuffix, len(s.existingFolders))
+	}
+
+	// Do selective scanning of existing folders.
+	for _, folder := range s.existingFolders {
+		select {
+		case <-done:
+			return s.newCache, ctx.Err()
+		default:
+		}
+		h := hashPath(folder.name)
+		if !h.mod(s.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
+			s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h.Key()])
+			continue
+		}
+
+		if s.withFilter != nil {
+			_, prefix := path2BucketObjectWithBasePath(basePath, folder.name)
+			if s.oldCache.Info.lifeCycle == nil || !s.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
+				// If folder isn't in filter, skip it completely.
+				if !s.withFilter.containsDir(folder.name) {
+					if s.dataUsageCrawlDebug {
+						logger.Info(logPrefix+"Skipping non-updated folder: %v"+logSuffix, folder)
+					}
+					s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h.Key()])
+					continue
+				}
+			}
+		}
+
+		// Update on this cycle...
+		du, err := s.deepScanFolder(ctx, folder.name)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		if du == nil {
+			logger.LogIf(ctx, errors.New("data-usage: no disk usage provided"))
+			continue
+		}
+		s.newCache.replaceHashed(h, folder.parent, *du)
+	}
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Finished crawl, %v entries"+logSuffix, len(s.newCache.Cache))
+	}
+	s.newCache.Info.LastUpdate = UTCNow()
+	s.newCache.Info.NextCycle++
+	return s.newCache, nil
+}
+
+// scanQueuedLevels will scan the provided folders.
+// Files found in the folders will be added to f.newCache.
+// If final is provided folders will be put into f.newFolders or f.existingFolders.
+// If final is not provided the folders found are returned from the function.
+func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool) ([]cachedFolder, error) {
+	var nextFolders []cachedFolder
+	done := ctx.Done()
+	for _, folder := range folders {
+		select {
+		case <-done:
+			return nil, ctx.Err()
+		default:
+		}
+		thisHash := hashPath(folder.name)
+
+		// If there are lifecycle rules for the prefix, remove the filter.
+		filter := f.withFilter
+		var activeLifeCycle *lifecycle.Lifecycle
+		if f.oldCache.Info.lifeCycle != nil && filter != nil {
+			_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
+			if f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" Prefix %q has active rules", prefix)
+				}
+				activeLifeCycle = f.oldCache.Info.lifeCycle
+				filter = nil
+			}
+		}
+
+		if _, ok := f.oldCache.Cache[thisHash.Key()]; filter != nil && ok {
+			// If folder isn't in filter and we have data, skip it completely.
+			if folder.name != dataUsageRoot && !filter.containsDir(folder.name) {
+				f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" Skipping non-updated folder: %v", folder.name)
+				}
+				continue
+			}
+		}
+		f.waitForLowActiveIO()
+		sleepDuration(dataCrawlSleepPerFolder, f.dataUsageCrawlMult)
+
+		cache := dataUsageEntry{}
+
+		err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error {
+			// Parse
+			entName = path.Clean(path.Join(folder.name, entName))
+			bucket, prefix := path2BucketObjectWithBasePath(f.root, entName)
+			if bucket == "" {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" no bucket (%s,%s)", f.root, entName)
+				}
+				return nil
+			}
+
+			if isReservedOrInvalidBucket(bucket, false) {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" invalid bucket: %v, entry: %v", bucket, entName)
+				}
+				return nil
+			}
+
+			select {
+			case <-done:
+				return ctx.Err()
+			default:
+			}
+
+			if typ&os.ModeDir != 0 {
+				h := hashPath(entName)
+				_, exists := f.oldCache.Cache[h.Key()]
+				cache.addChildString(entName)
+
+				this := cachedFolder{name: entName, parent: &thisHash}
+				cache.addChild(h)
+				if final {
+					if exists {
+						f.existingFolders = append(f.existingFolders, this)
+					} else {
+						f.newFolders = append(f.newFolders, this)
+					}
+				} else {
+					nextFolders = append(nextFolders, this)
+				}
+				return nil
+			}
+			f.waitForLowActiveIO()
+			// Dynamic time delay.
+			t := UTCNow()
+
+			// Get file size, ignore errors.
+			item := crawlItem{
+				Path:       path.Join(f.root, entName),
+				Typ:        typ,
+				bucket:     bucket,
+				prefix:     path.Dir(prefix),
+				objectName: path.Base(entName),
+				debug:      f.dataUsageCrawlDebug,
+				lifeCycle:  activeLifeCycle,
+			}
+			size, err := f.getSize(item)
+
+			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+			if err == errSkipFile || err == errFileNotFound {
+				return nil
+			}
+			logger.LogIf(ctx, err)
+			cache.Size += size
+			cache.Objects++
+			cache.ObjSizes.add(size)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+		f.newCache.replaceHashed(thisHash, folder.parent, cache)
+	}
+	return nextFolders, nil
+}
+
+// deepScanFolder will deep scan a folder and return the size if no error occurs.
+func (f *folderScanner) deepScanFolder(ctx context.Context, folder string) (*dataUsageEntry, error) {
+	var cache dataUsageEntry
+
+	done := ctx.Done()
+
+	var addDir func(entName string, typ os.FileMode) error
+	var dirStack = []string{f.root, folder}
+
+	addDir = func(entName string, typ os.FileMode) error {
+		select {
+		case <-done:
+			return ctx.Err()
+		default:
+		}
+
+		f.waitForLowActiveIO()
+		if typ&os.ModeDir != 0 {
+			dirStack = append(dirStack, entName)
+			err := readDirFn(path.Join(dirStack...), addDir)
+			dirStack = dirStack[:len(dirStack)-1]
+			sleepDuration(dataCrawlSleepPerFolder, f.dataUsageCrawlMult)
+			return err
+		}
+
+		// Dynamic time delay.
+		t := UTCNow()
+
+		// Get file size, ignore errors.
+		dirStack = append(dirStack, entName)
+		fileName := path.Join(dirStack...)
+		dirStack = dirStack[:len(dirStack)-1]
+
+		bucket, prefix := path2BucketObjectWithBasePath(f.root, fileName)
+		var activeLifeCycle *lifecycle.Lifecycle
+		if f.oldCache.Info.lifeCycle != nil {
+			if f.oldCache.Info.lifeCycle.HasActiveRules(prefix, false) {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" Prefix %q has active rules", prefix)
+				}
+				activeLifeCycle = f.oldCache.Info.lifeCycle
+			}
+		}
+
+		size, err := f.getSize(
+			crawlItem{
+				Path:       fileName,
+				Typ:        typ,
+				bucket:     bucket,
+				prefix:     path.Dir(prefix),
+				objectName: path.Base(entName),
+				debug:      f.dataUsageCrawlDebug,
+				lifeCycle:  activeLifeCycle,
+			})
+
+		// Don't sleep for really small amount of time
+		sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+
+		if err == errSkipFile {
+			return nil
+		}
+		logger.LogIf(ctx, err)
+		cache.Size += size
+		cache.Objects++
+		cache.ObjSizes.add(size)
+		return nil
+	}
+	err := readDirFn(path.Join(dirStack...), addDir)
+	if err != nil {
+		return nil, err
+	}
+	return &cache, nil
+}
+
+// crawlItem represents each file while walking.
+type crawlItem struct {
+	Path string
+	Typ  os.FileMode
+
+	bucket     string // Bucket.
+	prefix     string // Only the prefix if any, does not have final object name.
+	objectName string // Only the object name without prefixes.
+	lifeCycle  *lifecycle.Lifecycle
+	debug      bool
+}
+
+type getSizeFn func(item crawlItem) (int64, error)
+
+// transformMetaDir will transform a directory to prefix/file.ext
+func (i *crawlItem) transformMetaDir() {
+	split := strings.Split(i.prefix, SlashSeparator)
+	if len(split) > 1 {
+		i.prefix = path.Join(split[:len(split)-1]...)
+	} else {
+		i.prefix = ""
+	}
+	// Object name is last element
+	i.objectName = split[len(split)-1]
+}
+
+// actionMeta contains information used to apply actions.
+type actionMeta struct {
+	oi      ObjectInfo
+	trustOI bool // Set true if oi can be trusted and has been read with quorum.
+	meta    map[string]string
+}
+
+// applyActions will apply lifecycle checks on to a scanned item.
+// The resulting size on disk will always be returned.
+// The metadata will be compared to consensus on the object layer before any changes are applied.
+// If no metadata is supplied, -1 is returned if no action is taken.
+func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta actionMeta) (size int64) {
+	size, err := meta.oi.GetActualSize()
+	if i.debug {
+		logger.LogIf(ctx, err)
+	}
+	if i.lifeCycle == nil {
+		return size
+	}
+
+	action := i.lifeCycle.ComputeAction(i.objectPath(), meta.meta[xhttp.AmzObjectTagging], meta.oi.ModTime)
+	if i.debug {
+		logger.Info(color.Green("applyActions:")+" lifecycle: %q, Initial scan: %v", i.objectPath(), action)
+	}
+	switch action {
+	case lifecycle.DeleteAction:
+	default:
+		// No action.
+		return size
+	}
+
+	// These (expensive) operations should only run on items we are likely to delete.
+	// Load to ensure that we have the correct version and not an unsynced version.
+	if !meta.trustOI {
+		obj, err := o.GetObjectInfo(ctx, i.bucket, i.objectPath(), ObjectOptions{})
+		if err != nil {
+			// Do nothing - heal in the future.
+			logger.LogIf(ctx, err)
+			return size
+		}
+		size = obj.Size
+
+		// Recalculate action.
+		action = i.lifeCycle.ComputeAction(i.objectPath(), obj.UserTags, obj.ModTime)
+		if i.debug {
+			logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action)
+		}
+		switch action {
+		case lifecycle.DeleteAction:
+		default:
+			// No action.
+			return size
+		}
+	}
+
+	err = o.DeleteObject(ctx, i.bucket, i.objectPath())
+	if err != nil {
+		// Assume it is still there.
+		logger.LogIf(ctx, err)
+		return size
+	}
+
+	// Notify object deleted event.
+	sendEvent(eventArgs{
+		EventName:  event.ObjectRemovedDelete,
+		BucketName: i.bucket,
+		Object: ObjectInfo{
+			Name: i.objectPath(),
+		},
+		Host: "Internal: [ILM-EXPIRY]",
+	})
+	return 0
+}
+
+// objectPath returns the prefix and object name.
+func (i *crawlItem) objectPath() string {
+	return path.Join(i.prefix, i.objectName)
+}
+
+// sleepDuration multiplies the duration d by x and sleeps if is more than 100 micro seconds.
+// sleep is limited to max 1 second.
+func sleepDuration(d time.Duration, x float64) {
+	// Don't sleep for really small amount of time
+	if d := time.Duration(float64(d) * x); d > time.Microsecond*100 {
+		if d > time.Second {
+			d = time.Second
+		}
+		time.Sleep(d)
+	}
+}

cmd/data-update-tracker.go

@@ -46,7 +46,7 @@ const (
 	dataUpdateTrackerQueueSize = 10000
 
 	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
-	dataUpdateTrackerVersion      = 1
+	dataUpdateTrackerVersion      = 2
 	dataUpdateTrackerSaveInterval = 5 * time.Minute
 
 	// Reset bloom filters every n cycle
@@ -116,9 +116,7 @@ func (b bloomFilter) containsDir(in string) bool {
 	if len(split) == 0 {
 		return false
 	}
-	var tmp [dataUsageHashLen]byte
-	hashPath(path.Join(split...)).bytes(tmp[:])
-	return b.Test(tmp[:])
+	return b.TestString(hashPath(path.Join(split...)).String())
 }
 
 // bytes returns the bloom filter serialized as a byte slice.
@@ -366,6 +364,9 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 		return err
 	}
 	switch tmp[0] {
+	case 1:
+		logger.Info(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
+		return nil
 	case dataUpdateTrackerVersion:
 	default:
 		return errors.New("dataUpdateTracker: Unknown data version")
@@ -435,7 +436,6 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 // start a collector that picks up entries from objectUpdatedCh
 // and adds them  to the current bloom filter.
 func (d *dataUpdateTracker) startCollector(ctx context.Context) {
-	var tmp [dataUsageHashLen]byte
 	for {
 		select {
 		case <-ctx.Done():
@@ -463,8 +463,7 @@ func (d *dataUpdateTracker) startCollector(ctx context.Context) {
 				if d.debug && false {
 					logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
 				}
-				hashPath(path.Join(split[:i+1]...)).bytes(tmp[:])
-				d.Current.bf.Add(tmp[:])
+				d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
 			}
 			d.dirty = d.dirty || len(split) > 0
 			d.mu.Unlock()

cmd/data-usage-cache.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
@@ -29,17 +28,17 @@ import (
 	"time"
 
 	"github.com/cespare/xxhash/v2"
+	"github.com/klauspost/compress/zstd"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/tinylib/msgp/msgp"
 )
 
-const dataUsageHashLen = 8
-
 //go:generate msgp -file $GOFILE -unexported
 
 // dataUsageHash is the hash type used.
-type dataUsageHash uint64
+type dataUsageHash string
 
 // sizeHistogram is a size histogram.
 type sizeHistogram [dataUsageBucketLen]uint64
@@ -54,6 +53,12 @@ type dataUsageEntry struct {
 	Children dataUsageHashMap
 }
 
+// dataUsageCache contains a cache of data usage entries.
+type dataUsageCache struct {
+	Info  dataUsageCacheInfo
+	Cache map[string]dataUsageEntry
+}
+
 //msgp:ignore dataUsageEntryInfo
 type dataUsageEntryInfo struct {
 	Name   string
@@ -66,7 +71,8 @@ type dataUsageCacheInfo struct {
 	Name        string
 	LastUpdate  time.Time
 	NextCycle   uint32
-	BloomFilter []byte `msg:"BloomFilter,omitempty"`
+	BloomFilter []byte               `msg:"BloomFilter,omitempty"`
+	lifeCycle   *lifecycle.Lifecycle `msg:"-"`
 }
 
 // merge other data usage entry into this, excluding children.
@@ -80,7 +86,7 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 
 // mod returns true if the hash mod cycles == cycle.
 func (h dataUsageHash) mod(cycle uint32, cycles uint32) bool {
-	return uint32(h)%cycles == cycle%cycles
+	return uint32(xxhash.Sum64String(string(h)))%cycles == cycle%cycles
 }
 
 // addChildString will add a child based on its name.
@@ -92,25 +98,95 @@ func (e *dataUsageEntry) addChildString(name string) {
 // addChild will add a child based on its hash.
 // If it already exists it will not be added again.
 func (e *dataUsageEntry) addChild(hash dataUsageHash) {
-	if _, ok := e.Children[hash]; ok {
+	if _, ok := e.Children[hash.Key()]; ok {
 		return
 	}
 	if e.Children == nil {
 		e.Children = make(dataUsageHashMap, 1)
 	}
-	e.Children[hash] = struct{}{}
+	e.Children[hash.Key()] = struct{}{}
 }
 
 // find a path in the cache.
 // Returns nil if not found.
 func (d *dataUsageCache) find(path string) *dataUsageEntry {
-	due, ok := d.Cache[hashPath(path)]
+	due, ok := d.Cache[hashPath(path).Key()]
 	if !ok {
 		return nil
 	}
 	return &due
 }
 
+// Returns nil if not found.
+func (d *dataUsageCache) subCache(path string) dataUsageCache {
+	dst := dataUsageCache{Info: dataUsageCacheInfo{
+		Name:        path,
+		LastUpdate:  d.Info.LastUpdate,
+		BloomFilter: d.Info.BloomFilter,
+	}}
+	dst.copyWithChildren(d, dataUsageHash(hashPath(path).Key()), nil)
+	return dst
+}
+
+func (d *dataUsageCache) deleteRecursive(h dataUsageHash) {
+	if existing, ok := d.Cache[h.String()]; ok {
+		// Delete first if there should be a loop.
+		delete(d.Cache, h.Key())
+		for child := range existing.Children {
+			d.deleteRecursive(dataUsageHash(child))
+		}
+	}
+}
+
+// replaceRootChild will replace the child of root in d with the root of 'other'.
+func (d *dataUsageCache) replaceRootChild(other dataUsageCache) {
+	otherRoot := other.root()
+	if otherRoot == nil {
+		logger.LogIf(GlobalContext, errors.New("replaceRootChild: Source has no root"))
+		return
+	}
+	thisRoot := d.root()
+	if thisRoot == nil {
+		logger.LogIf(GlobalContext, errors.New("replaceRootChild: Root of current not found"))
+		return
+	}
+	thisRootHash := d.rootHash()
+	otherRootHash := other.rootHash()
+	if thisRootHash == otherRootHash {
+		logger.LogIf(GlobalContext, errors.New("replaceRootChild: Root of child matches root of destination"))
+		return
+	}
+	d.deleteRecursive(other.rootHash())
+	d.copyWithChildren(&other, other.rootHash(), &thisRootHash)
+}
+
+// keepBuckets will keep only the buckets specified specified by delete all others.
+func (d *dataUsageCache) keepBuckets(b []BucketInfo) {
+	lu := make(map[dataUsageHash]struct{})
+	for _, v := range b {
+		lu[hashPath(v.Name)] = struct{}{}
+	}
+	d.keepRootChildren(lu)
+}
+
+// keepRootChildren will keep the root children specified by delete all others.
+func (d *dataUsageCache) keepRootChildren(list map[dataUsageHash]struct{}) {
+	if d.root() == nil {
+		return
+	}
+	rh := d.rootHash()
+	for k := range d.Cache {
+		h := dataUsageHash(k)
+		if h == rh {
+			continue
+		}
+		if _, ok := list[h]; !ok {
+			delete(d.Cache, k)
+			d.deleteRecursive(h)
+		}
+	}
+}
+
 // dui converts the flattened version of the path to DataUsageInfo.
 // As a side effect d will be flattened, use a clone if this is not ok.
 func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
@@ -135,14 +211,14 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
 func (d *dataUsageCache) replace(path, parent string, e dataUsageEntry) {
 	hash := hashPath(path)
 	if d.Cache == nil {
-		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+		d.Cache = make(map[string]dataUsageEntry, 100)
 	}
-	d.Cache[hash] = e
+	d.Cache[hash.Key()] = e
 	if parent != "" {
 		phash := hashPath(parent)
-		p := d.Cache[phash]
+		p := d.Cache[phash.Key()]
 		p.addChild(hash)
-		d.Cache[phash] = p
+		d.Cache[phash.Key()] = p
 	}
 }
 
@@ -151,13 +227,13 @@ func (d *dataUsageCache) replace(path, parent string, e dataUsageEntry) {
 // If the parent does not exist, it will be added.
 func (d *dataUsageCache) replaceHashed(hash dataUsageHash, parent *dataUsageHash, e dataUsageEntry) {
 	if d.Cache == nil {
-		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+		d.Cache = make(map[string]dataUsageEntry, 100)
 	}
-	d.Cache[hash] = e
+	d.Cache[hash.Key()] = e
 	if parent != nil {
-		p := d.Cache[*parent]
+		p := d.Cache[parent.Key()]
 		p.addChild(hash)
-		d.Cache[*parent] = p
+		d.Cache[parent.Key()] = p
 	}
 }
 
@@ -166,24 +242,24 @@ func (d *dataUsageCache) replaceHashed(hash dataUsageHash, parent *dataUsageHash
 // If the parent does not exist, it will be added.
 func (d *dataUsageCache) copyWithChildren(src *dataUsageCache, hash dataUsageHash, parent *dataUsageHash) {
 	if d.Cache == nil {
-		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+		d.Cache = make(map[string]dataUsageEntry, 100)
 	}
-	e, ok := src.Cache[hash]
+	e, ok := src.Cache[hash.String()]
 	if !ok {
 		return
 	}
-	d.Cache[hash] = e
+	d.Cache[hash.Key()] = e
 	for ch := range e.Children {
-		if ch == hash {
+		if ch == hash.Key() {
 			logger.LogIf(GlobalContext, errors.New("dataUsageCache.copyWithChildren: Circular reference"))
 			return
 		}
-		d.copyWithChildren(src, ch, &hash)
+		d.copyWithChildren(src, dataUsageHash(ch), &hash)
 	}
 	if parent != nil {
-		p := d.Cache[*parent]
+		p := d.Cache[parent.Key()]
 		p.addChild(hash)
-		d.Cache[*parent] = p
+		d.Cache[parent.Key()] = p
 	}
 }
 
@@ -196,15 +272,14 @@ func (d *dataUsageCache) StringAll() string {
 	return strings.TrimSpace(s)
 }
 
-// insert the hash into dst.
-// dst must be at least dataUsageHashLen bytes long.
-func (h dataUsageHash) bytes(dst []byte) {
-	binary.LittleEndian.PutUint64(dst, uint64(h))
+// String returns a human readable representation of the string.
+func (h dataUsageHash) String() string {
+	return string(h)
 }
 
 // String returns a human readable representation of the string.
-func (h dataUsageHash) String() string {
-	return fmt.Sprintf("%x", uint64(h))
+func (h dataUsageHash) Key() string {
+	return string(h)
 }
 
 // flatten all children of the root into the root element and return it.
@@ -270,13 +345,6 @@ func (d *dataUsageCache) sizeRecursive(path string) *dataUsageEntry {
 	return &flat
 }
 
-// dataUsageCache contains a cache of data usage entries.
-//msgp:ignore dataUsageCache
-type dataUsageCache struct {
-	Info  dataUsageCacheInfo
-	Cache map[dataUsageHash]dataUsageEntry
-}
-
 // root returns the root of the cache.
 func (d *dataUsageCache) root() *dataUsageEntry {
 	return d.find(d.Info.Name)
@@ -291,7 +359,7 @@ func (d *dataUsageCache) rootHash() dataUsageHash {
 func (d *dataUsageCache) clone() dataUsageCache {
 	clone := dataUsageCache{
 		Info:  d.Info,
-		Cache: make(map[dataUsageHash]dataUsageEntry, len(d.Cache)),
+		Cache: make(map[string]dataUsageEntry, len(d.Cache)),
 	}
 	for k, v := range d.Cache {
 		clone.Cache[k] = v
@@ -326,7 +394,7 @@ func (d *dataUsageCache) merge(other dataUsageCache) {
 		existing := d.Cache[key]
 		// If not found, merging simply adds.
 		existing.merge(flat)
-		d.replaceHashed(key, &eHash, existing)
+		d.replaceHashed(dataUsageHash(key), &eHash, existing)
 	}
 }
 
@@ -337,13 +405,13 @@ func (d *dataUsageCache) load(ctx context.Context, store ObjectLayer, name strin
 	var buf bytes.Buffer
 	err := store.GetObject(ctx, dataUsageBucket, name, 0, -1, &buf, "", ObjectOptions{})
 	if err != nil {
-		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) {
+		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) && !errors.Is(err, InsufficientReadQuorum{}) {
 			return toObjectErr(err, dataUsageBucket, name)
 		}
 		*d = dataUsageCache{}
 		return nil
 	}
-	err = d.deserialize(buf.Bytes())
+	err = d.deserialize(&buf)
 	if err != nil {
 		*d = dataUsageCache{}
 		logger.LogIf(ctx, err)
@@ -374,89 +442,60 @@ func (d *dataUsageCache) save(ctx context.Context, store ObjectLayer, name strin
 // dataUsageCacheVer indicates the cache version.
 // Bumping the cache version will drop data from previous versions
 // and write new data with the new version.
-const dataUsageCacheVer = 1
+const dataUsageCacheVer = 2
 
 // serialize the contents of the cache.
 func (d *dataUsageCache) serialize() []byte {
-	// Alloc pessimistically
-	// dataUsageCacheVer
-	due := dataUsageEntry{}
-	msgLen := 1
-	msgLen += d.Info.Msgsize()
-	// len(d.Cache)
-	msgLen += binary.MaxVarintLen64
-	// Hashes (one for key, assume 1 child/node)
-	msgLen += len(d.Cache) * dataUsageHashLen * 2
-	msgLen += len(d.Cache) * due.Msgsize()
-
-	// Create destination buffer...
-	dst := make([]byte, 0, msgLen)
-
-	var n int
-	tmp := make([]byte, 1024)
-	// byte: version.
+	// Prepend version and compress.
+	dst := make([]byte, 0, d.Msgsize()+1)
 	dst = append(dst, dataUsageCacheVer)
-	// Info...
-	dst, err := d.Info.MarshalMsg(dst)
+	buf := bytes.NewBuffer(dst)
+	enc, err := zstd.NewWriter(buf,
+		zstd.WithEncoderLevel(zstd.SpeedFastest),
+		zstd.WithWindowSize(1<<20),
+		zstd.WithEncoderConcurrency(2))
 	if err != nil {
-		panic(err)
+		logger.LogIf(GlobalContext, err)
+		return nil
 	}
-	n = binary.PutUvarint(tmp, uint64(len(d.Cache)))
-	dst = append(dst, tmp[:n]...)
-
-	for k, v := range d.Cache {
-		// Put key
-		binary.LittleEndian.PutUint64(tmp[:dataUsageHashLen], uint64(k))
-		dst = append(dst, tmp[:8]...)
-		tmp, err = v.MarshalMsg(tmp[:0])
-		if err != nil {
-			panic(err)
-		}
-		// key, value pairs.
-		dst = append(dst, tmp...)
-
+	mEnc := msgp.NewWriter(enc)
+	err = d.EncodeMsg(mEnc)
+	if err != nil {
+		logger.LogIf(GlobalContext, err)
+		return nil
 	}
-	return dst
+	mEnc.Flush()
+	err = enc.Close()
+	if err != nil {
+		logger.LogIf(GlobalContext, err)
+		return nil
+	}
+	return buf.Bytes()
 }
 
 // deserialize the supplied byte slice into the cache.
-func (d *dataUsageCache) deserialize(b []byte) error {
-	if len(b) < 1 {
+func (d *dataUsageCache) deserialize(r io.Reader) error {
+	var b [1]byte
+	n, _ := r.Read(b[:])
+	if n != 1 {
 		return io.ErrUnexpectedEOF
 	}
 	switch b[0] {
 	case 1:
+		return errors.New("cache version deprecated (will autoupdate)")
+	case dataUsageCacheVer:
 	default:
 		return fmt.Errorf("dataUsageCache: unknown version: %d", int(b[0]))
 	}
-	b = b[1:]
 
-	// Info...
-	b, err := d.Info.UnmarshalMsg(b)
+	// Zstd compressed.
+	dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
 	if err != nil {
 		return err
 	}
-	cacheLen, n := binary.Uvarint(b)
-	if n <= 0 {
-		return fmt.Errorf("dataUsageCache: reading cachelen, n <= 0 ")
-	}
-	b = b[n:]
-	d.Cache = make(map[dataUsageHash]dataUsageEntry, cacheLen)
+	defer dec.Close()
 
-	for i := 0; i < int(cacheLen); i++ {
-		if len(b) <= dataUsageHashLen {
-			return io.ErrUnexpectedEOF
-		}
-		k := binary.LittleEndian.Uint64(b[:dataUsageHashLen])
-		b = b[dataUsageHashLen:]
-		var v dataUsageEntry
-		b, err = v.UnmarshalMsg(b)
-		if err != nil {
-			return err
-		}
-		d.Cache[dataUsageHash(k)] = v
-	}
-	return nil
+	return d.DecodeMsg(msgp.NewReader(dec))
 }
 
 // Trim this from start+end of hashes.
@@ -473,88 +512,91 @@ func hashPath(data string) dataUsageHash {
 	if data != dataUsageRoot {
 		data = strings.Trim(data, hashPathCutSet)
 	}
-	data = path.Clean(data)
-	return dataUsageHash(xxhash.Sum64String(data))
+	return dataUsageHash(path.Clean(data))
 }
 
-//msgp:ignore dataUsageEntryInfo
-type dataUsageHashMap map[dataUsageHash]struct{}
+//msgp:ignore dataUsageHashMap
+type dataUsageHashMap map[string]struct{}
 
-// MarshalMsg implements msgp.Marshaler
-func (d dataUsageHashMap) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, d.Msgsize())
-
-	// Write bin header manually
-	const mbin32 uint8 = 0xc6
-	sz := uint32(len(d)) * dataUsageHashLen
-	o = append(o, mbin32, byte(sz>>24), byte(sz>>16), byte(sz>>8), byte(sz))
-
-	var tmp [dataUsageHashLen]byte
-	for k := range d {
-		binary.LittleEndian.PutUint64(tmp[:], uint64(k))
-		o = append(o, tmp[:]...)
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageHashMap) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0002 uint32
+	zb0002, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	*z = make(dataUsageHashMap, zb0002)
+	for i := uint32(0); i < zb0002; i++ {
+		{
+			var zb0003 string
+			zb0003, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+			(*z)[zb0003] = struct{}{}
+		}
 	}
 	return
 }
 
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (d dataUsageHashMap) Msgsize() (s int) {
-	s = 5 + len(d)*dataUsageHashLen
+// EncodeMsg implements msgp.Encodable
+func (z dataUsageHashMap) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteArrayHeader(uint32(len(z)))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0004 := range z {
+		err = en.WriteString(zb0004)
+		if err != nil {
+			err = msgp.WrapError(err, zb0004)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z dataUsageHashMap) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendArrayHeader(o, uint32(len(z)))
+	for zb0004 := range z {
+		o = msgp.AppendString(o, zb0004)
+	}
 	return
 }
 
 // UnmarshalMsg implements msgp.Unmarshaler
-func (d *dataUsageHashMap) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var hashes []byte
-	hashes, bts, err = msgp.ReadBytesZC(bts)
+func (z *dataUsageHashMap) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0002 uint32
+	zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
 	if err != nil {
-		err = msgp.WrapError(err, "dataUsageHashMap")
+		err = msgp.WrapError(err)
 		return
 	}
-
-	var dst = make(dataUsageHashMap, len(hashes)/dataUsageHashLen)
-	for len(hashes) >= dataUsageHashLen {
-		dst[dataUsageHash(binary.LittleEndian.Uint64(hashes[:dataUsageHashLen]))] = struct{}{}
-		hashes = hashes[dataUsageHashLen:]
+	*z = make(dataUsageHashMap, zb0002)
+	for i := uint32(0); i < zb0002; i++ {
+		{
+			var zb0003 string
+			zb0003, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+			(*z)[zb0003] = struct{}{}
+		}
 	}
-	*d = dst
 	o = bts
 	return
 }
 
-func (d *dataUsageHashMap) DecodeMsg(dc *msgp.Reader) (err error) {
-	var zb0001 uint32
-	zb0001, err = dc.ReadBytesHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z dataUsageHashMap) Msgsize() (s int) {
+	s = msgp.ArrayHeaderSize
+	for zb0004 := range z {
+		s += msgp.StringPrefixSize + len(zb0004)
 	}
-	var dst = make(dataUsageHashMap, zb0001)
-	var tmp [8]byte
-	for i := uint32(0); i < zb0001; i++ {
-		_, err = io.ReadFull(dc, tmp[:])
-		if err != nil {
-			err = msgp.WrapError(err, "dataUsageHashMap")
-			return
-		}
-		dst[dataUsageHash(binary.LittleEndian.Uint64(tmp[:]))] = struct{}{}
-	}
-	return nil
-}
-func (d dataUsageHashMap) EncodeMsg(en *msgp.Writer) (err error) {
-	err = en.WriteBytesHeader(uint32(len(d)) * dataUsageHashLen)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	var tmp [dataUsageHashLen]byte
-	for k := range d {
-		binary.LittleEndian.PutUint64(tmp[:], uint64(k))
-		_, err = en.Write(tmp[:])
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-	}
-	return nil
+	return
 }

cmd/data-usage-cache_gen.go

@@ -6,6 +6,212 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageCache) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Info":
+			err = z.Info.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "Info")
+				return
+			}
+		case "Cache":
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Cache")
+				return
+			}
+			if z.Cache == nil {
+				z.Cache = make(map[string]dataUsageEntry, zb0002)
+			} else if len(z.Cache) > 0 {
+				for key := range z.Cache {
+					delete(z.Cache, key)
+				}
+			}
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 dataUsageEntry
+				za0001, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Cache")
+					return
+				}
+				err = za0002.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache", za0001)
+					return
+				}
+				z.Cache[za0001] = za0002
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *dataUsageCache) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "Info"
+	err = en.Append(0x82, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
+	if err != nil {
+		return
+	}
+	err = z.Info.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Info")
+		return
+	}
+	// write "Cache"
+	err = en.Append(0xa5, 0x43, 0x61, 0x63, 0x68, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteMapHeader(uint32(len(z.Cache)))
+	if err != nil {
+		err = msgp.WrapError(err, "Cache")
+		return
+	}
+	for za0001, za0002 := range z.Cache {
+		err = en.WriteString(za0001)
+		if err != nil {
+			err = msgp.WrapError(err, "Cache")
+			return
+		}
+		err = za0002.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Cache", za0001)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *dataUsageCache) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "Info"
+	o = append(o, 0x82, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
+	o, err = z.Info.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Info")
+		return
+	}
+	// string "Cache"
+	o = append(o, 0xa5, 0x43, 0x61, 0x63, 0x68, 0x65)
+	o = msgp.AppendMapHeader(o, uint32(len(z.Cache)))
+	for za0001, za0002 := range z.Cache {
+		o = msgp.AppendString(o, za0001)
+		o, err = za0002.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Cache", za0001)
+			return
+		}
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageCache) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Info":
+			bts, err = z.Info.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Info")
+				return
+			}
+		case "Cache":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Cache")
+				return
+			}
+			if z.Cache == nil {
+				z.Cache = make(map[string]dataUsageEntry, zb0002)
+			} else if len(z.Cache) > 0 {
+				for key := range z.Cache {
+					delete(z.Cache, key)
+				}
+			}
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 dataUsageEntry
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache")
+					return
+				}
+				bts, err = za0002.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache", za0001)
+					return
+				}
+				z.Cache[za0001] = za0002
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *dataUsageCache) Msgsize() (s int) {
+	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize
+	if z.Cache != nil {
+		for za0001, za0002 := range z.Cache {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
+		}
+	}
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
@@ -373,8 +579,8 @@ func (z *dataUsageEntry) Msgsize() (s int) {
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
 	{
-		var zb0001 uint64
-		zb0001, err = dc.ReadUint64()
+		var zb0001 string
+		zb0001, err = dc.ReadString()
 		if err != nil {
 			err = msgp.WrapError(err)
 			return
@@ -386,7 +592,7 @@ func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z dataUsageHash) EncodeMsg(en *msgp.Writer) (err error) {
-	err = en.WriteUint64(uint64(z))
+	err = en.WriteString(string(z))
 	if err != nil {
 		err = msgp.WrapError(err)
 		return
@@ -397,15 +603,15 @@ func (z dataUsageHash) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z dataUsageHash) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	o = msgp.AppendUint64(o, uint64(z))
+	o = msgp.AppendString(o, string(z))
 	return
 }
 
 // UnmarshalMsg implements msgp.Unmarshaler
 func (z *dataUsageHash) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	{
-		var zb0001 uint64
-		zb0001, bts, err = msgp.ReadUint64Bytes(bts)
+		var zb0001 string
+		zb0001, bts, err = msgp.ReadStringBytes(bts)
 		if err != nil {
 			err = msgp.WrapError(err)
 			return
@@ -418,7 +624,7 @@ func (z *dataUsageHash) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z dataUsageHash) Msgsize() (s int) {
-	s = msgp.Uint64Size
+	s = msgp.StringPrefixSize + len(string(z))
 	return
 }
 

cmd/data-usage-cache_gen_test.go

@@ -9,6 +9,119 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
+func TestMarshalUnmarshaldataUsageCache(t *testing.T) {
+	v := dataUsageCache{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgdataUsageCache(b *testing.B) {
+	v := dataUsageCache{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgdataUsageCache(b *testing.B) {
+	v := dataUsageCache{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaldataUsageCache(b *testing.B) {
+	v := dataUsageCache{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodedataUsageCache(t *testing.T) {
+	v := dataUsageCache{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodedataUsageCache Msgsize() is inaccurate")
+	}
+
+	vn := dataUsageCache{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodedataUsageCache(b *testing.B) {
+	v := dataUsageCache{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodedataUsageCache(b *testing.B) {
+	v := dataUsageCache{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshaldataUsageCacheInfo(t *testing.T) {
 	v := dataUsageCacheInfo{}
 	bts, err := v.MarshalMsg(nil)

cmd/data-usage.go

@@ -19,21 +19,11 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"encoding/binary"
 	"encoding/json"
-	"errors"
-	"os"
-	"path"
-	"strconv"
-	"time"
 
 	jsoniter "github.com/json-iterator/go"
-	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/color"
-	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/hash"
-	"github.com/willf/bloom"
 )
 
 const (
@@ -44,76 +34,11 @@ const (
 	dataUsageRoot   = SlashSeparator
 	dataUsageBucket = minioMetaBucket + SlashSeparator + bucketMetaPrefix
 
-	dataUsageObjName         = ".usage.json"
-	dataUsageCacheName       = ".usage-cache.bin"
-	dataUsageBloomName       = ".bloomcycle.bin"
-	dataUsageSleepPerFolder  = 1 * time.Millisecond
-	dataUsageSleepDefMult    = 10.0
-	dataUsageUpdateDirCycles = 16
-	dataUsageStartDelay      = 5 * time.Minute // Time to wait on startup and between cycles.
+	dataUsageObjName   = ".usage.json"
+	dataUsageCacheName = ".usage-cache.bin"
+	dataUsageBloomName = ".bloomcycle.bin"
 )
 
-// initDataUsageStats will start the crawler unless disabled.
-func initDataUsageStats(ctx context.Context, objAPI ObjectLayer) {
-	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
-		go runDataUsageInfo(ctx, objAPI)
-	}
-}
-
-func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer) {
-	// Load current bloom cycle
-	nextBloomCycle := intDataUpdateTracker.current() + 1
-	var buf bytes.Buffer
-	err := objAPI.GetObject(ctx, dataUsageBucket, dataUsageBloomName, 0, -1, &buf, "", ObjectOptions{})
-	if err != nil {
-		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) {
-			logger.LogIf(ctx, err)
-		}
-	} else {
-		if buf.Len() == 8 {
-			nextBloomCycle = binary.LittleEndian.Uint64(buf.Bytes())
-		}
-	}
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.NewTimer(dataUsageStartDelay).C:
-			// Wait before starting next cycle and wait on startup.
-			results := make(chan DataUsageInfo, 1)
-			go storeDataUsageInBackend(ctx, objAPI, results)
-			bf, err := globalNotificationSys.updateBloomFilter(ctx, nextBloomCycle)
-			logger.LogIf(ctx, err)
-			err = objAPI.CrawlAndGetDataUsage(ctx, bf, results)
-			close(results)
-			logger.LogIf(ctx, err)
-			if err == nil {
-				// Store new cycle...
-				nextBloomCycle++
-				if nextBloomCycle%dataUpdateTrackerResetEvery == 0 {
-					if intDataUpdateTracker.debug {
-						logger.Info(color.Green("runDataUsageInfo:") + " Resetting bloom filter for next runs.")
-					}
-					nextBloomCycle++
-				}
-				var tmp [8]byte
-				binary.LittleEndian.PutUint64(tmp[:], nextBloomCycle)
-				r, err := hash.NewReader(bytes.NewReader(tmp[:]), int64(len(tmp)), "", "", int64(len(tmp)), false)
-				if err != nil {
-					logger.LogIf(ctx, err)
-					continue
-				}
-
-				_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageBloomName, NewPutObjReader(r, nil, nil), ObjectOptions{})
-				if !isErrBucketNotFound(err) {
-					logger.LogIf(ctx, err)
-				}
-			}
-		}
-	}
-}
-
 // storeDataUsageInBackend will store all objects sent on the gui channel until closed.
 func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, gui <-chan DataUsageInfo) {
 	for dataUsageInfo := range gui {
@@ -172,354 +97,3 @@ func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsag
 
 	return dataUsageInfo, nil
 }
-
-// Item represents each file while walking.
-type Item struct {
-	Path string
-	Typ  os.FileMode
-}
-
-type getSizeFn func(item Item) (int64, error)
-
-type cachedFolder struct {
-	name   string
-	parent *dataUsageHash
-}
-
-type folderScanner struct {
-	root               string
-	getSize            getSizeFn
-	oldCache           dataUsageCache
-	newCache           dataUsageCache
-	withFilter         *bloomFilter
-	waitForLowActiveIO func()
-
-	dataUsageCrawlMult  float64
-	dataUsageCrawlDebug bool
-
-	newFolders      []cachedFolder
-	existingFolders []cachedFolder
-}
-
-// sleepDuration multiplies the duration d by x and sleeps if is more than 100 micro seconds.
-// sleep is limited to max 1 second.
-func sleepDuration(d time.Duration, x float64) {
-	// Don't sleep for really small amount of time
-	if d := time.Duration(float64(d) * x); d > time.Microsecond*100 {
-		if d > time.Second {
-			d = time.Second
-		}
-		time.Sleep(d)
-	}
-}
-
-// scanQueuedLevels will scan the provided folders.
-// Files found in the folders will be added to f.newCache.
-// If final is provided folders will be put into f.newFolders or f.existingFolders.
-// If final is not provided the folders found are returned from the function.
-func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool) ([]cachedFolder, error) {
-	var nextFolders []cachedFolder
-	done := ctx.Done()
-	for _, folder := range folders {
-		select {
-		case <-done:
-			return nil, ctx.Err()
-		default:
-		}
-		thisHash := hashPath(folder.name)
-
-		if _, ok := f.oldCache.Cache[thisHash]; f.withFilter != nil && ok {
-			// If folder isn't in filter and we have data, skip it completely.
-			if folder.name != dataUsageRoot && !f.withFilter.containsDir(folder.name) {
-				f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
-				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("data-usage:")+" Skipping non-updated folder: %v", folder.name)
-				}
-				continue
-			}
-		}
-		f.waitForLowActiveIO()
-		sleepDuration(dataUsageSleepPerFolder, f.dataUsageCrawlMult)
-
-		cache := dataUsageEntry{}
-
-		err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error {
-			// Parse
-			entName = path.Clean(path.Join(folder.name, entName))
-			bucket, _ := path2BucketObjectWithBasePath(f.root, entName)
-			if bucket == "" {
-				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("data-usage:")+" no bucket (%s,%s)", f.root, entName)
-				}
-				return nil
-			}
-
-			if isReservedOrInvalidBucket(bucket, false) {
-				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("data-usage:")+" invalid bucket: %v, entry: %v", bucket, entName)
-				}
-				return nil
-			}
-
-			select {
-			case <-done:
-				return ctx.Err()
-			default:
-			}
-
-			if typ&os.ModeDir != 0 {
-				h := hashPath(entName)
-				_, exists := f.oldCache.Cache[h]
-				cache.addChildString(entName)
-
-				this := cachedFolder{name: entName, parent: &thisHash}
-				cache.addChild(h)
-				if final {
-					if exists {
-						f.existingFolders = append(f.existingFolders, this)
-					} else {
-						f.newFolders = append(f.newFolders, this)
-					}
-				} else {
-					nextFolders = append(nextFolders, this)
-				}
-				return nil
-			}
-			f.waitForLowActiveIO()
-			// Dynamic time delay.
-			t := UTCNow()
-
-			// Get file size, ignore errors.
-			size, err := f.getSize(Item{Path: path.Join(f.root, entName), Typ: typ})
-
-			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
-			if err == errSkipFile || err == errFileNotFound {
-				return nil
-			}
-			logger.LogIf(ctx, err)
-			cache.Size += size
-			cache.Objects++
-			cache.ObjSizes.add(size)
-
-			return nil
-		})
-		if err != nil {
-			return nil, err
-		}
-		f.newCache.replaceHashed(thisHash, folder.parent, cache)
-	}
-	return nextFolders, nil
-}
-
-// deepScanFolder will deep scan a folder and return the size if no error occurs.
-func (f *folderScanner) deepScanFolder(ctx context.Context, folder string) (*dataUsageEntry, error) {
-	var cache dataUsageEntry
-
-	done := ctx.Done()
-
-	var addDir func(entName string, typ os.FileMode) error
-	var dirStack = []string{f.root, folder}
-
-	addDir = func(entName string, typ os.FileMode) error {
-		select {
-		case <-done:
-			return ctx.Err()
-		default:
-		}
-
-		f.waitForLowActiveIO()
-		if typ&os.ModeDir != 0 {
-			dirStack = append(dirStack, entName)
-			err := readDirFn(path.Join(dirStack...), addDir)
-			dirStack = dirStack[:len(dirStack)-1]
-			sleepDuration(dataUsageSleepPerFolder, f.dataUsageCrawlMult)
-			return err
-		}
-
-		// Dynamic time delay.
-		t := UTCNow()
-
-		// Get file size, ignore errors.
-		dirStack = append(dirStack, entName)
-		fileName := path.Join(dirStack...)
-		dirStack = dirStack[:len(dirStack)-1]
-
-		size, err := f.getSize(Item{Path: fileName, Typ: typ})
-
-		// Don't sleep for really small amount of time
-		sleepDuration(time.Since(t), f.dataUsageCrawlMult)
-
-		if err == errSkipFile {
-			return nil
-		}
-		logger.LogIf(ctx, err)
-		cache.Size += size
-		cache.Objects++
-		cache.ObjSizes.add(size)
-		return nil
-	}
-	err := readDirFn(path.Join(dirStack...), addDir)
-	if err != nil {
-		return nil, err
-	}
-	return &cache, nil
-}
-
-// updateUsage will crawl the basepath+cache.Info.Name and return an updated cache.
-// The returned cache will always be valid, but may not be updated from the existing.
-// Before each operation waitForLowActiveIO is called which can be used to temporarily halt the crawler.
-// If the supplied context is canceled the function will return at the first chance.
-func updateUsage(ctx context.Context, basePath string, cache dataUsageCache, waitForLowActiveIO func(), getSize getSizeFn) (dataUsageCache, error) {
-	t := UTCNow()
-
-	dataUsageDebug := env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn
-	logPrefix := color.Green("data-usage: ")
-	logSuffix := color.Blue(" - %v + %v", basePath, cache.Info.Name)
-	if dataUsageDebug {
-		defer func() {
-			logger.Info(logPrefix+" Crawl time: %v"+logSuffix, time.Since(t))
-		}()
-
-	}
-
-	if cache.Info.Name == "" {
-		cache.Info.Name = dataUsageRoot
-	}
-
-	delayMult, err := strconv.ParseFloat(env.Get(envDataUsageCrawlDelay, "10.0"), 64)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		delayMult = dataUsageSleepDefMult
-	}
-
-	s := folderScanner{
-		root:                basePath,
-		getSize:             getSize,
-		oldCache:            cache,
-		newCache:            dataUsageCache{Info: cache.Info},
-		waitForLowActiveIO:  waitForLowActiveIO,
-		newFolders:          nil,
-		existingFolders:     nil,
-		dataUsageCrawlMult:  delayMult,
-		dataUsageCrawlDebug: dataUsageDebug,
-	}
-
-	if len(cache.Info.BloomFilter) > 0 {
-		s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
-		_, err := s.withFilter.ReadFrom(bytes.NewBuffer(cache.Info.BloomFilter))
-		if err != nil {
-			logger.LogIf(ctx, err, logPrefix+"Error reading bloom filter")
-			s.withFilter = nil
-		}
-	}
-	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Start crawling. Bloom filter: %v"+logSuffix, s.withFilter != nil)
-	}
-
-	done := ctx.Done()
-	var flattenLevels = 3
-
-	// If we are scanning inside a bucket reduce depth by 1.
-	if cache.Info.Name != dataUsageRoot {
-		flattenLevels--
-	}
-
-	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Cycle: %v, Entries: %v"+logSuffix, cache.Info.NextCycle, len(cache.Cache))
-	}
-
-	// Always scan flattenLevels deep. Cache root is level 0.
-	todo := []cachedFolder{{name: cache.Info.Name}}
-	for i := 0; i < flattenLevels; i++ {
-		if s.dataUsageCrawlDebug {
-			logger.Info(logPrefix+"Level %v, scanning %v directories."+logSuffix, i, len(todo))
-		}
-		select {
-		case <-done:
-			return cache, ctx.Err()
-		default:
-		}
-		var err error
-		todo, err = s.scanQueuedLevels(ctx, todo, i == flattenLevels-1)
-		if err != nil {
-			// No useful information...
-			return cache, err
-		}
-	}
-
-	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"New folders: %v"+logSuffix, s.newFolders)
-	}
-
-	// Add new folders first
-	for _, folder := range s.newFolders {
-		select {
-		case <-done:
-			return s.newCache, ctx.Err()
-		default:
-		}
-		du, err := s.deepScanFolder(ctx, folder.name)
-		if err != nil {
-			logger.LogIf(ctx, err)
-			continue
-		}
-		if du == nil {
-			logger.Info(logPrefix + "no disk usage provided" + logSuffix)
-			continue
-		}
-
-		s.newCache.replace(folder.name, "", *du)
-		// Add to parent manually
-		if folder.parent != nil {
-			parent := s.newCache.Cache[*folder.parent]
-			parent.addChildString(folder.name)
-		}
-	}
-
-	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Existing folders: %v"+logSuffix, len(s.existingFolders))
-	}
-
-	// Do selective scanning of existing folders.
-	for _, folder := range s.existingFolders {
-		select {
-		case <-done:
-			return s.newCache, ctx.Err()
-		default:
-		}
-		h := hashPath(folder.name)
-		if !h.mod(s.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
-			s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h])
-			continue
-		}
-
-		if s.withFilter != nil {
-			// If folder isn't in filter, skip it completely.
-			if !s.withFilter.containsDir(folder.name) {
-				if s.dataUsageCrawlDebug {
-					logger.Info(logPrefix+"Skipping non-updated folder: %v"+logSuffix, folder)
-				}
-				s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h])
-				continue
-			}
-		}
-
-		// Update on this cycle...
-		du, err := s.deepScanFolder(ctx, folder.name)
-		if err != nil {
-			logger.LogIf(ctx, err)
-			continue
-		}
-		if du == nil {
-			logger.LogIf(ctx, errors.New("data-usage: no disk usage provided"))
-			continue
-		}
-		s.newCache.replaceHashed(h, folder.parent, *du)
-	}
-	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Finished crawl, %v entries"+logSuffix, len(s.newCache.Cache))
-	}
-	s.newCache.Info.LastUpdate = UTCNow()
-	s.newCache.Info.NextCycle++
-	return s.newCache, nil
-}

cmd/data-usage_test.go

@@ -17,9 +17,12 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"testing"
 )
@@ -34,6 +37,7 @@ func TestDataUsageUpdate(t *testing.T) {
 	if err != nil {
 		t.Skip(err)
 	}
+	const bucket = "bucket"
 	defer os.RemoveAll(base)
 	var files = []usageTestFile{
 		{name: "rootfile", size: 10000},
@@ -45,9 +49,9 @@ func TestDataUsageUpdate(t *testing.T) {
 		{name: "dir1/dira/dirasub/dcfile", size: 1000000},
 		{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
 	}
-	createUsageTestFiles(t, base, files)
+	createUsageTestFiles(t, base, bucket, files)
 
-	getSize := func(item Item) (i int64, err error) {
+	getSize := func(item crawlItem) (i int64, err error) {
 		if item.Typ&os.ModeDir == 0 {
 			s, err := os.Stat(item.Path)
 			if err != nil {
@@ -58,7 +62,7 @@ func TestDataUsageUpdate(t *testing.T) {
 		return 0, nil
 	}
 
-	got, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize)
+	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, func() {}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -105,16 +109,17 @@ func TestDataUsageUpdate(t *testing.T) {
 			oSizes:  sizeHistogram{0: 1, 1: 3},
 		},
 		{
-			path:   "/dir1/dira",
-			size:   300000,
-			objs:   2,
-			oSizes: sizeHistogram{0: 0, 1: 2},
+			path:   "/dir1",
+			size:   2000,
+			objs:   1,
+			oSizes: sizeHistogram{0: 0, 1: 1},
 		},
 		{
+			// Children are flattened
 			path:   "/dir1/dira/",
-			size:   300000,
-			objs:   2,
-			oSizes: sizeHistogram{0: 0, 1: 2},
+			size:   1300010,
+			objs:   4,
+			oSizes: sizeHistogram{0: 1, 1: 3},
 		},
 		{
 			path:  "/nonexistying",
@@ -123,8 +128,9 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 
 	for _, w := range want {
-		t.Run(w.path, func(t *testing.T) {
-			e := got.find(w.path)
+		p := path.Join(bucket, w.path)
+		t.Run(p, func(t *testing.T) {
+			e := got.find(p)
 			if w.isNil {
 				if e != nil {
 					t.Error("want nil, got", e)
@@ -134,6 +140,7 @@ func TestDataUsageUpdate(t *testing.T) {
 			if e == nil {
 				t.Fatal("got nil result")
 			}
+			t.Log(e.Children)
 			if w.flatten {
 				*e = got.flatten(*e)
 			}
@@ -175,8 +182,8 @@ func TestDataUsageUpdate(t *testing.T) {
 			size: 1000,
 		},
 	}
-	createUsageTestFiles(t, base, files)
-	got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+	createUsageTestFiles(t, base, bucket, files)
+	got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -222,7 +229,7 @@ func TestDataUsageUpdate(t *testing.T) {
 
 	for _, w := range want {
 		t.Run(w.path, func(t *testing.T) {
-			e := got.find(w.path)
+			e := got.find(path.Join(bucket, w.path))
 			if w.isNil {
 				if e != nil {
 					t.Error("want nil, got", e)
@@ -233,6 +240,7 @@ func TestDataUsageUpdate(t *testing.T) {
 				t.Fatal("got nil result")
 			}
 			if w.flatten {
+				t.Log(e.Children)
 				*e = got.flatten(*e)
 			}
 			if e.Size != int64(w.size) {
@@ -254,14 +262,14 @@ func TestDataUsageUpdate(t *testing.T) {
 		},
 	}
 
-	createUsageTestFiles(t, base, files)
-	err = os.RemoveAll(filepath.Join(base, "dir1/dira/dirasub/dcfile"))
+	createUsageTestFiles(t, base, bucket, files)
+	err = os.RemoveAll(filepath.Join(base, bucket, "dir1/dira/dirasub/dcfile"))
 	if err != nil {
 		t.Fatal(err)
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+		got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -291,8 +299,9 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 
 	for _, w := range want {
-		t.Run(w.path, func(t *testing.T) {
-			e := got.find(w.path)
+		p := path.Join(bucket, w.path)
+		t.Run(p, func(t *testing.T) {
+			e := got.find(p)
 			if w.isNil {
 				if e != nil {
 					t.Error("want nil, got", e)
@@ -335,9 +344,9 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		{name: "bucket/dir1/dira/dirasub/dcfile", size: 1000000},
 		{name: "bucket/dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
 	}
-	createUsageTestFiles(t, base, files)
+	createUsageTestFiles(t, base, "", files)
 
-	getSize := func(item Item) (i int64, err error) {
+	getSize := func(item crawlItem) (i int64, err error) {
 		if item.Typ&os.ModeDir == 0 {
 			s, err := os.Stat(item.Path)
 			if err != nil {
@@ -347,7 +356,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		}
 		return 0, nil
 	}
-	got, err := updateUsage(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, func() {}, getSize)
+	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, func() {}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -449,8 +458,8 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 			size: 1000,
 		},
 	}
-	createUsageTestFiles(t, base, files)
-	got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+	createUsageTestFiles(t, base, "", files)
+	got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -526,14 +535,14 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		},
 	}
 
-	createUsageTestFiles(t, base, files)
+	createUsageTestFiles(t, base, "", files)
 	err = os.RemoveAll(filepath.Join(base, "bucket/dir1/dira/dirasub/dcfile"))
 	if err != nil {
 		t.Fatal(err)
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+		got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -588,13 +597,13 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 }
 
-func createUsageTestFiles(t *testing.T, base string, files []usageTestFile) {
+func createUsageTestFiles(t *testing.T, base, bucket string, files []usageTestFile) {
 	for _, f := range files {
-		err := os.MkdirAll(filepath.Dir(filepath.Join(base, f.name)), os.ModePerm)
+		err := os.MkdirAll(filepath.Dir(filepath.Join(base, bucket, f.name)), os.ModePerm)
 		if err != nil {
 			t.Fatal(err)
 		}
-		err = ioutil.WriteFile(filepath.Join(base, f.name), make([]byte, f.size), os.ModePerm)
+		err = ioutil.WriteFile(filepath.Join(base, bucket, f.name), make([]byte, f.size), os.ModePerm)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -606,20 +615,28 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 	if err != nil {
 		t.Skip(err)
 	}
+	const bucket = "abucket"
 	defer os.RemoveAll(base)
 	var files = []usageTestFile{
 		{name: "rootfile", size: 10000},
 		{name: "rootfile2", size: 10000},
 		{name: "dir1/d1file", size: 2000},
 		{name: "dir2/d2file", size: 300},
+		{name: "dir2/d2file2", size: 300},
+		{name: "dir2/d2file3/", size: 300},
+		{name: "dir2/d2file4/", size: 300},
+		{name: "dir2/d2file5", size: 300},
 		{name: "dir1/dira/dafile", size: 100000},
 		{name: "dir1/dira/dbfile", size: 200000},
 		{name: "dir1/dira/dirasub/dcfile", size: 1000000},
 		{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
+		{name: "dir1/dira/dirasub/sublevel3/dccccfile20", size: 20},
+		{name: "dir1/dira/dirasub/sublevel3/dccccfile30", size: 30},
+		{name: "dir1/dira/dirasub/sublevel3/dccccfile40", size: 40},
 	}
-	createUsageTestFiles(t, base, files)
+	createUsageTestFiles(t, base, bucket, files)
 
-	getSize := func(item Item) (i int64, err error) {
+	getSize := func(item crawlItem) (i int64, err error) {
 		if item.Typ&os.ModeDir == 0 {
 			s, err := os.Stat(item.Path)
 			if err != nil {
@@ -629,23 +646,33 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 		}
 		return 0, nil
 	}
-	want, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize)
+	want, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, func() {}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	b := want.serialize()
 	var got dataUsageCache
-	err = got.deserialize(b)
+	err = got.deserialize(bytes.NewBuffer(b))
 	if err != nil {
 		t.Fatal(err)
 	}
-
+	t.Log("serialized size:", len(b), "bytes")
 	if got.Info.LastUpdate.IsZero() {
 		t.Error("lastupdate not set")
 	}
 
 	if !want.Info.LastUpdate.Equal(got.Info.LastUpdate) {
-		t.Fatalf("deserialize mismatch\nwant: %+v\ngot:  %+v", want, got)
+		t.Fatalf("deserialize LastUpdate mismatch\nwant: %+v\ngot:  %+v", want, got)
 	}
+	if len(want.Cache) != len(got.Cache) {
+		t.Errorf("deserialize mismatch length\nwant: %+v\ngot:  %+v", len(want.Cache), len(got.Cache))
+	}
+	for wkey, wval := range want.Cache {
+		gotv := got.Cache[wkey]
+		if fmt.Sprint(gotv) != fmt.Sprint(wval) {
+			t.Errorf("deserialize mismatch, key %v\nwant: %+v\ngot:  %+v", wkey, wval, gotv)
+		}
+	}
+
 }

cmd/disk-cache-backend.go

@@ -33,6 +33,7 @@ import (
 	"time"
 
 	"github.com/djherbis/atime"
+	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -125,16 +126,19 @@ func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
 
 // represents disk cache struct
 type diskCache struct {
-	gcCounter uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
 	// is set to 0 if drive is offline
-	online uint32
+	online       uint32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
+	purgeRunning int32
 
-	dir           string // caching directory
-	quotaPct      int    // max usage in %
+	triggerGC     chan struct{}
+	dir           string         // caching directory
+	stats         CacheDiskStats // disk cache stats for prometheus
+	quotaPct      int            // max usage in %
 	pool          sync.Pool
 	after         int // minimum accesses before an object is cached.
 	lowWatermark  int
 	highWatermark int
+	enableRange   bool
 	// nsMutex namespace lock
 	nsMutex *nsLockMap
 	// Object functions pointing to the corresponding functions of backend implementation.
@@ -142,16 +146,24 @@ type diskCache struct {
 }
 
 // Inits the disk cache dir if it is not initialized already.
-func newDiskCache(dir string, quotaPct, after, lowWatermark, highWatermark int) (*diskCache, error) {
+func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCache, error) {
+	quotaPct := config.MaxUse
+	if quotaPct == 0 {
+		quotaPct = config.Quota
+	}
+
 	if err := os.MkdirAll(dir, 0777); err != nil {
 		return nil, fmt.Errorf("Unable to initialize '%s' dir, %w", dir, err)
 	}
 	cache := diskCache{
 		dir:           dir,
+		triggerGC:     make(chan struct{}),
+		stats:         CacheDiskStats{Dir: dir},
 		quotaPct:      quotaPct,
-		after:         after,
-		lowWatermark:  lowWatermark,
-		highWatermark: highWatermark,
+		after:         config.After,
+		lowWatermark:  config.WatermarkLow,
+		highWatermark: config.WatermarkHigh,
+		enableRange:   config.Range,
 		online:        1,
 		pool: sync.Pool{
 			New: func() interface{} {
@@ -161,6 +173,8 @@ func newDiskCache(dir string, quotaPct, after, lowWatermark, highWatermark int)
 		},
 		nsMutex: newNSLock(false),
 	}
+	go cache.purgeWait(ctx)
+	cache.diskUsageHigh() // update if cache usage is already high.
 	cache.NewNSLockFn = func(ctx context.Context, cachePath string) RWLocker {
 		return cache.nsMutex.NewNSLock(ctx, nil, cachePath, "")
 	}
@@ -181,7 +195,12 @@ func (c *diskCache) diskUsageLow() bool {
 		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) < gcStopPct
+	low := int(usedPercent) < gcStopPct
+	atomic.StoreUint64(&c.stats.UsagePercent, usedPercent)
+	if low {
+		atomic.StoreInt32(&c.stats.UsageState, 0)
+	}
+	return low
 }
 
 // Returns if the disk usage reaches high water mark w.r.t the configured cache quota.
@@ -196,7 +215,12 @@ func (c *diskCache) diskUsageHigh() bool {
 		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) >= gcTriggerPct
+	high := int(usedPercent) >= gcTriggerPct
+	atomic.StoreUint64(&c.stats.UsagePercent, usedPercent)
+	if high {
+		atomic.StoreInt32(&c.stats.UsageState, 1)
+	}
+	return high
 }
 
 // Returns if size space can be allocated without exceeding
@@ -230,24 +254,36 @@ var (
 	errDoneForNow = errors.New("done for now")
 )
 
+func (c *diskCache) purgeWait(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+		case <-c.triggerGC: // wait here until someone triggers.
+			c.purge(ctx)
+		}
+	}
+}
+
 // Purge cache entries that were not accessed.
 func (c *diskCache) purge(ctx context.Context) {
-	if c.diskUsageLow() {
+	if atomic.LoadInt32(&c.purgeRunning) == 1 || c.diskUsageLow() {
 		return
 	}
+
 	toFree := c.toClear()
 	if toFree == 0 {
 		return
 	}
+
+	atomic.StoreInt32(&c.purgeRunning, 1) // do not run concurrent purge()
+	defer atomic.StoreInt32(&c.purgeRunning, 0)
+
 	// expiry for cleaning up old cache.json files that
 	// need to be cleaned up.
 	expiry := UTCNow().Add(-cacheExpiryDays)
 	// defaulting max hits count to 100
-	scorer, err := newFileScorer(toFree, time.Now().Unix(), 100)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return
-	}
+	// ignore error we know what value we are passing.
+	scorer, _ := newFileScorer(toFree, time.Now().Unix(), 100)
 
 	// this function returns FileInfo for cached range files and cache data file.
 	fiStatFn := func(ranges map[string]string, dataFile, pathPrefix string) map[string]os.FileInfo {
@@ -326,27 +362,20 @@ func (c *diskCache) purge(ctx context.Context) {
 		return
 	}
 
-	for _, path := range scorer.fileNames() {
-		removeAll(path)
-		slashIdx := strings.LastIndex(path, SlashSeparator)
-		pathPrefix := path[0:slashIdx]
-		fname := path[slashIdx+1:]
-		if fname == cacheDataFile {
-			removeAll(pathPrefix)
+	scorer.purgeFunc(func(qfile queuedFile) {
+		fileName := qfile.name
+		removeAll(fileName)
+		slashIdx := strings.LastIndex(fileName, SlashSeparator)
+		if slashIdx >= 0 {
+			fileNamePrefix := fileName[0:slashIdx]
+			fname := fileName[slashIdx+1:]
+			if fname == cacheDataFile {
+				removeAll(fileNamePrefix)
+			}
 		}
-	}
-}
+	})
 
-func (c *diskCache) incGCCounter() {
-	atomic.AddUint64(&c.gcCounter, 1)
-}
-
-func (c *diskCache) resetGCCounter() {
-	atomic.StoreUint64(&c.gcCounter, 0)
-}
-
-func (c *diskCache) gcCount() uint64 {
-	return atomic.LoadUint64(&c.gcCounter)
+	scorer.reset()
 }
 
 // sets cache drive status
@@ -630,7 +659,7 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
 // Caches the object to disk
 func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) error {
 	if c.diskUsageHigh() {
-		c.incGCCounter()
+		c.triggerGC <- struct{}{}
 		io.Copy(ioutil.Discard, data)
 		return errDiskFull
 	}

cmd/disk-cache-check-support_other.go

@@ -0,0 +1,56 @@
+// +build !windows
+
+/*
+ * MinIO Cloud Storage, (C) 2019-2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"io"
+	"io/ioutil"
+	"os"
+	"time"
+
+	"github.com/djherbis/atime"
+)
+
+// Return error if Atime is disabled on the O/S
+func checkAtimeSupport(dir string) (err error) {
+	file, err := ioutil.TempFile(dir, "prefix")
+	if err != nil {
+		return
+	}
+	defer os.Remove(file.Name())
+	defer file.Close()
+	finfo1, err := os.Stat(file.Name())
+	if err != nil {
+		return
+	}
+	// add a sleep to ensure atime change is detected
+	time.Sleep(10 * time.Millisecond)
+
+	if _, err = io.Copy(ioutil.Discard, file); err != nil {
+		return
+	}
+
+	finfo2, err := os.Stat(file.Name())
+
+	if atime.Get(finfo2).Equal(atime.Get(finfo1)) {
+		return errors.New("Atime not supported")
+	}
+	return
+}

cmd/disk-cache-check-support_windows.go

@@ -0,0 +1,60 @@
+// +build windows
+
+/*
+ * MinIO Cloud Storage, (C) 2019-2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"io/ioutil"
+	"os"
+
+	"github.com/djherbis/atime"
+	"golang.org/x/sys/windows/registry"
+)
+
+// Return error if Atime is disabled on the O/S
+func checkAtimeSupport(dir string) (err error) {
+	file, err := ioutil.TempFile(dir, "prefix")
+	if err != nil {
+		return
+	}
+	defer os.Remove(file.Name())
+	defer file.Close()
+	finfo1, err := os.Stat(file.Name())
+	if err != nil {
+		return
+	}
+	atime.Get(finfo1)
+
+	k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SYSTEM\CurrentControlSet\Control\FileSystem`, registry.QUERY_VALUE)
+	if err != nil {
+		return
+	}
+	defer k.Close()
+
+	setting, _, err := k.GetIntegerValue("NtfsDisableLastAccessUpdate")
+	if err != nil {
+		return
+	}
+
+	lowSetting := setting & 0xFFFF
+	if lowSetting != uint64(0x0000) && lowSetting != uint64(0x0002) {
+		return errors.New("Atime not supported")
+	}
+	return
+}

cmd/disk-cache-stats.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019, 2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,14 +16,27 @@
 
 package cmd
 
-import "sync/atomic"
+import (
+	"sync/atomic"
+)
+
+// CacheDiskStats represents cache disk statistics
+// such as current disk usage and available.
+type CacheDiskStats struct {
+	// indicates if usage is high or low, if high value is '1', if low its '0'
+	UsageState int32
+	// indicates the current usage percentage of this cache disk
+	UsagePercent uint64
+	Dir          string
+}
 
 // CacheStats - represents bytes served from cache,
 // cache hits and cache misses.
 type CacheStats struct {
-	BytesServed uint64
-	Hits        uint64
-	Misses      uint64
+	BytesServed  uint64
+	Hits         uint64
+	Misses       uint64
+	GetDiskStats func() []CacheDiskStats
 }
 
 // Increase total bytes served from cache

cmd/disk-cache-utils.go

@@ -404,6 +404,14 @@ func (f *fileScorer) trimQueue() {
 	}
 }
 
+func (f *fileScorer) purgeFunc(p func(qfile queuedFile)) {
+	e := f.queue.Front()
+	for e != nil {
+		p(e.Value.(queuedFile))
+		e = e.Next()
+	}
+}
+
 // fileNames returns all queued file names.
 func (f *fileScorer) fileNames() []string {
 	res := make([]string, 0, f.queue.Len())

cmd/disk-cache.go

@@ -21,14 +21,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
-	"os"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
-	"github.com/djherbis/atime"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/logger"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
@@ -267,10 +265,11 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 
 	// Reaching here implies cache miss
 	c.cacheStats.incMiss()
+
 	// Since we got here, we are serving the request from backend,
 	// and also adding the object to the cache.
 	if dcache.diskUsageHigh() {
-		dcache.incGCCounter()
+		dcache.triggerGC <- struct{}{} // this is non-blocking
 	}
 
 	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
@@ -295,6 +294,10 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 
 	if rs != nil {
 		go func() {
+			// if range caching is disabled, download entire object.
+			if !dcache.enableRange {
+				rs = nil
+			}
 			// fill cache in the background for range GET requests
 			bReader, bErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 			if bErr != nil {
@@ -304,7 +307,9 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			oi, _, _, err := dcache.statRange(ctx, bucket, object, rs)
 			// avoid cache overwrite if another background routine filled cache
 			if err != nil || oi.ETag != bReader.ObjInfo.ETag {
-				dcache.Put(ctx, bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
+				// use a new context to avoid locker prematurely timing out operation when the GetObjectNInfo returns.
+				dcache.Put(context.Background(), bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
+				return
 			}
 		}()
 		return bkReader, bkErr
@@ -525,11 +530,7 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 			return nil, false, errors.New("Atime support required for disk caching")
 		}
 
-		quota := config.MaxUse
-		if quota == 0 {
-			quota = config.Quota
-		}
-		cache, err := newDiskCache(dir, quota, config.After, config.WatermarkLow, config.WatermarkHigh)
+		cache, err := newDiskCache(ctx, dir, config)
 		if err != nil {
 			return nil, false, err
 		}
@@ -538,31 +539,6 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 	return caches, migrating, nil
 }
 
-// Return error if Atime is disabled on the O/S
-func checkAtimeSupport(dir string) (err error) {
-	file, err := ioutil.TempFile(dir, "prefix")
-	if err != nil {
-		return
-	}
-	defer os.Remove(file.Name())
-	finfo1, err := os.Stat(file.Name())
-	if err != nil {
-		return
-	}
-	// add a sleep to ensure atime change is detected
-	time.Sleep(10 * time.Millisecond)
-
-	if _, err = io.Copy(ioutil.Discard, file); err != nil {
-		return
-	}
-
-	finfo2, err := os.Stat(file.Name())
-
-	if atime.Get(finfo2).Equal(atime.Get(finfo1)) {
-		return errors.New("Atime not supported")
-	}
-	return
-}
 func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
 	logStartupMessage(color.Blue("Cache migration initiated ...."))
 
@@ -694,7 +670,17 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 			return newObjectLayerFn().CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo, srcOpts, dstOpts)
 		},
 	}
-
+	c.cacheStats.GetDiskStats = func() []CacheDiskStats {
+		cacheDiskStats := make([]CacheDiskStats, len(c.cache))
+		for i := range c.cache {
+			cacheDiskStats[i] = CacheDiskStats{
+				Dir: c.cache[i].stats.Dir,
+			}
+			atomic.StoreInt32(&cacheDiskStats[i].UsageState, atomic.LoadInt32(&c.cache[i].stats.UsageState))
+			atomic.StoreUint64(&cacheDiskStats[i].UsagePercent, atomic.LoadUint64(&c.cache[i].stats.UsagePercent))
+		}
+		return cacheDiskStats
+	}
 	if migrateSw {
 		go c.migrateCacheFromV1toV2(ctx)
 	}
@@ -714,19 +700,9 @@ func (c *cacheObjects) gc(ctx context.Context) {
 			if c.migrating {
 				continue
 			}
-			var wg sync.WaitGroup
 			for _, dcache := range c.cache {
-				if dcache.gcCount() == 0 {
-					continue
-				}
-				wg.Add(1)
-				go func(d *diskCache) {
-					defer wg.Done()
-					d.resetGCCounter()
-					d.purge(ctx)
-				}(dcache)
+				dcache.triggerGC <- struct{}{}
 			}
-			wg.Wait()
 		}
 	}
 }

cmd/dummy-handlers.go

@@ -89,7 +89,7 @@ func (api objectAPIHandlers) GetBucketAccelerateHandler(w http.ResponseWriter, r
 		return
 	}
 
-	const accelerateDefaultConfig = `<?xml version="1.0" encoding="UTF-8"?>\n<AccelerateConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`
+	const accelerateDefaultConfig = `<?xml version="1.0" encoding="UTF-8"?><AccelerateConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`
 	writeSuccessResponseXML(w, []byte(accelerateDefaultConfig))
 }
 
@@ -122,7 +122,7 @@ func (api objectAPIHandlers) GetBucketRequestPaymentHandler(w http.ResponseWrite
 		return
 	}
 
-	const requestPaymentDefaultConfig = `<?xml version="1.0" encoding="UTF-8"?>\n<RequestPaymentConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Payer>BucketOwner</Payer></RequestPaymentConfiguration>`
+	const requestPaymentDefaultConfig = `<?xml version="1.0" encoding="UTF-8"?><RequestPaymentConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Payer>BucketOwner</Payer></RequestPaymentConfiguration>`
 
 	writeSuccessResponseXML(w, []byte(requestPaymentDefaultConfig))
 }
@@ -156,7 +156,7 @@ func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	const loggingDefaultConfig = `<?xml version="1.0" encoding="UTF-8"?>\n\n<BucketLoggingStatus xmlns="http://s3.amazonaws.com/doc/2006-03-01/">\n  <!--<LoggingEnabled><TargetBucket>myLogsBucket</TargetBucket><TargetPrefix>add/this/prefix/to/my/log/files/access_log-</TargetPrefix></LoggingEnabled>-->\n</BucketLoggingStatus>\n`
+	const loggingDefaultConfig = `<?xml version="1.0" encoding="UTF-8"?><BucketLoggingStatus xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><!--<LoggingEnabled><TargetBucket>myLogsBucket</TargetBucket><TargetPrefix>add/this/prefix/to/my/log/files/access_log-</TargetPrefix></LoggingEnabled>--></BucketLoggingStatus>`
 	writeSuccessResponseXML(w, []byte(loggingDefaultConfig))
 }
 

cmd/encryption-v1.go

@@ -903,7 +903,7 @@ func getOpts(ctx context.Context, r *http.Request, bucket, object string) (Objec
 		if err != nil {
 			return opts, err
 		}
-		if partNumber < 0 {
+		if partNumber <= 0 {
 			return opts, errInvalidArgument
 		}
 	}

cmd/endpoint-ellipses.go

@@ -181,6 +181,11 @@ func getSetIndexes(args []string, totalSizes []uint64, customSetDriveCount uint6
 		// Returns possible set counts with symmetry.
 		setCounts = possibleSetCountsWithSymmetry(setCounts, argPatterns)
 
+		if len(setCounts) == 0 {
+			msg := fmt.Sprintf("No symmetric distribution detected with input endpoints provided %s, disks %d cannot be spread symmetrically by any supported erasure set sizes %d", args, commonSize, setSizes)
+			return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg)
+		}
+
 		// Final set size with all the symmetry accounted for.
 		setSize = commonSetDriveCount(commonSize, setCounts)
 	}

cmd/endpoint-ellipses_test.go

@@ -211,6 +211,12 @@ func TestGetSetIndexes(t *testing.T) {
 			nil,
 			false,
 		},
+		{
+			[]string{"data{1...17}/export{1...52}"},
+			[]uint64{14144},
+			nil,
+			false,
+		},
 		// Valid inputs.
 		{
 			[]string{"data{1...27}"},
@@ -272,6 +278,12 @@ func TestGetSetIndexes(t *testing.T) {
 			[][]uint64{{10}, {10}, {10}},
 			true,
 		},
+		{
+			[]string{"data{1...16}/export{1...52}"},
+			[]uint64{832},
+			[][]uint64{{16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16}},
+			true,
+		},
 	}
 
 	for _, testCase := range testCases {

cmd/endpoint.go

@@ -44,8 +44,6 @@ const (
 
 	// URLEndpointType - URL style endpoint type enum.
 	URLEndpointType
-
-	retryInterval = 5 // In Seconds.
 )
 
 // Endpoint - any type of endpoint.
@@ -302,7 +300,7 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 	resolvedList := make([]bool, len(endpoints))
 	// Mark the starting time
 	startTime := time.Now()
-	keepAliveTicker := time.NewTicker(retryInterval * time.Second)
+	keepAliveTicker := time.NewTicker(10 * time.Millisecond)
 	defer keepAliveTicker.Stop()
 	for {
 		// Break if the local endpoint is found already Or all the endpoints are resolved.

cmd/erasure-decode.go

@@ -145,9 +145,9 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 		wg.Add(1)
 		go func(i int) {
 			defer wg.Done()
-			disk := p.readers[i]
-			if disk == nil {
-				// Since disk is nil, trigger another read.
+			rr := p.readers[i]
+			if rr == nil {
+				// Since reader is nil, trigger another read.
 				readTriggerCh <- true
 				return
 			}
@@ -160,7 +160,7 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 			// For the last shard, the shardsize might be less than previous shard sizes.
 			// Hence the following statement ensures that the buffer size is reset to the right size.
 			p.buf[bufIdx] = p.buf[bufIdx][:p.shardSize]
-			_, err := disk.ReadAt(p.buf[bufIdx], p.offset)
+			_, err := rr.ReadAt(p.buf[bufIdx], p.offset)
 			if err != nil {
 				if _, ok := err.(*errHashMismatch); ok {
 					atomic.StoreInt32(&healRequired, 1)

cmd/format-fs.go

@@ -27,6 +27,7 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
+	"github.com/minio/minio/pkg/retry"
 )
 
 // FS format version strings.
@@ -344,7 +345,7 @@ func formatFSFixDeploymentID(ctx context.Context, fsFormatPath string) error {
 	defer cancel()
 
 	var wlk *lock.LockedFile
-	retryCh := newRetryTimerSimple(retryCtx)
+	retryCh := retry.NewTimerWithJitter(retryCtx, time.Second, 30*time.Second, retry.MaxJitter)
 	var stop bool
 	for !stop {
 		select {

cmd/fs-v1-helpers.go

@@ -363,11 +363,15 @@ func fsCreateFile(ctx context.Context, filePath string, reader io.Reader, buf []
 // wrapper to handle various operating system specific errors.
 func fsFAllocate(fd int, offset int64, len int64) (err error) {
 	e := Fallocate(fd, offset, len)
-	// Ignore errors when Fallocate is not supported in the current system
-	if e != nil && !isSysErrNoSys(e) && !isSysErrOpNotSupported(e) {
+	if e != nil {
 		switch {
 		case isSysErrNoSpace(e):
 			err = errDiskFull
+		case isSysErrNoSys(e) || isSysErrOpNotSupported(e):
+			// Ignore errors when Fallocate is not supported in the current system
+		case isSysErrInvalidArg(e):
+			// Workaround for Windows Docker Engine 19.03.8.
+			// See https://github.com/minio/minio/issues/9726
 		case isSysErrIO(e):
 			err = e
 		default:

cmd/fs-v1-multipart.go

@@ -731,9 +731,18 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		logger.LogIf(ctx, err)
 		return oi, toObjectErr(err, bucket, object)
 	}
-	fsRemoveAll(ctx, uploadIDDir)
-	// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
-	fsRemoveDir(ctx, fs.getMultipartSHADir(bucket, object))
+
+	// Purge multipart folders
+	{
+		fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, mustGetUUID())
+		defer fsRemoveAll(ctx, fsTmpObjPath) // remove multipart temporary files in background.
+
+		fsSimpleRenameFile(ctx, uploadIDDir, fsTmpObjPath)
+
+		// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
+		fsRemoveDir(ctx, fs.getMultipartSHADir(bucket, object))
+	}
+
 	fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object))
 	if err != nil {
 		return oi, toObjectErr(err, bucket, object)
@@ -782,12 +791,16 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 		return toObjectErr(err, bucket, object)
 	}
 
-	// Ignore the error returned as Windows fails to remove directory if a file in it
-	// is Open()ed by the backgroundAppend()
-	fsRemoveAll(ctx, uploadIDDir)
+	// Purge multipart folders
+	{
+		fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, mustGetUUID())
+		defer fsRemoveAll(ctx, fsTmpObjPath) // remove multipart temporary files in background.
 
-	// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
-	fsRemoveDir(ctx, fs.getMultipartSHADir(bucket, object))
+		fsSimpleRenameFile(ctx, uploadIDDir, fsTmpObjPath)
+
+		// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
+		fsRemoveDir(ctx, fs.getMultipartSHADir(bucket, object))
+	}
 
 	return nil
 }

cmd/fs-v1.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -39,6 +38,7 @@ import (
 	"github.com/minio/minio/cmd/config"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/madmin"
@@ -238,66 +238,130 @@ func (fs *FSObjects) waitForLowActiveIO() {
 // CrawlAndGetDataUsage returns data usage stats of the current FS deployment
 func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo) error {
 	// Load bucket totals
-	var oldCache dataUsageCache
-	err := oldCache.load(ctx, fs, dataUsageCacheName)
+	var totalCache dataUsageCache
+	err := totalCache.load(ctx, fs, dataUsageCacheName)
 	if err != nil {
 		return err
 	}
-	if oldCache.Info.Name == "" {
-		oldCache.Info.Name = dataUsageRoot
-	}
+	totalCache.Info.Name = dataUsageRoot
 	buckets, err := fs.ListBuckets(ctx)
 	if err != nil {
 		return err
 	}
-	oldCache.Info.BloomFilter = nil
+	totalCache.Info.BloomFilter = nil
 	if bf != nil {
-		oldCache.Info.BloomFilter = bf.bytes()
+		totalCache.Info.BloomFilter = bf.bytes()
 	}
 
-	if false && intDataUpdateTracker.debug {
-		b, _ := json.MarshalIndent(bf, "", "  ")
-		logger.Info("Bloom filter: %v", string(b))
+	// Clear totals.
+	var root dataUsageEntry
+	if r := totalCache.root(); r != nil {
+		root.Children = r.Children
 	}
-	cache, err := updateUsage(ctx, fs.fsPath, oldCache, fs.waitForLowActiveIO, func(item Item) (int64, error) {
-		bucket, object := path2BucketObject(strings.TrimPrefix(item.Path, fs.fsPath))
+	totalCache.replace(dataUsageRoot, "", root)
+
+	// Delete all buckets that does not exist anymore.
+	totalCache.keepBuckets(buckets)
+
+	for _, b := range buckets {
+		// Load bucket cache.
+		var bCache dataUsageCache
+		err := bCache.load(ctx, fs, path.Join(b.Name, dataUsageCacheName))
+		if err != nil {
+			return err
+		}
+		if bCache.Info.Name == "" {
+			bCache.Info.Name = b.Name
+		}
+		bCache.Info.BloomFilter = totalCache.Info.BloomFilter
+
+		cache, err := fs.crawlBucket(ctx, b.Name, bCache)
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+		logger.LogIf(ctx, err)
+		cache.Info.BloomFilter = nil
+
+		if cache.root() == nil {
+			if intDataUpdateTracker.debug {
+				logger.Info(color.Green("CrawlAndGetDataUsage:") + " No root added. Adding empty")
+			}
+			cache.replace(cache.Info.Name, dataUsageRoot, dataUsageEntry{})
+		}
+		if cache.Info.LastUpdate.After(bCache.Info.LastUpdate) {
+			if intDataUpdateTracker.debug {
+				logger.Info(color.Green("CrawlAndGetDataUsage:")+" Saving bucket %q cache with %d entries", b.Name, len(cache.Cache))
+			}
+			logger.LogIf(ctx, cache.save(ctx, fs, path.Join(b.Name, dataUsageCacheName)))
+		}
+		// Merge, save and send update.
+		// We do it even if unchanged.
+		cl := cache.clone()
+		entry := cl.flatten(*cl.root())
+		totalCache.replace(cl.Info.Name, dataUsageRoot, entry)
+		if intDataUpdateTracker.debug {
+			logger.Info(color.Green("CrawlAndGetDataUsage:")+" Saving totals cache with %d entries", len(totalCache.Cache))
+		}
+		logger.LogIf(ctx, totalCache.save(ctx, fs, dataUsageCacheName))
+		cloned := totalCache.clone()
+		updates <- cloned.dui(dataUsageRoot, buckets)
+	}
+
+	return nil
+}
+
+// crawlBucket crawls a single bucket in FS mode.
+// The updated cache for the bucket is returned.
+// A partially updated bucket may be returned.
+func (fs *FSObjects) crawlBucket(ctx context.Context, bucket string, cache dataUsageCache) (dataUsageCache, error) {
+	// Get bucket policy
+	// Check if the current bucket has a configured lifecycle policy
+	lc, err := globalLifecycleSys.Get(bucket)
+	if err == nil && lc.HasActiveRules("", true) {
+		if intDataUpdateTracker.debug {
+			logger.Info(color.Green("crawlBucket:") + " lifecycle: Active rules found")
+		}
+		cache.Info.lifeCycle = lc
+	}
+
+	// Load bucket info.
+	cache, err = crawlDataFolder(ctx, fs.fsPath, cache, fs.waitForLowActiveIO, func(item crawlItem) (int64, error) {
+		bucket, object := item.bucket, item.objectPath()
 		fsMetaBytes, err := ioutil.ReadFile(pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile))
 		if err != nil && !os.IsNotExist(err) {
 			return 0, errSkipFile
 		}
-		// Get file size, symlinks which cannot be
-		// followed are automatically filtered by fastwalk.
-		fi, err := os.Stat(item.Path)
-		if err != nil {
+
+		fsMeta := newFSMetaV1()
+		metaOk := false
+		if len(fsMetaBytes) > 0 {
+			var json = jsoniter.ConfigCompatibleWithStandardLibrary
+			if err = json.Unmarshal(fsMetaBytes, &fsMeta); err == nil {
+				metaOk = true
+			}
+		}
+		if !metaOk {
+			fsMeta = fs.defaultFsJSON(object)
+		}
+
+		// Stat the file.
+		fi, fiErr := os.Stat(item.Path)
+		if fiErr != nil {
 			return 0, errSkipFile
 		}
-		if len(fsMetaBytes) > 0 {
-			fsMeta := newFSMetaV1()
-			var json = jsoniter.ConfigCompatibleWithStandardLibrary
-			if err = json.Unmarshal(fsMetaBytes, &fsMeta); err != nil {
-				return 0, errSkipFile
-			}
-			return fsMeta.ToObjectInfo(bucket, object, fi).GetActualSize()
+
+		oi := fsMeta.ToObjectInfo(bucket, object, fi)
+		sz := item.applyActions(ctx, fs, actionMeta{oi: oi, meta: fsMeta.Meta})
+		if sz >= 0 {
+			return sz, nil
 		}
+
 		return fi.Size(), nil
-
 	})
-	cache.Info.BloomFilter = nil
 
-	// Even if there was an error, the new cache may have better info.
-	if cache.Info.LastUpdate.After(oldCache.Info.LastUpdate) {
-		if intDataUpdateTracker.debug {
-			logger.Info(color.Green("CrawlAndGetDataUsage:")+" Saving cache with %d entries", len(cache.Cache))
-		}
-		logger.LogIf(ctx, cache.save(ctx, fs, dataUsageCacheName))
-		updates <- cache.dui(dataUsageRoot, buckets)
-	} else {
-		if intDataUpdateTracker.debug {
-			logger.Info(color.Green("CrawlAndGetDataUsage:")+" Cache not updated, %d entries", len(cache.Cache))
-		}
-	}
-
-	return err
+	return cache, err
 }
 
 /// Bucket operations
@@ -361,6 +425,45 @@ func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket, locatio
 	return nil
 }
 
+// GetBucketPolicy - only needed for FS in NAS mode
+func (fs *FSObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
+	meta, err := loadBucketMetadata(ctx, fs, bucket)
+	if err != nil {
+		return nil, BucketPolicyNotFound{Bucket: bucket}
+	}
+	if meta.policyConfig == nil {
+		return nil, BucketPolicyNotFound{Bucket: bucket}
+	}
+	return meta.policyConfig, nil
+}
+
+// SetBucketPolicy - only needed for FS in NAS mode
+func (fs *FSObjects) SetBucketPolicy(ctx context.Context, bucket string, p *policy.Policy) error {
+	meta, err := loadBucketMetadata(ctx, fs, bucket)
+	if err != nil {
+		return err
+	}
+
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	configData, err := json.Marshal(p)
+	if err != nil {
+		return err
+	}
+	meta.PolicyConfigJSON = configData
+
+	return meta.Save(ctx, fs)
+}
+
+// DeleteBucketPolicy - only needed for FS in NAS mode
+func (fs *FSObjects) DeleteBucketPolicy(ctx context.Context, bucket string) error {
+	meta, err := loadBucketMetadata(ctx, fs, bucket)
+	if err != nil {
+		return err
+	}
+	meta.PolicyConfigJSON = nil
+	return meta.Save(ctx, fs)
+}
+
 // GetBucketInfo - fetch bucket metadata info.
 func (fs *FSObjects) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
 	atomic.AddInt64(&fs.activeIOCount, 1)
@@ -574,13 +677,11 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		switch lockType {
 		case writeLock:
 			if err = lock.GetLock(globalObjectTimeout); err != nil {
-				logger.LogIf(ctx, err)
 				return nil, err
 			}
 			nsUnlocker = lock.Unlock
 		case readLock:
 			if err = lock.GetRLock(globalObjectTimeout); err != nil {
-				logger.LogIf(ctx, err)
 				return nil, err
 			}
 			nsUnlocker = lock.RUnlock
@@ -1385,9 +1486,5 @@ func (fs *FSObjects) IsReady(_ context.Context) bool {
 		return false
 	}
 
-	globalObjLayerMutex.RLock()
-	res := globalObjectAPI != nil && !globalSafeMode
-	globalObjLayerMutex.RUnlock()
-
-	return res
+	return newObjectLayerFn() != nil
 }

cmd/gateway-main.go

@@ -45,6 +45,61 @@ var (
 	}
 )
 
+// GatewayLocker implements custom NewNSLock implementation
+type GatewayLocker struct {
+	ObjectLayer
+	nsMutex *nsLockMap
+}
+
+// NewNSLock - implements gateway level locker
+func (l *GatewayLocker) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
+	return l.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
+}
+
+// Walk - implements common gateway level Walker, to walk on all objects recursively at a prefix
+func (l *GatewayLocker) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo) error {
+	walk := func(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo) error {
+		var marker string
+
+		// Make sure the results channel is ready to be read when we're done.
+		defer close(results)
+
+		for {
+			// set maxKeys to '0' to list maximum possible objects in single call.
+			loi, err := l.ObjectLayer.ListObjects(ctx, bucket, prefix, marker, "", 0)
+			if err != nil {
+				return err
+			}
+			marker = loi.NextMarker
+			for _, obj := range loi.Objects {
+				select {
+				case results <- obj:
+				case <-ctx.Done():
+					return nil
+				}
+			}
+			if !loi.IsTruncated {
+				break
+			}
+		}
+		return nil
+	}
+
+	if err := l.ObjectLayer.Walk(ctx, bucket, prefix, results); err != nil {
+		if _, ok := err.(NotImplemented); ok {
+			return walk(ctx, bucket, prefix, results)
+		}
+		return err
+	}
+
+	return nil
+}
+
+// NewGatewayLayerWithLocker - initialize gateway with locker.
+func NewGatewayLayerWithLocker(gwLayer ObjectLayer) ObjectLayer {
+	return &GatewayLocker{ObjectLayer: gwLayer, nsMutex: newNSLock(false)}
+}
+
 // RegisterGatewayCommand registers a new command for gateway.
 func RegisterGatewayCommand(cmd cli.Command) error {
 	cmd.Flags = append(append(cmd.Flags, ServerFlags...), GlobalFlags...)
@@ -147,7 +202,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	gatewayHandleEnvVars()
 
 	// Set system resources to maximum.
-	logger.LogIf(GlobalContext, setMaxResources())
+	setMaxResources()
 
 	// Set when gateway is enabled
 	globalIsGateway = true
@@ -203,13 +258,16 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Add API router.
 	registerAPIRouter(router, encryptionEnabled, allowSSEKMS)
 
+	// Use all the middlewares
+	router.Use(registerMiddlewares)
+
 	var getCert certs.GetCertificateFunc
 	if globalTLSCerts != nil {
 		getCert = globalTLSCerts.GetCertificate
 	}
 
 	httpServer := xhttp.NewServer([]string{globalCLIContext.Addr},
-		criticalErrorHandler{registerHandlers(router, globalHandlers...)}, getCert)
+		criticalErrorHandler{router}, getCert)
 	httpServer.BaseContext = func(listener net.Listener) context.Context {
 		return GlobalContext
 	}
@@ -274,7 +332,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	if enableIAMOps {
 		// Initialize IAM sys.
-		logger.FatalIf(globalIAMSys.Init(GlobalContext, newObject), "Unable to initialize IAM system")
+		startBackgroundIAMLoad(GlobalContext)
 	}
 
 	if globalCacheConfig.Enabled {

cmd/gateway-unsupported.go

@@ -25,28 +25,11 @@ import (
 	"github.com/minio/minio-go/v6/pkg/tags"
 	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
-	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
 
 	"github.com/minio/minio/pkg/madmin"
 )
 
-// GatewayLocker implements custom NeNSLock implementation
-type GatewayLocker struct {
-	ObjectLayer
-	nsMutex *nsLockMap
-}
-
-// NewNSLock - implements gateway level locker
-func (l *GatewayLocker) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
-	return l.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
-}
-
-// NewGatewayLayerWithLocker - initialize gateway with locker.
-func NewGatewayLayerWithLocker(gwLayer ObjectLayer) ObjectLayer {
-	return &GatewayLocker{ObjectLayer: gwLayer, nsMutex: newNSLock(false)}
-}
-
 // GatewayUnsupported list of unsupported call stubs for gateway.
 type GatewayUnsupported struct{}
 
@@ -205,35 +188,6 @@ func (a GatewayUnsupported) GetMetrics(ctx context.Context) (*Metrics, error) {
 	return &Metrics{}, NotImplemented{}
 }
 
-// SetBucketTagging - not implemented
-func (a GatewayUnsupported) SetBucketTagging(ctx context.Context, bucket string, t *tags.Tags) error {
-	logger.LogIf(ctx, NotImplemented{})
-	return NotImplemented{}
-}
-
-// GetBucketObjectLockConfig - not implemented
-func (a GatewayUnsupported) GetBucketObjectLockConfig(ctx context.Context, bucket string) (*objectlock.Config, error) {
-	logger.LogIf(ctx, NotImplemented{})
-	return nil, NotImplemented{}
-}
-
-// SetBucketObjectLockConfig - not implemented
-func (a GatewayUnsupported) SetBucketObjectLockConfig(ctx context.Context, bucket string, _ *objectlock.Config) error {
-	logger.LogIf(ctx, NotImplemented{})
-	return NotImplemented{}
-}
-
-// GetBucketTagging - not implemented
-func (a GatewayUnsupported) GetBucketTagging(ctx context.Context, bucket string) (*tags.Tags, error) {
-	return nil, NotImplemented{}
-}
-
-// DeleteBucketTagging - not implemented.
-func (a GatewayUnsupported) DeleteBucketTagging(ctx context.Context, bucket string) error {
-	logger.LogIf(ctx, NotImplemented{})
-	return NotImplemented{}
-}
-
 // PutObjectTags - not implemented.
 func (a GatewayUnsupported) PutObjectTags(ctx context.Context, bucket, object string, tags string) error {
 	logger.LogIf(ctx, NotImplemented{})

cmd/gateway/azure/gateway-azure.go

@@ -23,6 +23,7 @@ import (
 	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -181,6 +182,9 @@ func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro
 
 	credential, err := azblob.NewSharedKeyCredential(creds.AccessKey, creds.SecretKey)
 	if err != nil {
+		if _, ok := err.(base64.CorruptInputError); ok {
+			return &azureObjects{}, errors.New("invalid Azure credentials")
+		}
 		return &azureObjects{}, err
 	}
 
@@ -624,6 +628,17 @@ func (a *azureObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketI
 
 // DeleteBucket - delete a container on azure, uses Azure equivalent `ContainerURL.Delete`.
 func (a *azureObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+	if !forceDelete {
+		// Check if the container is empty before deleting it.
+		result, err := a.ListObjects(ctx, bucket, "", "", "", 1)
+		if err != nil {
+			return azureToObjectError(err, bucket)
+		}
+		if len(result.Objects) > 0 {
+			return minio.BucketNotEmpty{Bucket: bucket}
+		}
+	}
+
 	containerURL := a.client.NewContainerURL(bucket)
 	_, err := containerURL.Delete(ctx, azblob.ContainerAccessConditions{})
 	return azureToObjectError(err, bucket)
@@ -1035,6 +1050,11 @@ func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object st
 	return uploadID, nil
 }
 
+func (a *azureObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, uploadID string, partID int,
+	startOffset int64, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (info minio.PartInfo, err error) {
+	return a.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.PutObjReader, dstOpts)
+}
+
 // PutObjectPart - Use Azure equivalent `BlobURL.StageBlock`.
 func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *minio.PutObjReader, opts minio.ObjectOptions) (info minio.PartInfo, err error) {
 	data := r.Reader

cmd/gateway/hdfs/gateway-hdfs.go

@@ -31,18 +31,18 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/colinmarc/hdfs/v2"
+	"github.com/colinmarc/hdfs/v2/hadoopconf"
 	"github.com/minio/cli"
-	krb "github.com/minio/gokrb5/v7/client"
-	"github.com/minio/gokrb5/v7/config"
-	"github.com/minio/gokrb5/v7/credentials"
-	"github.com/minio/hdfs/v3"
-	"github.com/minio/hdfs/v3/hadoopconf"
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/env"
 	xnet "github.com/minio/minio/pkg/net"
+	krb "gopkg.in/jcmturner/gokrb5.v7/client"
+	"gopkg.in/jcmturner/gokrb5.v7/config"
+	"gopkg.in/jcmturner/gokrb5.v7/credentials"
 )
 
 const (
@@ -277,6 +277,9 @@ func (n *hdfsObjects) DeleteBucket(ctx context.Context, bucket string, forceDele
 	if !hdfsIsValidBucketName(bucket) {
 		return minio.BucketNameInvalid{Bucket: bucket}
 	}
+	if forceDelete {
+		return hdfsToObjectErr(ctx, n.clnt.RemoveAll(minio.PathJoin(hdfsSeparator, bucket)), bucket)
+	}
 	return hdfsToObjectErr(ctx, n.clnt.Remove(minio.PathJoin(hdfsSeparator, bucket)), bucket)
 }
 

cmd/gateway/nas/gateway-nas.go

@@ -22,7 +22,6 @@ import (
 	"github.com/minio/cli"
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/pkg/auth"
-	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 )
 
 const (
@@ -122,16 +121,6 @@ type nasObjects struct {
 	minio.ObjectLayer
 }
 
-// GetBucketObjectLockConfig - not implemented
-func (n *nasObjects) GetBucketObjectLockConfig(ctx context.Context, bucket string) (*objectlock.Config, error) {
-	return nil, minio.NotImplemented{}
-}
-
-// SetBucketObjectLockConfig - not implemented
-func (n *nasObjects) SetBucketObjectLockConfig(ctx context.Context, bucket string, _ *objectlock.Config) error {
-	return minio.NotImplemented{}
-}
-
 // IsReady returns whether the layer is ready to take requests.
 func (n *nasObjects) IsReady(ctx context.Context) bool {
 	si, _ := n.StorageInfo(ctx, false)

cmd/gateway/s3/gateway-s3.go

@@ -622,7 +622,7 @@ func (l *s3Objects) GetMultipartInfo(ctx context.Context, bucket, object, upload
 
 // ListObjectParts returns all object parts for specified object in specified bucket
 func (l *s3Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int, opts minio.ObjectOptions) (lpi minio.ListPartsInfo, e error) {
-	result, err := l.Client.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
+	result, err := l.Client.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts)
 	if err != nil {
 		return lpi, err
 	}
@@ -630,7 +630,7 @@ func (l *s3Objects) ListObjectParts(ctx context.Context, bucket string, object s
 	if lpi.IsTruncated && maxParts > len(lpi.Parts) {
 		partNumberMarker = lpi.NextPartNumberMarker
 		for {
-			result, err = l.Client.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
+			result, err = l.Client.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts)
 			if err != nil {
 				return lpi, err
 			}

cmd/generic-handlers.go

@@ -30,17 +30,18 @@ import (
 	"github.com/minio/minio/cmd/http/stats"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/handlers"
+	"github.com/minio/minio/pkg/wildcard"
 	"github.com/rs/cors"
 )
 
-// HandlerFunc - useful to chain different middleware http.Handler
-type HandlerFunc func(http.Handler) http.Handler
+// MiddlewareFunc - useful to chain different http.Handler middlewares
+type MiddlewareFunc func(http.Handler) http.Handler
 
-func registerHandlers(h http.Handler, handlerFns ...HandlerFunc) http.Handler {
-	for _, hFn := range handlerFns {
-		h = hFn(h)
+func registerMiddlewares(next http.Handler) http.Handler {
+	for _, handlerFn := range globalHandlers {
+		next = handlerFn(next)
 	}
-	return h
+	return next
 }
 
 // Adds limiting body size middleware
@@ -410,7 +411,14 @@ func setCorsHandler(h http.Handler) http.Handler {
 	}
 
 	c := cors.New(cors.Options{
-		AllowedOrigins: []string{"*"},
+		AllowOriginFunc: func(origin string) bool {
+			for _, allowedOrigin := range globalAPIConfig.getCorsAllowOrigins() {
+				if wildcard.MatchSimple(allowedOrigin, origin) {
+					return true
+				}
+			}
+			return false
+		},
 		AllowedMethods: []string{
 			http.MethodGet,
 			http.MethodPut,
@@ -787,6 +795,7 @@ func (h criticalErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
 	defer func() {
 		if err := recover(); err == logger.ErrCritical { // handle
 			writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInternalError), r.URL, guessIsBrowserReq(r))
+			return
 		} else if err != nil {
 			panic(err) // forward other panic calls
 		}

cmd/global-heal.go

@@ -34,7 +34,7 @@ var leaderLockTimeout = newDynamicTimeout(time.Minute, time.Minute)
 
 // NewBgHealSequence creates a background healing sequence
 // operation which crawls all objects and heal them.
-func newBgHealSequence(numDisks int) *healSequence {
+func newBgHealSequence() *healSequence {
 
 	reqInfo := &logger.ReqInfo{API: "BackgroundHeal"}
 	ctx, cancelCtx := context.WithCancel(logger.SetReqInfo(GlobalContext, reqInfo))
@@ -54,7 +54,6 @@ func newBgHealSequence(numDisks int) *healSequence {
 		currentStatus: healSequenceStatus{
 			Summary:      healNotStartedStatus,
 			HealSettings: hs,
-			NumDisks:     numDisks,
 		},
 		cancelCtx:          cancelCtx,
 		ctx:                ctx,
@@ -79,7 +78,7 @@ func getLocalBackgroundHealStatus() madmin.BgHealState {
 }
 
 // healErasureSet lists and heals all objects in a specific erasure set
-func healErasureSet(ctx context.Context, setIndex int, xlObj *xlObjects) error {
+func healErasureSet(ctx context.Context, setIndex int, xlObj *xlObjects, drivesPerSet int) error {
 	buckets, err := xlObj.ListBuckets(ctx)
 	if err != nil {
 		return err
@@ -108,12 +107,38 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *xlObjects) error {
 			path: bucket.Name,
 		}
 
-		// List all objects in the current bucket and heal them
-		listDir := listDirFactory(ctx, xlObj.getLoadBalancedDisks()...)
-		walkResultCh := startTreeWalk(ctx, bucket.Name, "", "", true, listDir, nil)
-		for walkEntry := range walkResultCh {
+		var entryChs []FileInfoCh
+		for _, disk := range xlObj.getLoadBalancedDisks() {
+			if disk == nil {
+				// Disk can be offline
+				continue
+			}
+			entryCh, err := disk.Walk(bucket.Name, "", "", true, xlMetaJSONFile, readMetadata, ctx.Done())
+			if err != nil {
+				// Disk walk returned error, ignore it.
+				continue
+			}
+			entryChs = append(entryChs, FileInfoCh{
+				Ch: entryCh,
+			})
+		}
+
+		entriesValid := make([]bool, len(entryChs))
+		entries := make([]FileInfo, len(entryChs))
+
+		for {
+			entry, quorumCount, ok := lexicallySortedEntry(entryChs, entries, entriesValid)
+			if !ok {
+				return nil
+			}
+
+			if quorumCount == drivesPerSet {
+				// Skip good entries.
+				continue
+			}
+
 			bgSeq.sourceCh <- healSource{
-				path: pathJoin(bucket.Name, walkEntry.entry),
+				path: pathJoin(bucket.Name, entry.Name),
 			}
 		}
 	}
@@ -128,7 +153,7 @@ func deepHealObject(objectPath string) {
 
 	bgSeq.sourceCh <- healSource{
 		path: objectPath,
-		opts: madmin.HealOpts{ScanMode: madmin.HealDeepScan},
+		opts: &madmin.HealOpts{ScanMode: madmin.HealDeepScan},
 	}
 }
 
@@ -173,7 +198,7 @@ func execLeaderTasks(ctx context.Context, z *xlZones) {
 			for _, zone := range z.zones {
 				// Heal set by set
 				for i, set := range zone.sets {
-					if err := healErasureSet(ctx, i, set); err != nil {
+					if err := healErasureSet(ctx, i, set, zone.drivesPerSet); err != nil {
 						logger.LogIf(ctx, err)
 						continue
 					}

cmd/globals.go

@@ -154,9 +154,9 @@ var (
 	globalLifecycleSys       *LifecycleSys
 	globalBucketSSEConfigSys *BucketSSEConfigSys
 
-	// globalAPIThrottling controls S3 requests throttling when
-	// enabled in the config or in the shell environment.
-	globalAPIThrottling apiThrottling
+	// globalAPIConfig controls S3 API requests throttling,
+	// healthcheck readiness deadlines and cors settings.
+	globalAPIConfig apiConfig
 
 	globalStorageClass storageclass.Config
 	globalLDAPConfig   xldap.Config
@@ -275,8 +275,6 @@ var (
 	// If writes to FS backend should be O_SYNC.
 	globalFSOSync bool
 
-	// Deadline by which /minio/health/ready should respond.
-	globalReadyDeadline time.Duration
 	// Add new variable global values here.
 )
 

cmd/handler-api.go

@@ -20,34 +20,61 @@ import (
 	"net/http"
 	"sync"
 	"time"
+
+	"github.com/minio/minio/cmd/config/api"
 )
 
-type apiThrottling struct {
-	mu      sync.RWMutex
-	enabled bool
+type apiConfig struct {
+	mu sync.RWMutex
 
 	requestsDeadline time.Duration
 	requestsPool     chan struct{}
+	readyDeadline    time.Duration
+	corsAllowOrigins []string
 }
 
-func (t *apiThrottling) init(max int, deadline time.Duration) {
-	if max <= 0 {
-		return
-	}
-
+func (t *apiConfig) init(cfg api.Config) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
-	t.requestsPool = make(chan struct{}, max)
-	t.requestsDeadline = deadline
-	t.enabled = true
+	t.readyDeadline = cfg.APIReadyDeadline
+	t.corsAllowOrigins = cfg.APICorsAllowOrigin
+	if cfg.APIRequestsMax <= 0 {
+		return
+	}
+
+	apiRequestsMax := cfg.APIRequestsMax
+	if len(globalEndpoints.Hosts()) > 0 {
+		apiRequestsMax /= len(globalEndpoints.Hosts())
+	}
+
+	t.requestsPool = make(chan struct{}, apiRequestsMax)
+	t.requestsDeadline = cfg.APIRequestsDeadline
 }
 
-func (t *apiThrottling) get() (chan struct{}, <-chan time.Time) {
+func (t *apiConfig) getCorsAllowOrigins() []string {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 
-	if !t.enabled {
+	return t.corsAllowOrigins
+}
+
+func (t *apiConfig) getReadyDeadline() time.Duration {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	if t.readyDeadline == 0 {
+		return 10 * time.Second
+	}
+
+	return t.readyDeadline
+}
+
+func (t *apiConfig) getRequestsPool() (chan struct{}, <-chan time.Time) {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	if t.requestsPool == nil {
 		return nil, nil
 	}
 
@@ -57,7 +84,7 @@ func (t *apiThrottling) get() (chan struct{}, <-chan time.Time) {
 // maxClients throttles the S3 API calls
 func maxClients(f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		pool, deadlineTimer := globalAPIThrottling.get()
+		pool, deadlineTimer := globalAPIConfig.getRequestsPool()
 		if pool == nil {
 			f.ServeHTTP(w, r)
 			return

cmd/handler-utils.go

@@ -401,7 +401,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioPeerVersionMismatch",
 			Description:    desc,
-			HTTPStatusCode: http.StatusBadRequest,
+			HTTPStatusCode: http.StatusUpgradeRequired,
 		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, storageRESTPrefix):
 		desc := fmt.Sprintf("Expected 'storage' API version '%s', instead found '%s', please upgrade the servers",
@@ -409,7 +409,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioStorageVersionMismatch",
 			Description:    desc,
-			HTTPStatusCode: http.StatusBadRequest,
+			HTTPStatusCode: http.StatusUpgradeRequired,
 		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, lockRESTPrefix):
 		desc := fmt.Sprintf("Expected 'lock' API version '%s', instead found '%s', please upgrade the servers",
@@ -417,7 +417,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioLockVersionMismatch",
 			Description:    desc,
-			HTTPStatusCode: http.StatusBadRequest,
+			HTTPStatusCode: http.StatusUpgradeRequired,
 		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, adminPathPrefix):
 		var desc string
@@ -431,7 +431,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 		writeErrorResponseJSON(r.Context(), w, APIError{
 			Code:           "XMinioAdminVersionMismatch",
 			Description:    desc,
-			HTTPStatusCode: http.StatusBadRequest,
+			HTTPStatusCode: http.StatusUpgradeRequired,
 		}, r.URL)
 	default:
 		desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path)

cmd/healthcheck-handler.go

@@ -17,18 +17,25 @@
 package cmd
 
 import (
+	"context"
 	"net/http"
 )
 
-// ReadinessCheckHandler returns if the server is ready to receive requests.
-// For FS - Checks if the backend disk is available
-// For Erasure backend - Checks if all the erasure sets are writable
-func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ReadinessCheckHandler")
+// ClusterCheckHandler returns if the server is ready for requests.
+func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ClusterCheckCheckHandler")
 
-	objLayer := newObjectLayerWithoutSafeModeFn()
+	objLayer := newObjectLayerFn()
 	// Service not initialized yet
-	if objLayer == nil || !objLayer.IsReady(ctx) {
+	if objLayer == nil {
+		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getReadyDeadline())
+	defer cancel()
+
+	if !objLayer.IsReady(ctx) {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
@@ -36,6 +43,13 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
+// ReadinessCheckHandler Checks if the process is up. Always returns success.
+func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
+	// TODO: only implement this function to notify that this pod is
+	// busy, at a local scope in future, for now '200 OK'.
+	writeResponse(w, http.StatusOK, nil, mimeNone)
+}
+
 // LivenessCheckHandler - Checks if the process is up. Always returns success.
 func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	writeResponse(w, http.StatusOK, nil, mimeNone)

cmd/healthcheck-router.go

@@ -26,6 +26,7 @@ const (
 	healthCheckPath          = "/health"
 	healthCheckLivenessPath  = "/live"
 	healthCheckReadinessPath = "/ready"
+	healthCheckClusterPath   = "/cluster"
 	healthCheckPathPrefix    = minioReservedBucketPath + healthCheckPath
 )
 
@@ -35,6 +36,9 @@ func registerHealthCheckRouter(router *mux.Router) {
 	// Healthcheck router
 	healthRouter := router.PathPrefix(healthCheckPathPrefix).Subrouter()
 
+	// Cluster check handler to verify cluster is active
+	healthRouter.Methods(http.MethodGet).Path(healthCheckClusterPath).HandlerFunc(httpTraceAll(ClusterCheckHandler))
+
 	// Liveness handler
 	healthRouter.Methods(http.MethodGet).Path(healthCheckLivenessPath).HandlerFunc(httpTraceAll(LivenessCheckHandler))
 	healthRouter.Methods(http.MethodHead).Path(healthCheckLivenessPath).HandlerFunc(httpTraceAll(LivenessCheckHandler))

cmd/iam-etcd-store.go

@@ -449,18 +449,43 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
 func (ies *IAMEtcdStore) loadAll(ctx context.Context, sys *IAMSys) error {
 	iamUsersMap := make(map[string]auth.Credentials)
 	iamGroupsMap := make(map[string]GroupInfo)
-	iamPolicyDocsMap := make(map[string]iampolicy.Policy)
 	iamUserPolicyMap := make(map[string]MappedPolicy)
 	iamGroupPolicyMap := make(map[string]MappedPolicy)
 
-	isMinIOUsersSys := false
 	ies.rlock()
-	if sys.usersSysType == MinIOUsersSysType {
-		isMinIOUsersSys = true
-	}
+	isMinIOUsersSys := sys.usersSysType == MinIOUsersSysType
 	ies.runlock()
 
-	if err := ies.loadPolicyDocs(ctx, iamPolicyDocsMap); err != nil {
+	ies.lock()
+	if err := ies.loadPolicyDocs(ctx, sys.iamPolicyDocsMap); err != nil {
+		ies.unlock()
+		return err
+	}
+	// Sets default canned policies, if none are set.
+	setDefaultCannedPolicies(sys.iamPolicyDocsMap)
+
+	ies.unlock()
+
+	if isMinIOUsersSys {
+		if err := ies.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
+			return err
+		}
+		if err := ies.loadGroups(ctx, iamGroupsMap); err != nil {
+			return err
+		}
+	}
+
+	// load polices mapped to users
+	if err := ies.loadMappedPolicies(ctx, regularUser, false, iamUserPolicyMap); err != nil {
+		return err
+	}
+
+	// load policies mapped to groups
+	if err := ies.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
+		return err
+	}
+
+	if err := ies.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
 		return err
 	}
 
@@ -469,30 +494,10 @@ func (ies *IAMEtcdStore) loadAll(ctx context.Context, sys *IAMSys) error {
 		return err
 	}
 
-	if isMinIOUsersSys {
-		// load long term users
-		if err := ies.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
-			return err
-		}
-		if err := ies.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
-			return err
-		}
-		if err := ies.loadGroups(ctx, iamGroupsMap); err != nil {
-			return err
-		}
-		if err := ies.loadMappedPolicies(ctx, regularUser, false, iamUserPolicyMap); err != nil {
-			return err
-		}
-	}
-
-	// load STS policy mappings into the same map
+	// load STS policy mappings
 	if err := ies.loadMappedPolicies(ctx, stsUser, false, iamUserPolicyMap); err != nil {
 		return err
 	}
-	// load policies mapped to groups
-	if err := ies.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
-		return err
-	}
 
 	ies.lock()
 	defer ies.Unlock()
@@ -506,13 +511,6 @@ func (ies *IAMEtcdStore) loadAll(ctx context.Context, sys *IAMSys) error {
 		sys.iamUsersMap[k] = v
 	}
 
-	for k, v := range iamPolicyDocsMap {
-		sys.iamPolicyDocsMap[k] = v
-	}
-
-	// Sets default canned policies, if none are set.
-	setDefaultCannedPolicies(sys.iamPolicyDocsMap)
-
 	for k, v := range iamUserPolicyMap {
 		sys.iamUserPolicyMap[k] = v
 	}
@@ -535,6 +533,7 @@ func (ies *IAMEtcdStore) loadAll(ctx context.Context, sys *IAMSys) error {
 	}
 
 	sys.buildUserGroupMemberships()
+	sys.storeFallback = false
 
 	return nil
 }

cmd/iam-object-store.go

@@ -410,33 +410,27 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IA
 func (iamOS *IAMObjectStore) loadAll(ctx context.Context, sys *IAMSys) error {
 	iamUsersMap := make(map[string]auth.Credentials)
 	iamGroupsMap := make(map[string]GroupInfo)
-	iamPolicyDocsMap := make(map[string]iampolicy.Policy)
 	iamUserPolicyMap := make(map[string]MappedPolicy)
 	iamGroupPolicyMap := make(map[string]MappedPolicy)
 
-	isMinIOUsersSys := false
 	iamOS.rlock()
-	if sys.usersSysType == MinIOUsersSysType {
-		isMinIOUsersSys = true
-	}
+	isMinIOUsersSys := sys.usersSysType == MinIOUsersSysType
 	iamOS.runlock()
 
-	if err := iamOS.loadPolicyDocs(ctx, iamPolicyDocsMap); err != nil {
+	iamOS.lock()
+	if err := iamOS.loadPolicyDocs(ctx, sys.iamPolicyDocsMap); err != nil {
+		iamOS.unlock()
 		return err
 	}
+	// Sets default canned policies, if none are set.
+	setDefaultCannedPolicies(sys.iamPolicyDocsMap)
 
-	// load STS temp users
-	if err := iamOS.loadUsers(ctx, stsUser, iamUsersMap); err != nil {
-		return err
-	}
+	iamOS.unlock()
 
 	if isMinIOUsersSys {
 		if err := iamOS.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
 			return err
 		}
-		if err := iamOS.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
-			return err
-		}
 		if err := iamOS.loadGroups(ctx, iamGroupsMap); err != nil {
 			return err
 		}
@@ -447,13 +441,22 @@ func (iamOS *IAMObjectStore) loadAll(ctx context.Context, sys *IAMSys) error {
 		return err
 	}
 
-	// load STS policy mappings
-	if err := iamOS.loadMappedPolicies(ctx, stsUser, false, iamUserPolicyMap); err != nil {
+	// load policies mapped to groups
+	if err := iamOS.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
 		return err
 	}
 
-	// load policies mapped to groups
-	if err := iamOS.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
+	if err := iamOS.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
+		return err
+	}
+
+	// load STS temp users
+	if err := iamOS.loadUsers(ctx, stsUser, iamUsersMap); err != nil {
+		return err
+	}
+
+	// load STS policy mappings
+	if err := iamOS.loadMappedPolicies(ctx, stsUser, false, iamUserPolicyMap); err != nil {
 		return err
 	}
 
@@ -469,13 +472,6 @@ func (iamOS *IAMObjectStore) loadAll(ctx context.Context, sys *IAMSys) error {
 		sys.iamUsersMap[k] = v
 	}
 
-	for k, v := range iamPolicyDocsMap {
-		sys.iamPolicyDocsMap[k] = v
-	}
-
-	// Sets default canned policies, if none are set.
-	setDefaultCannedPolicies(sys.iamPolicyDocsMap)
-
 	for k, v := range iamUserPolicyMap {
 		sys.iamUserPolicyMap[k] = v
 	}
@@ -498,6 +494,7 @@ func (iamOS *IAMObjectStore) loadAll(ctx context.Context, sys *IAMSys) error {
 	}
 
 	sys.buildUserGroupMemberships()
+	sys.storeFallback = false
 
 	return nil
 }
@@ -588,14 +585,6 @@ func listIAMConfigItems(ctx context.Context, objAPI ObjectLayer, pathPrefix stri
 				return
 			}
 
-			// Attempt a slow down load only when server is
-			// active and initialized.
-			if !globalSafeMode {
-				// Slow down listing and loading for config items to
-				// reduce load on the server
-				waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()))
-			}
-
 			marker = lo.NextMarker
 			lister := dirList(lo)
 			if !dirs {

cmd/iam.go

@@ -21,8 +21,10 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/config"
@@ -30,6 +32,7 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/retry"
 )
 
 // UsersSysType - defines the type of users and groups system that is
@@ -213,7 +216,8 @@ type IAMSys struct {
 	iamGroupPolicyMap map[string]MappedPolicy
 
 	// Persistence layer for IAM subsystem
-	store IAMStorageAPI
+	store         IAMStorageAPI
+	storeFallback bool
 }
 
 // IAMUserType represents a user type inside MinIO server
@@ -403,10 +407,18 @@ func (sys *IAMSys) doIAMConfigMigration(ctx context.Context) error {
 	return sys.store.migrateBackendFormat(ctx)
 }
 
-// Init - initializes config system from iam.json
-func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) error {
+// Loads IAM users and policies in background, any un-handled
+// error means this code can potentially crash the server
+// in such a situation manual intervention is necessary.
+func startBackgroundIAMLoad(ctx context.Context) {
+	go globalIAMSys.Init(ctx, newObjectLayerWithoutSafeModeFn())
+}
+
+// Init - initializes config system by reading entries from config/iam
+func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 	if objAPI == nil {
-		return errServerNotInitialized
+		logger.LogIf(ctx, errServerNotInitialized)
+		return
 	}
 
 	if globalEtcdClient == nil {
@@ -419,18 +431,81 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) error {
 		sys.EnableLDAPSys()
 	}
 
-	// Migrate IAM configuration
-	if err := sys.doIAMConfigMigration(ctx); err != nil {
-		return err
+	retryCtx, cancel := context.WithCancel(ctx)
+
+	// Indicate to our routine to exit cleanly upon return.
+	defer cancel()
+
+	// Hold the lock for migration only.
+	txnLk := objAPI.NewNSLock(retryCtx, minioMetaBucket, minioConfigPrefix+"/iam.lock")
+
+	// Initializing IAM sub-system needs a retry mechanism for
+	// the following reasons:
+	//  - Read quorum is lost just after the initialization
+	//    of the object layer.
+	//  - Write quorum not met when upgrading configuration
+	//    version is needed, migration is needed etc.
+	rquorum := InsufficientReadQuorum{}
+	wquorum := InsufficientWriteQuorum{}
+
+	for range retry.NewTimerWithJitter(retryCtx, time.Second, 5*time.Second, retry.MaxJitter) {
+		// let one of the server acquire the lock, if not let them timeout.
+		// which shall be retried again by this loop.
+		if err := txnLk.GetLock(newDynamicTimeout(1*time.Second, 5*time.Second)); err != nil {
+			logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. trying to acquire lock")
+			continue
+		}
+
+		if globalEtcdClient != nil {
+			// ****  WARNING ****
+			// Migrating to encrypted backend on etcd should happen before initialization of
+			// IAM sub-system, make sure that we do not move the above codeblock elsewhere.
+			if err := migrateIAMConfigsEtcdToEncrypted(ctx, globalEtcdClient); err != nil {
+				txnLk.Unlock()
+				logger.LogIf(ctx, fmt.Errorf("Unable to decrypt an encrypted ETCD backend for IAM users and policies: %w", err))
+				logger.LogIf(ctx, errors.New("IAM sub-system is partially initialized, some users may not be available"))
+				return
+			}
+		}
+
+		// These messages only meant primarily for distributed setup, so only log during distributed setup.
+		if globalIsDistXL {
+			logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. lock acquired")
+		}
+
+		// Migrate IAM configuration, if necessary.
+		if err := sys.doIAMConfigMigration(ctx); err != nil {
+			txnLk.Unlock()
+			if errors.Is(err, errDiskNotFound) ||
+				errors.Is(err, errConfigNotFound) ||
+				errors.Is(err, context.Canceled) ||
+				errors.Is(err, context.DeadlineExceeded) ||
+				errors.As(err, &rquorum) ||
+				errors.As(err, &wquorum) ||
+				isErrBucketNotFound(err) {
+				logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v)", err)
+				continue
+			}
+			logger.LogIf(ctx, fmt.Errorf("Unable to migrate IAM users and policies to new format: %w", err))
+			logger.LogIf(ctx, errors.New("IAM sub-system is partially initialized, some users may not be available"))
+			return
+		}
+
+		// Successfully migrated, proceed to load the users.
+		txnLk.Unlock()
+		break
 	}
 
 	err := sys.store.loadAll(ctx, sys)
 
-	// Invalidate the old cred after finishing IAM initialization
+	// Invalidate the old cred always, even upon error to avoid any leakage.
 	globalOldCred = auth.Credentials{}
 
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize IAM sub-system, some users may not be available %w", err))
+	}
+
 	go sys.store.watch(ctx, sys)
-	return err
 }
 
 // DeletePolicy - deletes a canned policy from backend or etcd.
@@ -514,6 +589,10 @@ func (sys *IAMSys) ListPolicies() (map[string]iampolicy.Policy, error) {
 	sys.store.rlock()
 	defer sys.store.runlock()
 
+	if sys.storeFallback {
+		return nil, errIAMNotInitialized
+	}
+
 	policyDocsMap := make(map[string]iampolicy.Policy, len(sys.iamPolicyDocsMap))
 	for k, v := range sys.iamPolicyDocsMap {
 		policyDocsMap[k] = v
@@ -663,6 +742,10 @@ func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
 		return nil, errIAMActionNotAllowed
 	}
 
+	if sys.storeFallback {
+		return nil, errIAMNotInitialized
+	}
+
 	for k, v := range sys.iamUsersMap {
 		if !v.IsTemp() && !v.IsServiceAccount() {
 			users[k] = madmin.UserInfo{
@@ -897,6 +980,10 @@ func (sys *IAMSys) ListServiceAccounts(ctx context.Context, accessKey string) ([
 	sys.store.rlock()
 	defer sys.store.runlock()
 
+	if sys.storeFallback {
+		return nil, errIAMNotInitialized
+	}
+
 	var serviceAccounts []string
 
 	for k, v := range sys.iamUsersMap {
@@ -1028,6 +1115,44 @@ func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 		return cred, false
 	}
 
+	sys.store.rlock()
+	fallback := sys.storeFallback
+	sys.store.runlock()
+	if fallback {
+		sys.store.lock()
+		// If user is already found proceed.
+		if _, found := sys.iamUsersMap[accessKey]; !found {
+			sys.store.loadUser(accessKey, regularUser, sys.iamUsersMap)
+			if _, found = sys.iamUsersMap[accessKey]; found {
+				// found user, load its mapped policies
+				sys.store.loadMappedPolicy(accessKey, regularUser, false, sys.iamUserPolicyMap)
+			} else {
+				sys.store.loadUser(accessKey, srvAccUser, sys.iamUsersMap)
+				if svc, found := sys.iamUsersMap[accessKey]; found {
+					// Found service account, load its parent user and its mapped policies.
+					if sys.usersSysType == MinIOUsersSysType {
+						sys.store.loadUser(svc.ParentUser, regularUser, sys.iamUsersMap)
+					}
+					sys.store.loadMappedPolicy(svc.ParentUser, regularUser, false, sys.iamUserPolicyMap)
+				} else {
+					// None found fall back to STS users.
+					sys.store.loadUser(accessKey, stsUser, sys.iamUsersMap)
+					if _, found = sys.iamUsersMap[accessKey]; found {
+						// STS user found, load its mapped policy.
+						sys.store.loadMappedPolicy(accessKey, stsUser, false, sys.iamUserPolicyMap)
+					}
+				}
+			}
+		}
+		// Load associated policies if any.
+		for _, policy := range sys.iamUserPolicyMap[accessKey].toSlice() {
+			if _, found := sys.iamPolicyDocsMap[policy]; !found {
+				sys.store.loadPolicyDoc(policy, sys.iamPolicyDocsMap)
+			}
+		}
+		sys.store.unlock()
+	}
+
 	sys.store.rlock()
 	defer sys.store.runlock()
 
@@ -1276,6 +1401,10 @@ func (sys *IAMSys) ListGroups() (r []string, err error) {
 	sys.store.rlock()
 	defer sys.store.runlock()
 
+	if sys.storeFallback {
+		return nil, errIAMNotInitialized
+	}
+
 	if sys.usersSysType != MinIOUsersSysType {
 		return nil, errIAMActionNotAllowed
 	}
@@ -1798,7 +1927,9 @@ func NewIAMSys() *IAMSys {
 		iamUsersMap:             make(map[string]auth.Credentials),
 		iamPolicyDocsMap:        make(map[string]iampolicy.Policy),
 		iamUserPolicyMap:        make(map[string]MappedPolicy),
+		iamGroupPolicyMap:       make(map[string]MappedPolicy),
 		iamGroupsMap:            make(map[string]GroupInfo),
 		iamUserGroupMemberships: make(map[string]set.StringSet),
+		storeFallback:           true,
 	}
 }

cmd/lock-rest-client.go

@@ -18,15 +18,14 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"crypto/tls"
 	"errors"
 	"io"
-	"sync/atomic"
-	"time"
-
 	"net/url"
 
 	"github.com/minio/minio/cmd/http"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
 	"github.com/minio/minio/pkg/dsync"
@@ -36,7 +35,6 @@ import (
 type lockRESTClient struct {
 	restClient *rest.Client
 	endpoint   Endpoint
-	connected  int32
 }
 
 func toLockError(err error) error {
@@ -62,10 +60,6 @@ func (client *lockRESTClient) String() string {
 // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
 func (client *lockRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
-	if !client.IsOnline() {
-		return nil, errors.New("Lock rest server node is down")
-	}
-
 	if values == nil {
 		values = make(url.Values)
 	}
@@ -75,26 +69,16 @@ func (client *lockRESTClient) call(method string, values url.Values, body io.Rea
 		return respBody, nil
 	}
 
-	if isNetworkError(err) {
-		time.AfterFunc(defaultRetryUnit, func() {
-			// After 1 seconds, take this lock client online for a retry.
-			atomic.StoreInt32(&client.connected, 1)
-		})
-
-		atomic.StoreInt32(&client.connected, 0)
-	}
-
 	return nil, toLockError(err)
 }
 
 // IsOnline - returns whether REST client failed to connect or not.
 func (client *lockRESTClient) IsOnline() bool {
-	return atomic.LoadInt32(&client.connected) == 1
+	return client.restClient.IsOnline()
 }
 
 // Close - marks the client as closed.
 func (client *lockRESTClient) Close() error {
-	atomic.StoreInt32(&client.connected, 0)
 	client.restClient.Close()
 	return nil
 }
@@ -173,9 +157,16 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
 	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
 	restClient, err := rest.NewClient(serverURL, trFn, newAuthToken)
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
-		return &lockRESTClient{endpoint: endpoint, restClient: restClient, connected: 0}
+		logger.Fatal(err, "Unable to create lock rest client")
+	}
+	restClient.HealthCheckFn = func() bool {
+		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
+		respBody, err := restClient.CallWithContext(ctx, lockRESTMethodHealth, nil, nil, -1)
+		xhttp.DrainBody(respBody)
+		cancel()
+		var ne *rest.NetworkError
+		return !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &ne)
 	}
 
-	return &lockRESTClient{endpoint: endpoint, restClient: restClient, connected: 1}
+	return &lockRESTClient{endpoint: endpoint, restClient: restClient}
 }

cmd/lock-rest-client_test.go

@@ -29,7 +29,7 @@ func TestLockRESTlient(t *testing.T) {
 		t.Fatalf("unexpected error %v", err)
 	}
 	lkClient := newlockRESTClient(endpoint)
-	if lkClient.connected == 0 {
+	if !lkClient.IsOnline() {
 		t.Fatalf("unexpected error. connection failed")
 	}
 

cmd/lock-rest-server-common.go

@@ -27,6 +27,7 @@ const (
 )
 
 const (
+	lockRESTMethodHealth  = "/health"
 	lockRESTMethodLock    = "/lock"
 	lockRESTMethodRLock   = "/rlock"
 	lockRESTMethodUnlock  = "/unlock"

cmd/lock-rest-server.go

@@ -78,6 +78,11 @@ func getLockArgs(r *http.Request) (args dsync.LockArgs, err error) {
 	return args, nil
 }
 
+// HealthHandler returns success if request is authenticated.
+func (l *lockRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) {
+	l.IsValid(w, r)
+}
+
 // LockHandler - Acquires a lock.
 func (l *lockRESTServer) LockHandler(w http.ResponseWriter, r *http.Request) {
 	if !l.IsValid(w, r) {
@@ -345,6 +350,7 @@ func registerLockRESTHandlers(router *mux.Router, endpointZones EndpointZones) {
 			}
 
 			subrouter := router.PathPrefix(path.Join(lockRESTPrefix, endpoint.Path)).Subrouter()
+			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodHealth).HandlerFunc(httpTraceHdrs(lockServer.HealthHandler)).Queries(queries...)
 			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodLock).HandlerFunc(httpTraceHdrs(lockServer.LockHandler)).Queries(queries...)
 			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRLock).HandlerFunc(httpTraceHdrs(lockServer.RLockHandler)).Queries(queries...)
 			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodUnlock).HandlerFunc(httpTraceHdrs(lockServer.UnlockHandler)).Queries(queries...)

cmd/logger/audit.go

@@ -135,7 +135,7 @@ func AddAuditTarget(t Target) {
 }
 
 // AuditLog - logs audit logs to all audit targets.
-func AuditLog(w http.ResponseWriter, r *http.Request, api string, reqClaims map[string]interface{}) {
+func AuditLog(w http.ResponseWriter, r *http.Request, api string, reqClaims map[string]interface{}, filterKeys ...string) {
 	// Fast exit if there is not audit target configured
 	if len(AuditTargets) == 0 {
 		return
@@ -162,6 +162,12 @@ func AuditLog(w http.ResponseWriter, r *http.Request, api string, reqClaims map[
 	}
 
 	entry := audit.ToEntry(w, r, reqClaims, globalDeploymentID)
+	for _, filterKey := range filterKeys {
+		delete(entry.ReqClaims, filterKey)
+		delete(entry.ReqQuery, filterKey)
+		delete(entry.ReqHeader, filterKey)
+		delete(entry.RespHeader, filterKey)
+	}
 	entry.API.Name = api
 	entry.API.Bucket = bucket
 	entry.API.Object = object

cmd/merge-walk-pool.go

@@ -28,6 +28,7 @@ const (
 
 // mergeWalk - represents the go routine that does the merge walk.
 type mergeWalk struct {
+	added      time.Time
 	entryChs   []FileInfoCh
 	endWalkCh  chan struct{}   // To signal when mergeWalk go-routine should end.
 	endTimerCh chan<- struct{} // To signal when timer go-routine should end.
@@ -56,7 +57,7 @@ func NewMergeWalkPool(timeout time.Duration) *MergeWalkPool {
 // Release - selects a mergeWalk from the pool based on the input
 // listParams, removes it from the pool, and returns the MergeWalkResult
 // channel.
-// Returns nil if listParams does not have an asccociated mergeWalk.
+// Returns nil if listParams does not have an associated mergeWalk.
 func (t *MergeWalkPool) Release(params listParams) ([]FileInfoCh, chan struct{}) {
 	t.Lock()
 	defer t.Unlock()
@@ -65,6 +66,7 @@ func (t *MergeWalkPool) Release(params listParams) ([]FileInfoCh, chan struct{})
 		if len(walks) > 0 {
 			// Pop out the first valid walk entry.
 			walk := walks[0]
+			walks[0] = mergeWalk{} // clear references.
 			walks = walks[1:]
 			if len(walks) > 0 {
 				t.pool[params] = walks
@@ -91,17 +93,54 @@ func (t *MergeWalkPool) Set(params listParams, resultChs []FileInfoCh, endWalkCh
 	t.Lock()
 	defer t.Unlock()
 
+	// If we are above the limit delete at least one entry from the pool.
+	if len(t.pool) > treeWalkEntryLimit {
+		age := time.Now()
+		var oldest listParams
+		for k, v := range t.pool {
+			if len(v) == 0 {
+				delete(t.pool, k)
+				continue
+			}
+			// The first element is the oldest, so we only check that.
+			if v[0].added.Before(age) {
+				oldest = k
+			}
+		}
+		// Invalidate and delete oldest.
+		if walks, ok := t.pool[oldest]; ok {
+			walk := walks[0]
+			walks[0] = mergeWalk{} // clear references.
+			walks = walks[1:]
+			if len(walks) > 0 {
+				t.pool[params] = walks
+			} else {
+				delete(t.pool, params)
+			}
+			walk.endTimerCh <- struct{}{}
+		}
+	}
+
 	// Should be a buffered channel so that Release() never blocks.
 	endTimerCh := make(chan struct{}, 1)
 
 	walkInfo := mergeWalk{
+		added:      UTCNow(),
 		entryChs:   resultChs,
 		endWalkCh:  endWalkCh,
 		endTimerCh: endTimerCh,
 	}
 
 	// Append new walk info.
-	t.pool[params] = append(t.pool[params], walkInfo)
+	walks := t.pool[params]
+	if len(walks) < treeWalkSameEntryLimit {
+		t.pool[params] = append(walks, walkInfo)
+	} else {
+		// We are at limit, invalidate oldest, move list down and add new as last.
+		walks[0].endTimerCh <- struct{}{}
+		copy(walks, walks[1:])
+		walks[len(walks)-1] = walkInfo
+	}
 
 	// Timer go-routine which times out after t.timeOut seconds.
 	go func(endTimerCh <-chan struct{}, walkInfo mergeWalk) {

cmd/merge-walk-pool_test.go

@@ -74,7 +74,7 @@ func TestManyMergeWalksSameParam(t *testing.T) {
 		break
 	default:
 		// Create many treeWalk go-routines for the same params.
-		for i := 0; i < 10; i++ {
+		for i := 0; i < treeWalkSameEntryLimit; i++ {
 			endWalkCh := make(chan struct{})
 			walkChs := make([]FileInfoCh, 0)
 			tw.Set(params, walkChs, endWalkCh)
@@ -82,16 +82,62 @@ func TestManyMergeWalksSameParam(t *testing.T) {
 
 		tw.Lock()
 		if walks, ok := tw.pool[params]; ok {
-			if len(walks) != 10 {
+			if len(walks) != treeWalkSameEntryLimit {
 				t.Error("There aren't as many walks as were Set")
 			}
 		}
 		tw.Unlock()
-		for i := 0; i < 10; i++ {
+		for i := 0; i < treeWalkSameEntryLimit; i++ {
 			tw.Lock()
 			if walks, ok := tw.pool[params]; ok {
-				// Before ith Release we should have 10-i treeWalk go-routines.
-				if 10-i != len(walks) {
+				// Before ith Release we should have n-i treeWalk go-routines.
+				if treeWalkSameEntryLimit-i != len(walks) {
+					t.Error("There aren't as many walks as were Set")
+				}
+			}
+			tw.Unlock()
+			tw.Release(params)
+		}
+	}
+
+}
+
+// Test if multiple merge walkers for the same listParams are managed as expected by the pool
+// but that treeWalkSameEntryLimit is respected.
+func TestManyMergeWalksSameParamPrune(t *testing.T) {
+	// Create a treeWalkPool.
+	tw := NewMergeWalkPool(5 * time.Second)
+
+	// Create sample params.
+	params := listParams{
+		bucket: "test-bucket",
+	}
+
+	select {
+	// This timeout is an upper-bound. This is started
+	// before the first treeWalk go-routine's timeout period starts.
+	case <-time.After(5 * time.Second):
+		break
+	default:
+		// Create many treeWalk go-routines for the same params.
+		for i := 0; i < treeWalkSameEntryLimit*4; i++ {
+			endWalkCh := make(chan struct{})
+			walkChs := make([]FileInfoCh, 0)
+			tw.Set(params, walkChs, endWalkCh)
+		}
+
+		tw.Lock()
+		if walks, ok := tw.pool[params]; ok {
+			if len(walks) > treeWalkSameEntryLimit {
+				t.Error("There aren't as many walks as were Set")
+			}
+		}
+		tw.Unlock()
+		for i := 0; i < treeWalkSameEntryLimit; i++ {
+			tw.Lock()
+			if walks, ok := tw.pool[params]; ok {
+				// Before ith Release we should have n-i treeWalk go-routines.
+				if treeWalkSameEntryLimit-i != len(walks) {
 					t.Error("There aren't as many walks as were Set")
 				}
 			}

cmd/metrics.go

@@ -260,6 +260,27 @@ func cacheMetricsPrometheus(ch chan<- prometheus.Metric) {
 		prometheus.CounterValue,
 		float64(cacheObjLayer.CacheStats().getBytesServed()),
 	)
+	for _, cdStats := range cacheObjLayer.CacheStats().GetDiskStats() {
+		// Cache disk usage percentage
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				prometheus.BuildFQName("cache", "usage", "percent"),
+				"Total percentage cache usage",
+				[]string{"disk"}, nil),
+			prometheus.GaugeValue,
+			float64(cdStats.UsagePercent),
+			cdStats.Dir,
+		)
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				prometheus.BuildFQName("cache", "usage", "high"),
+				"Indicates cache usage is high or low, relative to current cache 'quota' settings",
+				[]string{"disk"}, nil),
+			prometheus.GaugeValue,
+			float64(cdStats.UsageState),
+			cdStats.Dir,
+		)
+	}
 }
 
 // collects http metrics for MinIO server in Prometheus specific format

cmd/namespace-lock.go

@@ -28,9 +28,9 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/minio/lsync"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dsync"
+	"github.com/minio/minio/pkg/lsync"
 )
 
 // local lock servers
@@ -79,7 +79,7 @@ func (n *nsLockMap) lock(ctx context.Context, volume string, path string, lockSo
 	nsLk, found := n.lockMap[resource]
 	if !found {
 		nsLk = &nsLock{
-			LRWMutex: lsync.NewLRWMutex(ctx),
+			LRWMutex: lsync.NewLRWMutex(),
 		}
 		// Add a count to indicate that a parallel unlock doesn't clear this entry.
 	}
@@ -89,9 +89,9 @@ func (n *nsLockMap) lock(ctx context.Context, volume string, path string, lockSo
 
 	// Locking here will block (until timeout).
 	if readLock {
-		locked = nsLk.GetRLock(opsID, lockSource, timeout)
+		locked = nsLk.GetRLock(ctx, opsID, lockSource, timeout)
 	} else {
-		locked = nsLk.GetLock(opsID, lockSource, timeout)
+		locked = nsLk.GetLock(ctx, opsID, lockSource, timeout)
 	}
 
 	if !locked { // We failed to get the lock
@@ -139,6 +139,7 @@ func (n *nsLockMap) unlock(volume string, path string, readLock bool) {
 type distLockInstance struct {
 	rwMutex *dsync.DRWMutex
 	opsID   string
+	ctx     context.Context
 }
 
 // Lock - block until write lock is taken or timeout has occurred.
@@ -146,7 +147,7 @@ func (di *distLockInstance) GetLock(timeout *dynamicTimeout) (timedOutErr error)
 	lockSource := getSource(2)
 	start := UTCNow()
 
-	if !di.rwMutex.GetLock(di.opsID, lockSource, timeout.Timeout()) {
+	if !di.rwMutex.GetLock(di.ctx, di.opsID, lockSource, timeout.Timeout()) {
 		timeout.LogFailure()
 		return OperationTimedOut{}
 	}
@@ -163,7 +164,7 @@ func (di *distLockInstance) Unlock() {
 func (di *distLockInstance) GetRLock(timeout *dynamicTimeout) (timedOutErr error) {
 	lockSource := getSource(2)
 	start := UTCNow()
-	if !di.rwMutex.GetRLock(di.opsID, lockSource, timeout.Timeout()) {
+	if !di.rwMutex.GetRLock(di.ctx, di.opsID, lockSource, timeout.Timeout()) {
 		timeout.LogFailure()
 		return OperationTimedOut{}
 	}
@@ -191,10 +192,10 @@ type localLockInstance struct {
 func (n *nsLockMap) NewNSLock(ctx context.Context, lockersFn func() []dsync.NetLocker, volume string, paths ...string) RWLocker {
 	opsID := mustGetUUID()
 	if n.isDistXL {
-		drwmutex := dsync.NewDRWMutex(ctx, &dsync.Dsync{
+		drwmutex := dsync.NewDRWMutex(&dsync.Dsync{
 			GetLockersFn: lockersFn,
 		}, pathsJoinPrefix(volume, paths...)...)
-		return &distLockInstance{drwmutex, opsID}
+		return &distLockInstance{drwmutex, opsID, ctx}
 	}
 	sort.Strings(paths)
 	return &localLockInstance{ctx, n, volume, paths, opsID}

cmd/notification.go

@@ -1164,7 +1164,7 @@ func (sys *NotificationSys) ServerInfo() []madmin.ServerProperties {
 }
 
 // GetLocalDiskIDs - return disk ids of the local disks of the peers.
-func (sys *NotificationSys) GetLocalDiskIDs() []string {
+func (sys *NotificationSys) GetLocalDiskIDs(ctx context.Context) []string {
 	var diskIDs []string
 	var mu sync.Mutex
 
@@ -1176,7 +1176,7 @@ func (sys *NotificationSys) GetLocalDiskIDs() []string {
 		wg.Add(1)
 		go func(client *peerRESTClient) {
 			defer wg.Done()
-			ids := client.GetLocalDiskIDs()
+			ids := client.GetLocalDiskIDs(ctx)
 			mu.Lock()
 			diskIDs = append(diskIDs, ids...)
 			mu.Unlock()

cmd/obdinfo.go

@@ -27,7 +27,6 @@ import (
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/madmin"
 	cpuhw "github.com/shirou/gopsutil/cpu"
-	"github.com/shirou/gopsutil/host"
 	memhw "github.com/shirou/gopsutil/mem"
 	"github.com/shirou/gopsutil/process"
 )
@@ -368,36 +367,3 @@ func getLocalProcOBD(ctx context.Context, r *http.Request) madmin.ServerProcOBDI
 		Processes: sysProcs,
 	}
 }
-
-func getLocalOsInfoOBD(ctx context.Context, r *http.Request) madmin.ServerOsOBDInfo {
-	addr := r.Host
-	if globalIsDistXL {
-		addr = GetLocalPeer(globalEndpoints)
-	}
-
-	info, err := host.InfoWithContext(ctx)
-	if err != nil {
-		return madmin.ServerOsOBDInfo{
-			Addr:  addr,
-			Error: err.Error(),
-		}
-	}
-
-	sensors, err := host.SensorsTemperaturesWithContext(ctx)
-	if err != nil {
-		return madmin.ServerOsOBDInfo{
-			Addr:  addr,
-			Error: err.Error(),
-		}
-	}
-
-	// ignore user err, as it cannot be obtained reliably inside containers
-	users, _ := host.UsersWithContext(ctx)
-
-	return madmin.ServerOsOBDInfo{
-		Addr:    addr,
-		Info:    info,
-		Sensors: sensors,
-		Users:   users,
-	}
-}

cmd/obdinfo_other.go

@@ -1,4 +1,4 @@
-// +build !freebsd
+// +build !freebsd,!netbsd,!openbsd,!solaris
 
 /*
  * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
@@ -26,8 +26,42 @@ import (
 
 	"github.com/minio/minio/pkg/madmin"
 	diskhw "github.com/shirou/gopsutil/disk"
+	"github.com/shirou/gopsutil/host"
 )
 
+func getLocalOsInfoOBD(ctx context.Context, r *http.Request) madmin.ServerOsOBDInfo {
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(globalEndpoints)
+	}
+
+	info, err := host.InfoWithContext(ctx)
+	if err != nil {
+		return madmin.ServerOsOBDInfo{
+			Addr:  addr,
+			Error: err.Error(),
+		}
+	}
+
+	sensors, err := host.SensorsTemperaturesWithContext(ctx)
+	if err != nil {
+		return madmin.ServerOsOBDInfo{
+			Addr:  addr,
+			Error: err.Error(),
+		}
+	}
+
+	// ignore user err, as it cannot be obtained reliably inside containers
+	users, _ := host.UsersWithContext(ctx)
+
+	return madmin.ServerOsOBDInfo{
+		Addr:    addr,
+		Info:    info,
+		Sensors: sensors,
+		Users:   users,
+	}
+}
+
 func getLocalDiskHwOBD(ctx context.Context, r *http.Request) madmin.ServerDiskHwOBDInfo {
 	addr := r.Host
 	if globalIsDistXL {

cmd/obdinfo_unix.go

@@ -1,3 +1,5 @@
+// +build freebsd netbsd openbsd solaris
+
 /*
  * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
  *
@@ -20,6 +22,7 @@ package cmd
 import (
 	"context"
 	"net/http"
+	"runtime"
 
 	"github.com/minio/minio/pkg/madmin"
 )
@@ -32,6 +35,18 @@ func getLocalDiskHwOBD(ctx context.Context, r *http.Request) madmin.ServerDiskHw
 
 	return madmin.ServerDiskHwOBDInfo{
 		Addr:  addr,
-		Error: "unsupported platform",
+		Error: "unsupported platform: " + runtime.GOOS,
+	}
+}
+
+func getLocalOsInfoOBD(ctx context.Context, r *http.Request) madmin.ServerOsOBDInfo {
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(globalEndpoints)
+	}
+
+	return madmin.ServerOsOBDInfo{
+		Addr:  addr,
+		Error: "unsupported platform: " + runtime.GOOS,
 	}
 }

cmd/object-api-interface.go

@@ -114,8 +114,7 @@ type ObjectLayer interface {
 	IsNotificationSupported() bool
 	IsListenBucketSupported() bool
 	IsEncryptionSupported() bool
-
-	// Compression support check.
+	IsTaggingSupported() bool
 	IsCompressionSupported() bool
 
 	// Backend related metrics
@@ -124,9 +123,6 @@ type ObjectLayer interface {
 	// Check Readiness
 	IsReady(ctx context.Context) bool
 
-	// Object Tagging Support check.
-	IsTaggingSupported() bool
-
 	// ObjectTagging operations
 	PutObjectTags(context.Context, string, string, string) error
 	GetObjectTags(context.Context, string, string) (*tags.Tags, error)

cmd/object-api-listobjects_test.go

@@ -573,7 +573,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 
 	for i, testCase := range testCases {
 		testCase := testCase
-		t.Run(fmt.Sprintf("Test%d-%s", i+1, instanceType), func(t *testing.T) {
+		t.Run(fmt.Sprintf("%s-Test%d", instanceType, i+1), func(t *testing.T) {
 			result, err := obj.ListObjects(context.Background(), testCase.bucketName,
 				testCase.prefix, testCase.marker, testCase.delimeter, int(testCase.maxKeys))
 			if err != nil && testCase.shouldPass {

cmd/object-api-utils.go

@@ -183,6 +183,16 @@ func checkObjectNameForLengthAndSlash(bucket, object string) error {
 			Object: object,
 		}
 	}
+	if runtime.GOOS == globalWindowsOSName {
+		// Explicitly disallowed characters on windows.
+		// Avoids most problematic names.
+		if strings.ContainsAny(object, `:*?"|<>`) {
+			return ObjectNameInvalid{
+				Bucket: bucket,
+				Object: object,
+			}
+		}
+	}
 	return nil
 }
 

cmd/object-handlers-common.go

@@ -173,9 +173,9 @@ func checkPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	}
 
 	// Check if the part number is correct.
-	if opts.PartNumber > 0 && len(objInfo.Parts) > 0 && opts.PartNumber != len(objInfo.Parts) {
+	if opts.PartNumber > 1 && opts.PartNumber > len(objInfo.Parts) {
 		writeHeaders()
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL, guessIsBrowserReq(r))
+		w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
 		return true
 	}
 
@@ -257,7 +257,7 @@ func isETagEqual(left, right string) bool {
 // after analyzing the current bucket lifecycle rules if any.
 func setAmzExpirationHeader(w http.ResponseWriter, bucket string, objInfo ObjectInfo) {
 	if lc, err := globalLifecycleSys.Get(bucket); err == nil {
-		ruleID, expiryTime := lc.PredictExpiryTime(objInfo.Name, objInfo.UserTags)
+		ruleID, expiryTime := lc.PredictExpiryTime(objInfo.Name, objInfo.ModTime, objInfo.UserTags)
 		if !expiryTime.IsZero() {
 			w.Header()[xhttp.AmzExpiration] = []string{
 				fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, expiryTime.Format(http.TimeFormat), ruleID),