fix: initialize config once per startup (#9851)

fixes #9857

.github/workflows/codeql.yml

@@ -0,0 +1,51 @@
+name: "Code scanning - action"
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '0 19 * * 0'
+
+jobs:
+  CodeQL-Build:
+
+    # CodeQL runs on ubuntu-latest and windows-latest
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+      
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+         languages: go, javascript
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/go.yml

@@ -15,24 +15,32 @@ jobs:
     strategy:
       matrix:
         go-version: [1.13.x]
-        os: [ubuntu-latest]
+        os: [ubuntu-latest, windows-latest]
     steps:
-      - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }}
-        uses: actions/setup-go@v1
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '12'
+      - uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go-version }}
-        id: go
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v1
-
       - name: Build on ${{ matrix.os }}
+        if: matrix.os == 'windows-latest'
         env:
           CGO_ENABLED: 0
           GO111MODULE: on
-          SIMPLE_CI: 1
+          MINIO_CI_CD: 1
         run: |
-          sudo apt-get install devscripts
+          go build --ldflags="-s -w" -o %GOPATH%\bin\minio.exe
+          go test -v --timeout 50m ./...
+      - name: Build on ${{ matrix.os }}
+        if: matrix.os == 'ubuntu-latest'
+        env:
+          CGO_ENABLED: 0
+          GO111MODULE: on
+          MINIO_CI_CD: 1
+        run: |
+          sudo apt-get install devscripts shellcheck
           make
           diff -au <(gofmt -s -d cmd) <(printf "")
           diff -au <(gofmt -s -d pkg) <(printf "")
@@ -40,3 +48,5 @@ jobs:
           make crosscompile
           make verify
           make verify-healing
+          cd browser && npm install && npm run test && cd ..
+          bash -c 'shopt -s globstar; shellcheck mint/**/*.sh'

.golangci.yml

@@ -0,0 +1,27 @@
+linters-settings:
+  golint:
+    min-confidence: 0
+
+  misspell:
+    locale: US
+
+linters:
+  disable-all: true
+  enable:
+    - typecheck
+    - goimports
+    - misspell
+    - govet
+    - golint
+    - ineffassign
+    - gosimple
+    - deadcode
+    - structcheck
+
+issues:
+  exclude-use-default: false
+  exclude:
+      - should have a package comment
+      - error strings should not be capitalized or end with punctuation or a newline
+service:
+  golangci-lint-version: 1.20.0 # use the fixed version to not introduce new linters unexpectedly

.travis.yml

@@ -1,52 +0,0 @@
-go_import_path: github.com/minio/minio
-
-language: go
-
-addons:
-  apt:
-    packages:
-    - shellcheck
-
-services:
-  - docker
-
-# this ensures PRs based on a local branch are not built twice
-# the downside is that a PR targeting a different branch is not built
-# but as a workaround you can add the branch to this list
-branches:
-  only:
-    - master
-
-matrix:
-  include:
-    - os: linux
-      dist: trusty
-      sudo: required
-      env:
-        - ARCH=x86_64
-        - CGO_ENABLED=0
-        - GO111MODULE=on
-        - SIMPLE_CI=1
-      go: 1.13.x
-      script:
-        - cd browser && npm install && npm run test && cd ..
-        - bash -c 'shopt -s globstar; shellcheck mint/**/*.sh'
-
-    - os: windows
-      env:
-        - ARCH=x86_64
-        - CGO_ENABLED=0
-        - GO111MODULE=on
-        - SIMPLE_CI=1
-      go: 1.13.x
-      script:
-        - go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
-        - for d in $(go list ./... | grep -v browser); do go test -v --timeout 20m "$d"; done
-
-before_script:
-  # Add an IPv6 config - see the corresponding Travis issue
-  # https://github.com/travis-ci/travis-ci/issues/8361
-  - if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'; fi
-
-before_install:
- - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi

CREDITS

@@ -4487,214 +4487,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ================================================================
 
-github.com/google/gofuzz
-https://github.com/google/gofuzz
-----------------------------------------------------------------
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-================================================================
-
 github.com/google/martian
 https://github.com/google/martian
 ----------------------------------------------------------------
@@ -10936,25 +10728,6 @@ THE SOFTWARE.
 
 ================================================================
 
-github.com/kurin/blazer
-https://github.com/kurin/blazer
-----------------------------------------------------------------
-Copyright 2016, the Blazer authors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-================================================================
-
 github.com/lib/pq
 https://github.com/lib/pq
 ----------------------------------------------------------------
@@ -17942,6 +17715,31 @@ https://go.opencensus.io
    limitations under the License.
 ================================================================
 
+go.uber.org/atomic
+https://go.uber.org/atomic
+----------------------------------------------------------------
+Copyright (c) 2016 Uber Technologies, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+================================================================
+
 go.uber.org/multierr
 https://go.uber.org/multierr
 ----------------------------------------------------------------

Dockerfile.dev.browser

@@ -0,0 +1,13 @@
+FROM ubuntu
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends --no-install-suggests \
+      git golang make npm && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+ENV PATH=$PATH:/root/go/bin
+
+RUN go get github.com/go-bindata/go-bindata/go-bindata && \
+    go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs
+

Dockerfile.simpleci

@@ -1,80 +0,0 @@
-#-------------------------------------------------------------
-# Stage 1: Build and Unit tests
-#-------------------------------------------------------------
-FROM golang:1.13
-
-COPY . /go/src/github.com/minio/minio
-WORKDIR /go/src/github.com/minio/minio
-
-RUN apt-get update && apt-get install -y jq
-ENV GO111MODULE=on
-ENV SIMPLE_CI 1
-
-RUN git config --global http.cookiefile /gitcookie/.gitcookie
-
-RUN apt-get update && \
-      apt-get -y install sudo
-RUN touch /etc/sudoers
-
-RUN echo "root ALL=(ALL) ALL" >> /etc/sudoers
-RUN echo "ci ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
-RUN echo "Defaults    env_reset" >> /etc/sudoers
-RUN echo 'Defaults    secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go:/usr/local/go/bin"' >> /etc/sudoers
-
-RUN mkdir -p /home/ci/.cache
-
-RUN groupadd -g 999 ci && \
-    useradd -r -u 999 -g ci ci && \
-    chown -R ci:ci /go /home/ci && \
-    chmod -R a+rw /go
-
-USER ci
-
-# -- tests --
-RUN make
-RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")'
-RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")'
-RUN make test-race
-RUN make crosscompile
-RUN make verify
-
-## -- add healing tests
-RUN make verify-healing
-
-#-------------------------------------------------------------
-# Stage 2: Test Frontend
-#-------------------------------------------------------------
-FROM node:10.15-stretch-slim
-
-ENV SIMPLE_CI 1
-
-COPY browser /minio/browser
-WORKDIR /minio/browser
-
-RUN yarn
-RUN yarn test
-
-#-------------------------------------------------------------
-# Stage 3: Run Gateway Tests
-#-------------------------------------------------------------
-FROM ubuntu:18.04
-
-COPY --from=0 /go/src/github.com/minio/minio/minio /usr/bin/minio
-COPY buildscripts/gateway-tests.sh /usr/bin/gateway-tests.sh
-COPY mint /mint
-
-ENV DEBIAN_FRONTEND noninteractive
-ENV LANG C.UTF-8
-ENV GOROOT /usr/local/go
-ENV GOPATH /usr/local/gopath
-ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
-ENV SIMPLE_CI 1
-ENV MINT_ROOT_DIR /mint
-
-RUN apt-get --yes update && apt-get --yes upgrade && \
-    apt-get --yes --quiet install wget jq curl git dnsmasq && \
-    cd /mint && /mint/release.sh
-
-WORKDIR /mint
-
-RUN /usr/bin/gateway-tests.sh

Makefile

@@ -8,8 +8,6 @@ GOOS := $(shell go env GOOS)
 VERSION ?= $(shell git describe --tags)
 TAG ?= "minio/minio:$(VERSION)"
 
-BUILD_LDFLAGS := '$(LDFLAGS)'
-
 all: build
 
 checks:
@@ -18,22 +16,12 @@ checks:
 
 getdeps:
 	@mkdir -p ${GOPATH}/bin
-	@which golint 1>/dev/null || (echo "Installing golint" && GO111MODULE=off go get -u golang.org/x/lint/golint)
-ifeq ($(GOARCH),s390x)
-	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && GO111MODULE=off go get honnef.co/go/tools/cmd/staticcheck)
-else
-	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet https://github.com/dominikh/go-tools/releases/download/2020.1.3/staticcheck_${GOOS}_${GOARCH}.tar.gz && tar xf staticcheck_${GOOS}_${GOARCH}.tar.gz && mv staticcheck/staticcheck ${GOPATH}/bin/staticcheck && chmod +x ${GOPATH}/bin/staticcheck && rm -f staticcheck_${GOOS}_${GOARCH}.tar.gz && rm -rf staticcheck)
-endif
-	@which misspell 1>/dev/null || (echo "Installing misspell" && GO111MODULE=off go get -u github.com/client9/misspell/cmd/misspell)
+	@which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0)
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
 
-verifiers: getdeps vet fmt lint staticcheck spelling
-
-vet:
-	@echo "Running $@ check"
-	@GO111MODULE=on go vet github.com/minio/minio/...
+verifiers: getdeps fmt lint
 
 fmt:
 	@echo "Running $@ check"
@@ -42,21 +30,8 @@ fmt:
 
 lint:
 	@echo "Running $@ check"
-	@GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd/...
-	@GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg/...
-
-staticcheck:
-	@echo "Running $@ check"
-	@GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/cmd/...
-	@GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/pkg/...
-
-spelling:
-	@echo "Running $@ check"
-	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find cmd/`
-	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find pkg/`
-	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find docs/`
-	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
-	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
+	@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
+	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ./.golangci.yml
 
 # Builds minio, runs the verifiers then runs the tests.
 check: test
@@ -71,19 +46,19 @@ test-race: verifiers build
 # Verify minio binary
 verify:
 	@echo "Verifying build with race"
-	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 # Verify healing of disks with minio binary
 verify-healing:
 	@echo "Verify healing build with race"
-	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 
 # Builds minio locally.
 build: checks
 	@echo "Building minio binary to './minio'"
-	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath  --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
 docker: build
 	@docker build -t $(TAG) . -f Dockerfile.dev

README.md

@@ -3,7 +3,7 @@
 
 [![MinIO](https://raw.githubusercontent.com/minio/minio/master/.github/logo.svg?sanitize=true)](https://min.io)
 
-MinIO is High Performance Object Storage released under Apache License v2.0. It is API compatible with Amazon S3 cloud storage service. Using MinIO build high performance infrastructure for machine learning, analytics and application data workloads.
+MinIO is a High Performance Object Storage released under Apache License v2.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads.
 
 ## Docker Container
 ### Stable
@@ -22,7 +22,7 @@ docker run -p 9000:9000 minio/minio:edge server /data
 
 ## macOS
 ### Homebrew (recommended)
-Install minio packages using [Homebrew](http://brew.sh/)
+Install minio packages using [Homebrew](https://brew.sh/)
 ```sh
 brew install minio/stable/minio
 minio server /data

browser/README.md

@@ -2,6 +2,7 @@
 
 ``MinIO Browser`` provides minimal set of UI to manage buckets and objects on ``minio`` server. ``MinIO Browser`` is written in javascript and released under [Apache 2.0 License](./LICENSE).
 
+
 ## Installation
 
 ### Install node
@@ -11,6 +12,11 @@ exec -l $SHELL
 nvm install stable
 ```
 
+### Install node dependencies
+```sh
+npm install
+```
+
 ### Install `go-bindata` and `go-bindata-assetfs`
 
 If you do not have a working Golang environment, please follow [Install Golang](https://golang.org/doc/install)
@@ -30,13 +36,16 @@ npm run release
 
 This generates ui-assets.go in the current directory. Now do `make` in the parent directory to build the minio binary with the newly generated ``ui-assets.go``
 
+
+## Run MinIO Browser with live reload
+
 ### Run MinIO Browser with live reload
 
 ```sh
 npm run dev
 ```
 
-Open [http://localhost:8080/minio/](http://localhost:8080/minio/) in your browser to play with the application
+Open [http://localhost:8080/minio/](http://localhost:8080/minio/) in your browser to play with the application.
 
 ### Run MinIO Browser with live reload on custom port
 
@@ -54,7 +63,7 @@ index 3ccdaba..9496c56 100644
 +    port: 8888,
      proxy: {
        '/minio/webrpc': {
-        target: 'http://localhost:9000',
+         target: 'http://localhost:9000',
 @@ -97,7 +98,7 @@ var exports = {
  if (process.env.NODE_ENV === 'dev') {
    exports.entry = [
@@ -70,4 +79,102 @@ index 3ccdaba..9496c56 100644
 npm run dev
 ```
 
-Open [http://localhost:8888/minio/](http://localhost:8888/minio/) in your browser to play with the application
+Open [http://localhost:8888/minio/](http://localhost:8888/minio/) in your browser to play with the application.
+
+### Run MinIO Browser with live reload on any IP
+
+Edit `browser/webpack.config.js`
+
+```diff
+diff --git a/browser/webpack.config.js b/browser/webpack.config.js
+index 8bdbba53..139f6049 100644
+--- a/browser/webpack.config.js
++++ b/browser/webpack.config.js
+@@ -71,6 +71,7 @@ var exports = {
+     historyApiFallback: {
+       index: '/minio/'
+     },
++    host: '0.0.0.0',
+     proxy: {
+       '/minio/webrpc': {
+         target: 'http://localhost:9000',
+```
+
+```sh
+npm run dev
+```
+
+Open [http://IP:8080/minio/](http://IP:8080/minio/) in your browser to play with the application.
+
+
+## Run tests
+
+    npm run test
+
+
+## Docker development environment
+
+This approach will download the sources on your machine such that you are able to use your IDE or editor of choice.
+A Docker container will be used in order to provide a controlled build environment without messing with your host system.
+
+### Prepare host system
+
+Install [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) and [Docker](https://docs.docker.com/get-docker/).
+
+### Development within container
+
+Prepare and build container
+```
+git clone git@github.com:minio/minio.git
+cd minio
+docker build -t minio-dev -f Dockerfile.dev.browser .
+```
+
+Run container, build and run core
+```sh
+docker run -it --rm --name minio-dev -v "$PWD":/minio minio-dev
+
+cd /minio/browser
+npm install
+npm run release
+cd /minio
+make
+./minio server /data
+```
+Note `Endpoint` IP (the one which is _not_ `127.0.0.1`), `AccessKey` and `SecretKey` (both default to `minioadmin`) in order to enter them in the browser later.
+
+
+Open another terminal.
+Connect to container
+```sh
+docker exec -it minio-dev bash
+```
+
+Apply patch to allow access from outside container
+```sh
+cd /minio
+git apply --ignore-whitespace <<EOF
+diff --git a/browser/webpack.config.js b/browser/webpack.config.js
+index 8bdbba53..139f6049 100644
+--- a/browser/webpack.config.js
++++ b/browser/webpack.config.js
+@@ -71,6 +71,7 @@ var exports = {
+     historyApiFallback: {
+       index: '/minio/'
+     },
++    host: '0.0.0.0',
+     proxy: {
+       '/minio/webrpc': {
+         target: 'http://localhost:9000',
+EOF
+```
+
+Build and run frontend with auto-reload
+```sh
+cd /minio/browser
+npm install
+npm run dev
+```
+
+Open [http://IP:8080/minio/](http://IP:8080/minio/) in your browser to play with the application.
+

browser/app/js/browser/StorageInfo.js

@@ -26,13 +26,10 @@ export class StorageInfo extends React.Component {
   }
   render() {
     const { used } = this.props.storageInfo
-
-    if (!used) {
+    if (!used || used == 0) {
       return <noscript />
     }
 
-    const totalUsed = used.reduce((v1, v2) => v1 + v2, 0)
-
     return (
       <div className="feh-used">
         <div className="fehu-chart">
@@ -41,7 +38,7 @@ export class StorageInfo extends React.Component {
         <ul>
           <li>
             <span>Used: </span>
-            {humanize.filesize(totalUsed)}
+            {humanize.filesize(used)}
           </li>
         </ul>
       </div>

browser/app/js/browser/__tests__/StorageInfo.test.js

@@ -21,7 +21,7 @@ import { StorageInfo } from "../StorageInfo"
 describe("StorageInfo", () => {
   it("should render without crashing", () => {
     shallow(
-      <StorageInfo storageInfo={{ used: [60] }} fetchStorageInfo={jest.fn()} />
+      <StorageInfo storageInfo={ {used: 60} } fetchStorageInfo={jest.fn()} />
     )
   })
 
@@ -29,7 +29,7 @@ describe("StorageInfo", () => {
     const fetchStorageInfo = jest.fn()
     shallow(
       <StorageInfo
-        storageInfo={{ used: [60] }}
+        storageInfo={ {used: 60} }
         fetchStorageInfo={fetchStorageInfo}
       />
     )
@@ -40,7 +40,7 @@ describe("StorageInfo", () => {
     const fetchStorageInfo = jest.fn()
     const wrapper = shallow(
       <StorageInfo
-        storageInfo={{ used: null }}
+      storageInfo={ {used: 0} }
         fetchStorageInfo={fetchStorageInfo}
       />
     )

browser/app/js/browser/__tests__/actions.test.js

@@ -20,7 +20,9 @@ import * as actionsCommon from "../actions"
 
 jest.mock("../../web", () => ({
   StorageInfo: jest.fn(() => {
-    return Promise.resolve({ storageInfo: { Used: [60] } })
+    return Promise.resolve({
+      used: 60
+    })
   }),
   ServerInfo: jest.fn(() => {
     return Promise.resolve({
@@ -39,7 +41,7 @@ describe("Common actions", () => {
   it("creates common/SET_STORAGE_INFO after fetching the storage details ", () => {
     const store = mockStore()
     const expectedActions = [
-      { type: "common/SET_STORAGE_INFO", storageInfo: { used: [60] } }
+      { type: "common/SET_STORAGE_INFO", storageInfo: { used: 60 } }
     ]
     return store.dispatch(actionsCommon.fetchStorageInfo()).then(() => {
       const actions = store.getActions()

browser/app/js/browser/__tests__/reducer.test.js

@@ -21,11 +21,7 @@ describe("common reducer", () => {
   it("should return the initial state", () => {
     expect(reducer(undefined, {})).toEqual({
       sidebarOpen: false,
-      storageInfo: {
-        total: [0],
-        free: [0],
-        used: [0]
-      },
+      storageInfo: {used: 0},
       serverInfo: {}
     })
   })
@@ -62,11 +58,11 @@ describe("common reducer", () => {
         {},
         {
           type: actionsCommon.SET_STORAGE_INFO,
-          storageInfo: { total: [100], free: [40] }
+          storageInfo: { }
         }
       )
     ).toEqual({
-      storageInfo: { total: [100], free: [40] }
+      storageInfo: { }
     })
   })
 

browser/app/js/browser/actions.js

@@ -33,8 +33,7 @@ export const fetchStorageInfo = () => {
   return function(dispatch) {
     return web.StorageInfo().then(res => {
       const storageInfo = {
-        total: res.storageInfo.Total,
-        used: res.storageInfo.Used
+        used: res.used
       }
       dispatch(setStorageInfo(storageInfo))
     })

browser/app/js/browser/reducer.js

@@ -19,7 +19,7 @@ import * as actionsCommon from "./actions"
 export default (
   state = {
     sidebarOpen: false,
-    storageInfo: { total: [0], free: [0], used: [0] },
+    storageInfo: {used: 0},
     serverInfo: {}
   },
   action

browser/app/js/buckets/actions.js

@@ -31,19 +31,38 @@ export const SET_POLICIES = "buckets/SET_POLICIES"
 
 export const fetchBuckets = () => {
   return function(dispatch) {
+    const { bucket, prefix } = pathSlice(history.location.pathname)
     return web.ListBuckets().then(res => {
       const buckets = res.buckets ? res.buckets.map(bucket => bucket.name) : []
-      dispatch(setList(buckets))
       if (buckets.length > 0) {
-        const { bucket, prefix } = pathSlice(history.location.pathname)
+        dispatch(setList(buckets))
         if (bucket && buckets.indexOf(bucket) > -1) {
           dispatch(selectBucket(bucket, prefix))
         } else {
           dispatch(selectBucket(buckets[0]))
         }
       } else {
-        dispatch(selectBucket(""))
-        history.replace("/")
+        if (bucket) {
+          dispatch(setList([bucket]))
+          dispatch(selectBucket(bucket, prefix))
+        } else {
+          dispatch(selectBucket(""))
+          history.replace("/")
+        }
+      }
+    })
+    .catch(err => {
+      if (bucket && err.message === "Access Denied." || err.message.indexOf('Prefix access is denied') > -1 ) {
+        dispatch(setList([bucket]))
+        dispatch(selectBucket(bucket, prefix))
+      } else {
+        dispatch(
+          alertActions.set({
+            type: "danger",
+            message: err.message,
+            autoClear: true,
+          })
+        )
       }
     })
   }

browser/app/js/objects/ObjectActions.js

@@ -47,6 +47,11 @@ export class ObjectActions extends React.Component {
       SHARE_OBJECT_EXPIRY_MINUTES
     )
   }
+  handleDownload(e) {
+    e.preventDefault()
+    const { object, downloadObject } = this.props
+    downloadObject(object.name)
+  }
   deleteObject() {
     const { object, deleteObject } = this.props
     deleteObject(object.name)
@@ -82,6 +87,7 @@ export class ObjectActions extends React.Component {
           <a
             href=""
             className="fiad-action"
+            title="Share"
             onClick={this.shareObject.bind(this)}
           >
             <i className="fas fa-share-alt" />
@@ -90,6 +96,7 @@ export class ObjectActions extends React.Component {
             <a
               href=""
               className="fiad-action"
+              title="Preview"
               onClick={this.showPreviewModal.bind(this)}
             >
               <i className="far fa-file-image" />
@@ -98,6 +105,15 @@ export class ObjectActions extends React.Component {
           <a
             href=""
             className="fiad-action"
+            title="Download"
+            onClick={this.handleDownload.bind(this)}
+          >
+            <i className="fas fa-cloud-download-alt" />
+          </a>
+          <a
+            href=""
+            className="fiad-action"
+            title="Delete"
             onClick={this.showDeleteConfirmModal.bind(this)}
           >
             <i className="fas fa-trash-alt" />
@@ -134,6 +150,7 @@ const mapStateToProps = (state, ownProps) => {
 
 const mapDispatchToProps = (dispatch) => {
   return {
+    downloadObject: object => dispatch(objectsActions.downloadObject(object)),
     shareObject: (object, days, hours, minutes) =>
       dispatch(objectsActions.shareObject(object, days, hours, minutes)),
     deleteObject: (object) => dispatch(objectsActions.deleteObject(object)),

browser/app/js/objects/PrefixActions.js

@@ -0,0 +1,95 @@
+/*
+ * MinIO Cloud Storage (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from "react"
+import { connect } from "react-redux"
+import { Dropdown } from "react-bootstrap"
+import DeleteObjectConfirmModal from "./DeleteObjectConfirmModal"
+import * as actions from "./actions"
+
+export class PrefixActions extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      showDeleteConfirmation: false,
+    }
+  }
+  handleDownload(e) {
+    e.preventDefault()
+    const { object, downloadPrefix } = this.props
+    downloadPrefix(object.name)
+  }
+  deleteObject() {
+    const { object, deleteObject } = this.props
+    deleteObject(object.name)
+  }
+  showDeleteConfirmModal(e) {
+    e.preventDefault()
+    this.setState({ showDeleteConfirmation: true })
+  }
+  hideDeleteConfirmModal() {
+    this.setState({
+      showDeleteConfirmation: false,
+    })
+  }
+  render() {
+    const { object, showShareObjectModal, shareObjectName } = this.props
+    return (
+      <Dropdown id={`obj-actions-${object.name}`}>
+        <Dropdown.Toggle noCaret className="fia-toggle" />
+        <Dropdown.Menu>
+          <a
+            href=""
+            className="fiad-action"
+            title="Download as zip"
+            onClick={this.handleDownload.bind(this)}
+          >
+            <i className="fas fa-cloud-download-alt" />
+          </a>
+          <a
+            href=""
+            className="fiad-action"
+            title="Delete"
+            onClick={this.showDeleteConfirmModal.bind(this)}
+          >
+            <i className="fas fa-trash-alt" />
+          </a>
+        </Dropdown.Menu>
+        {this.state.showDeleteConfirmation && (
+          <DeleteObjectConfirmModal
+            deleteObject={this.deleteObject.bind(this)}
+            hideDeleteConfirmModal={this.hideDeleteConfirmModal.bind(this)}
+          />
+        )}
+      </Dropdown>
+    )
+  }
+}
+
+const mapStateToProps = (state, ownProps) => {
+  return {
+    object: ownProps.object,
+  }
+}
+
+const mapDispatchToProps = (dispatch) => {
+  return {
+    downloadPrefix: object => dispatch(actions.downloadPrefix(object)),
+    deleteObject: (object) => dispatch(actions.deleteObject(object)),
+  }
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(PrefixActions)

browser/app/js/objects/PrefixContainer.js

@@ -17,22 +17,32 @@
 import React from "react"
 import { connect } from "react-redux"
 import ObjectItem from "./ObjectItem"
+import PrefixActions from "./PrefixActions"
 import * as actionsObjects from "./actions"
+import { getCheckedList } from "./selectors"
 
-export const PrefixContainer = ({ object, currentPrefix, selectPrefix }) => {
+export const PrefixContainer = ({
+  object,
+  currentPrefix,
+  checkedObjectsCount,
+  selectPrefix
+}) => {
   const props = {
     name: object.name,
     contentType: object.contentType,
     onClick: () => selectPrefix(`${currentPrefix}${object.name}`)
   }
-
+  if (checkedObjectsCount == 0) {
+    props.actionButtons = <PrefixActions object={object} />
+  }
   return <ObjectItem {...props} />
 }
 
 const mapStateToProps = (state, ownProps) => {
   return {
     object: ownProps.object,
-    currentPrefix: state.objects.currentPrefix
+    currentPrefix: state.objects.currentPrefix,
+    checkedObjectsCount: getCheckedList(state).length
   }
 }
 

browser/app/js/objects/PreviewObjectModal.js

@@ -23,7 +23,10 @@ class PreviewObjectModal extends React.Component {
     this.state = {
       url: "",
     }
-    props.getObjectURL(props.object.name, (url) => {
+  }
+
+  componentDidMount() {
+    this.props.getObjectURL(this.props.object.name, (url) => {
       this.setState({
         url: url,
       })

browser/app/js/objects/__tests__/ObjectActions.test.js

@@ -67,6 +67,20 @@ describe("ObjectActions", () => {
   })
 
 
+  it("should call downloadObject when single object is selected and download button is clicked", () => {
+    const downloadObject = jest.fn()
+    const wrapper = shallow(
+      <ObjectActions
+        object={{ name: "obj1" }}
+        currentPrefix={"pre1/"}
+        downloadObject={downloadObject} />
+    )
+    wrapper
+      .find("a")
+      .at(1)
+      .simulate("click", { preventDefault: jest.fn() })
+    expect(downloadObject).toHaveBeenCalled()
+  })
 
 
   it("should show PreviewObjectModal when preview action is clicked", () => {
@@ -106,7 +120,7 @@ describe("ObjectActions", () => {
     )
     expect(wrapper
       .find("a")
-      .length).toBe(2) // find only the other 2
+      .length).toBe(3) // find only the other 2
   })
 
   it("should call shareObject with object and expiry", () => {

browser/app/js/objects/__tests__/PrefixActions.test.js

@@ -0,0 +1,84 @@
+/*
+ * MinIO Cloud Storage (C) 2018 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from "react"
+import { shallow } from "enzyme"
+import { PrefixActions } from "../PrefixActions"
+
+describe("PrefixActions", () => {
+  it("should render without crashing", () => {
+    shallow(<PrefixActions object={{ name: "abc/" }} currentPrefix={"pre1/"} />)
+  })
+
+  it("should show DeleteObjectConfirmModal when delete action is clicked", () => {
+    const wrapper = shallow(
+      <PrefixActions object={{ name: "abc/" }} currentPrefix={"pre1/"} />
+    )
+    wrapper
+      .find("a")
+      .last()
+      .simulate("click", { preventDefault: jest.fn() })
+    expect(wrapper.state("showDeleteConfirmation")).toBeTruthy()
+    expect(wrapper.find("DeleteObjectConfirmModal").length).toBe(1)
+  })
+
+  it("should hide DeleteObjectConfirmModal when Cancel button is clicked", () => {
+    const wrapper = shallow(
+      <PrefixActions object={{ name: "abc/" }} currentPrefix={"pre1/"} />
+    )
+    wrapper
+      .find("a")
+      .last()
+      .simulate("click", { preventDefault: jest.fn() })
+    wrapper.find("DeleteObjectConfirmModal").prop("hideDeleteConfirmModal")()
+    wrapper.update()
+    expect(wrapper.state("showDeleteConfirmation")).toBeFalsy()
+    expect(wrapper.find("DeleteObjectConfirmModal").length).toBe(0)
+  })
+
+  it("should call deleteObject with object name", () => {
+    const deleteObject = jest.fn()
+    const wrapper = shallow(
+      <PrefixActions
+        object={{ name: "abc/" }}
+        currentPrefix={"pre1/"}
+        deleteObject={deleteObject}
+      />
+    )
+    wrapper
+      .find("a")
+      .last()
+      .simulate("click", { preventDefault: jest.fn() })
+    wrapper.find("DeleteObjectConfirmModal").prop("deleteObject")()
+    expect(deleteObject).toHaveBeenCalledWith("abc/")
+  })
+
+
+  it("should call downloadPrefix when single object is selected and download button is clicked", () => {
+    const downloadPrefix = jest.fn()
+    const wrapper = shallow(
+      <PrefixActions
+        object={{ name: "abc/" }}
+        currentPrefix={"pre1/"}
+        downloadPrefix={downloadPrefix} />
+    )
+    wrapper
+      .find("a")
+      .first()
+      .simulate("click", { preventDefault: jest.fn() })
+    expect(downloadPrefix).toHaveBeenCalled()
+  })
+})

browser/app/js/objects/__tests__/PrefixContainer.test.js

@@ -41,4 +41,22 @@ describe("PrefixContainer", () => {
     wrapper.find("Connect(ObjectItem)").prop("onClick")()
     expect(selectPrefix).toHaveBeenCalledWith("xyz/abc/")
   })
+
+  it("should pass actions to ObjectItem", () => {
+    const wrapper = shallow(
+      <PrefixContainer object={{ name: "abc/" }} checkedObjectsCount={0} />
+    )
+    expect(wrapper.find("Connect(ObjectItem)").prop("actionButtons")).not.toBe(
+      undefined
+    )
+  })
+
+  it("should pass empty actions to ObjectItem when checkedObjectCount is more than 0", () => {
+    const wrapper = shallow(
+      <PrefixContainer object={{ name: "abc/" }} checkedObjectsCount={1} />
+    )
+    expect(wrapper.find("Connect(ObjectItem)").prop("actionButtons")).toBe(
+      undefined
+    )
+  })
 })

browser/app/js/objects/__tests__/actions.test.js

@@ -68,6 +68,9 @@ jest.mock("../../web", () => ({
     .mockImplementationOnce(() => {
       return Promise.reject({ message: "Error in creating token" })
     })
+    .mockImplementationOnce(() => {
+      return Promise.resolve({ token: "test" })
+    })
     .mockImplementationOnce(() => {
       return Promise.resolve({ token: "test" })
     }),
@@ -485,6 +488,34 @@ describe("Objects actions", () => {
     })
   })
 
+  it("should download prefix", () => {
+    const open = jest.fn()
+    const send = jest.fn()
+    const xhrMockClass = () => ({
+      open: open,
+      send: send
+    })
+    window.XMLHttpRequest = jest.fn().mockImplementation(xhrMockClass)
+
+    const store = mockStore({
+      buckets: { currentBucket: "bk1" },
+      objects: { currentPrefix: "pre1/" }
+    })
+    return store.dispatch(actionsObjects.downloadPrefix("pre2/")).then(() => {
+      const requestUrl = `${
+        location.origin
+      }${minioBrowserPrefix}/zip?token=test`
+      expect(open).toHaveBeenCalledWith("POST", requestUrl, true)
+      expect(send).toHaveBeenCalledWith(
+        JSON.stringify({
+          bucketName: "bk1",
+          prefix: "pre1/",
+          objects: ["pre2/"]
+        })
+      )
+    })
+  })
+
   it("creates objects/CHECKED_LIST_ADD action", () => {
     const store = mockStore()
     const expectedActions = [

browser/app/js/objects/actions.js

@@ -360,6 +360,19 @@ export const downloadObject = (object) => {
   }
 }
 
+export const downloadPrefix = (object) => {
+  return function (dispatch, getState) {
+    return downloadObjects(
+      getCurrentBucket(getState()),
+      getCurrentPrefix(getState()),
+      [object],
+      `${object.slice(0, -1)}.zip`,
+      dispatch
+    )
+  }
+}
+
+
 export const checkObject = (object) => ({
   type: CHECKED_LIST_ADD,
   object,
@@ -376,21 +389,28 @@ export const resetCheckedList = () => ({
 
 export const downloadCheckedObjects = () => {
   return function (dispatch, getState) {
-    const state = getState()
+    return downloadObjects(
+      getCurrentBucket(getState()),
+      getCurrentPrefix(getState()),
+      getCheckedList(getState()),
+      null,
+      dispatch
+    )
+  }
+}
+
+const downloadObjects = (bucketName, prefix, objects, filename, dispatch) => {
     const req = {
-      bucketName: getCurrentBucket(state),
-      prefix: getCurrentPrefix(state),
-      objects: getCheckedList(state),
+      bucketName: bucketName,
+      prefix: prefix,
+      objects: objects,
     }
-    if (!web.LoggedIn()) {
-      const requestUrl = location.origin + "/minio/zip?token="
-      downloadZip(requestUrl, req, dispatch)
-    } else {
+    if (web.LoggedIn()) {
       return web
         .CreateURLToken()
         .then((res) => {
           const requestUrl = `${location.origin}${minioBrowserPrefix}/zip?token=${res.token}`
-          downloadZip(requestUrl, req, dispatch)
+          downloadZip(requestUrl, req, filename, dispatch)
         })
         .catch((err) =>
           dispatch(
@@ -400,11 +420,13 @@ export const downloadCheckedObjects = () => {
             })
           )
         )
+    } else {
+      const requestUrl = `${location.origin}${minioBrowserPrefix}/zip?token=`
+      downloadZip(requestUrl, req, filename, dispatch)
     }
-  }
 }
 
-const downloadZip = (url, req, dispatch) => {
+const downloadZip = (url, req, filename, dispatch) => {
   var anchor = document.createElement("a")
   document.body.appendChild(anchor)
 
@@ -422,7 +444,7 @@ const downloadZip = (url, req, dispatch) => {
       var separator = req.prefix.length > 1 ? "-" : ""
 
       anchor.href = blobUrl
-      anchor.download =
+      anchor.download = filename ||
         req.bucketName + separator + req.prefix.slice(0, -1) + ".zip"
 
       anchor.click()

browser/app/js/uploads/Dropzone.js

@@ -48,18 +48,29 @@ export class Dropzone extends React.Component {
     const rejectStyle = {
       backgroundColor: "#ffdddd"
     }
+    const getStyle = (isDragActive, isDragAccept, isDragReject) => ({
+      ...style,
+      ...(isDragActive ? activeStyle : {}),
+      ...(isDragReject ? rejectStyle : {})
+    })
 
     // disableClick means that it won't trigger a file upload box when
     // the user clicks on a file.
     return (
       <ReactDropzone
-        style={style}
-        activeStyle={activeStyle}
-        rejectStyle={rejectStyle}
-        disableClick={true}
         onDrop={this.onDrop.bind(this)}
       >
-        {this.props.children}
+        {({getRootProps, getInputProps, isDragActive, isDragAccept, isDragReject}) => (
+          <div
+            {...getRootProps({
+              onClick: event => event.stopPropagation()
+            })}
+            style={getStyle(isDragActive, isDragAccept, isDragReject)}
+          >
+            <input {...getInputProps()} />
+            {this.props.children}
+          </div>
+        )}
       </ReactDropzone>
     )
   }

browser/app/js/uploads/actions.js

@@ -89,11 +89,16 @@ export const uploadFile = file => {
       return
     }
     const currentPrefix = getCurrentPrefix(state)
-    const objectName = `${currentPrefix}${file.name}`
+    var _filePath = file.path || file.name
+    if (_filePath.charAt(0) == '/') {
+      _filePath = _filePath.substring(1)
+    }
+    const filePath = _filePath
+    const objectName = `${currentPrefix}${filePath}`
     const uploadUrl = `${
       window.location.origin
     }${minioBrowserPrefix}/upload/${currentBucket}/${objectName}`
-    const slug = `${currentBucket}-${currentPrefix}-${file.name}`
+    const slug = `${currentBucket}-${currentPrefix}-${filePath}`
 
     let xhr = new XMLHttpRequest()
     xhr.open("PUT", uploadUrl, true)
@@ -141,7 +146,7 @@ export const uploadFile = file => {
         dispatch(
           alertActions.set({
             type: "success",
-            message: "File '" + file.name + "' uploaded successfully."
+            message: "File '" + filePath + "' uploaded successfully."
           })
         )
         dispatch(objectsActions.selectPrefix(currentPrefix))
@@ -153,7 +158,7 @@ export const uploadFile = file => {
       dispatch(
         alertActions.set({
           type: "danger",
-          message: "Error occurred uploading '" + file.name + "'."
+          message: "Error occurred uploading '" + filePath + "'."
         })
       )
     })

browser/package.json

@@ -28,71 +28,70 @@
   },
   "homepage": "https://github.com/minio/minio",
   "devDependencies": {
-    "async": "^1.5.2",
+    "async": "^3.2.0",
     "babel-cli": "^6.26.0",
     "babel-core": "^6.26.3",
-    "babel-jest": "^22.1.0",
+    "babel-jest": "^23.6.0",
     "babel-loader": "^7.1.2",
     "babel-plugin-syntax-object-rest-spread": "^6.13.0",
-    "babel-plugin-transform-object-rest-spread": "^6.8.0",
-    "babel-polyfill": "^6.23.0",
+    "babel-plugin-transform-object-rest-spread": "^6.26.0",
+    "babel-polyfill": "^6.26.0",
     "babel-preset-es2015": "^6.14.0",
     "babel-preset-react": "^6.11.1",
     "babel-register": "^6.26.0",
-    "copy-webpack-plugin": "^4.6.0",
-    "css-loader": "^0.23.1",
-    "enzyme": "^3.10.0",
-    "enzyme-adapter-react-16": "^1.1.1",
-    "esformatter": "^0.10.0",
-    "esformatter-jsx": "^7.4.1",
+    "copy-webpack-plugin": "^6.0.1",
+    "css-loader": "^3.5.3",
+    "enzyme": "^3.11.0",
+    "enzyme-adapter-react-16": "^1.15.2",
+    "esformatter": "^0.11.3",
+    "esformatter-jsx": "^8.0.1",
     "esformatter-jsx-ignore": "^1.0.6",
-    "html-webpack-plugin": "^3.2.0",
-    "jest": "^22.1.4",
-    "jest-enzyme": "^4.0.2",
-    "json-loader": "^0.5.4",
-    "less": "^3.9.0",
-    "less-loader": "^4.1.0",
-    "purgecss-webpack-plugin": "^1.4.0",
-    "style-loader": "^0.13.1",
-    "url-loader": "^0.5.7",
-    "webpack-cli": "^3.2.0",
-    "webpack-dev-server": "^3.1.14"
+    "html-webpack-plugin": "^4.3.0",
+    "jest": "^23.6.0",
+    "jest-enzyme": "^7.1.2",
+    "json-loader": "^0.5.7",
+    "less": "^3.11.1",
+    "less-loader": "^6.1.0",
+    "purgecss-webpack-plugin": "^2.2.0",
+    "style-loader": "^1.2.1",
+    "url-loader": "^4.1.0",
+    "webpack-cli": "^3.3.11",
+    "webpack-dev-server": "^3.11.0"
   },
   "dependencies": {
-    "@fortawesome/fontawesome-free": "^5.10.0",
+    "@fortawesome/fontawesome-free": "^5.13.0",
     "bootstrap": "^3.4.1",
-    "classnames": "^2.2.3",
-    "core-js": "^3.2.1",
-    "expect": "^1.20.2",
-    "glob-all": "^3.1.0",
-    "history": "^4.7.2",
+    "classnames": "^2.2.6",
+    "core-js": "^3.6.5",
+    "expect": "^26.0.1",
+    "glob-all": "^3.2.1",
+    "history": "^4.10.1",
     "humanize": "0.0.9",
     "identity-obj-proxy": "^3.0.0",
-    "json-loader": "^0.5.4",
     "jwt-decode": "^2.2.0",
-    "local-storage-fallback": "^4.0.2",
+    "local-storage-fallback": "^4.1.1",
     "material-design-iconic-font": "^2.2.0",
-    "mime-db": "^1.25.0",
-    "mime-types": "^2.1.13",
-    "moment": "^2.24.0",
-    "query-string": "^6.8.2",
-    "react": "^16.2.0",
-    "react-addons-test-utils": "^0.14.8",
-    "react-bootstrap": "^0.32.0",
-    "react-copy-to-clipboard": "^5.0.1",
+    "mime-db": "^1.44.0",
+    "mime-types": "^2.1.27",
+    "moment": "^2.26.0",
+    "query-string": "^6.12.1",
+    "react": "^16.13.1",
+    "react-addons-test-utils": "^15.6.2",
+    "react-bootstrap": "^0.32.4",
+    "react-copy-to-clipboard": "^5.0.2",
     "react-custom-scrollbars": "^4.2.1",
-    "react-dom": "^16.2.0",
-    "react-dropzone": "^4.2.3",
-    "react-infinite-scroller": "^1.0.6",
+    "react-dom": "^16.13.1",
+    "react-dropzone": "^11.0.1",
+    "react-infinite-scroller": "^1.2.4",
     "react-onclickout": "^2.0.8",
-    "react-redux": "^5.0.6",
-    "react-router-dom": "^4.2.0",
-    "redux": "^3.7.2",
-    "redux-mock-store": "^1.5.1",
-    "redux-thunk": "^2.2.0",
-    "reselect": "^3.0.1",
-    "superagent": "^3.8.2",
+    "react-redux": "^5.1.2",
+    "react-router-dom": "^5.2.0",
+    "redux": "^4.0.5",
+    "redux-mock-store": "^1.5.4",
+    "redux-thunk": "^2.3.0",
+    "reselect": "^4.0.0",
+    "superagent": "^5.2.2",
     "superagent-es6-promise": "^1.0.0",
-    "webpack": "^4.28.3"
+    "webpack": "^4.43.0"
   }
 }

browser/webpack.config.js

@@ -73,17 +73,17 @@ var exports = {
     },
     proxy: {
       '/minio/webrpc': {
-	target: 'http://localhost:9000',
-  secure: false,
-  headers: {'Host': "localhost:9000"}
+        target: 'http://localhost:9000',
+        secure: false,
+        headers: {'Host': "localhost:9000"}
       },
       '/minio/upload/*': {
-	target: 'http://localhost:9000',
-	secure: false
+        target: 'http://localhost:9000',
+        secure: false
       },
       '/minio/download/*': {
-	target: 'http://localhost:9000',
-	secure: false
+        target: 'http://localhost:9000',
+        secure: false
       },
       '/minio/zip': {
         target: 'http://localhost:9000',
@@ -92,7 +92,7 @@ var exports = {
     }
   },
   plugins: [
-    new CopyWebpackPlugin([
+    new CopyWebpackPlugin({patterns: [
       {from: 'app/css/loader.css'},
       {from: 'app/img/browsers/chrome.png'},
       {from: 'app/img/browsers/firefox.png'},
@@ -102,7 +102,7 @@ var exports = {
       {from: 'app/img/favicon/favicon-32x32.png'},
       {from: 'app/img/favicon/favicon-96x96.png'},
       {from: 'app/index.html'}
-    ]),
+    ]}),
     new webpack.ContextReplacementPlugin(/moment[\\\/]locale$/, /^\.\/(en)$/),
     new PurgecssPlugin({
       paths: glob.sync([

browser/webpack.production.config.js

@@ -67,7 +67,7 @@ var exports = {
     fs:'empty'
   },
   plugins: [
-    new CopyWebpackPlugin([
+    new CopyWebpackPlugin({patterns: [
       {from: 'app/css/loader.css'},
       {from: 'app/img/browsers/chrome.png'},
       {from: 'app/img/browsers/firefox.png'},
@@ -77,7 +77,7 @@ var exports = {
       {from: 'app/img/favicon/favicon-32x32.png'},
       {from: 'app/img/favicon/favicon-96x96.png'},
       {from: 'app/index.html'}
-    ]),
+    ]}),
     new webpack.ContextReplacementPlugin(/moment[\\\/]locale$/, /^\.\/(en)$/),
     new PurgecssPlugin({
       paths: glob.sync([

cmd/acl-handlers.go

@@ -100,18 +100,18 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
 		}
 
 		if len(acl.AccessControlList.Grants) == 0 {
-			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
 		if acl.AccessControlList.Grants[0].Permission != "FULL_CONTROL" {
-			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
 	if aclHeader != "" && aclHeader != "private" {
-		writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -159,6 +159,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 		},
 		Permission: "FULL_CONTROL",
 	})
+
 	if err := xml.NewEncoder(w).Encode(acl); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
@@ -214,18 +215,18 @@ func (api objectAPIHandlers) PutObjectACLHandler(w http.ResponseWriter, r *http.
 		}
 
 		if len(acl.AccessControlList.Grants) == 0 {
-			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
 		if acl.AccessControlList.Grants[0].Permission != "FULL_CONTROL" {
-			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
 	if aclHeader != "" && aclHeader != "private" {
-		writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/admin-handlers-config-kv.go

@@ -60,7 +60,9 @@ func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *htt
 
 // DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
 func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "DelConfigKVHandler")
+	ctx := newContext(r, w, "DeleteConfigKV")
+
+	defer logger.AuditLog(w, r, "DeleteConfigKV", mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -100,7 +102,9 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 // SetConfigKVHandler - PUT /minio/admin/v3/set-config-kv
 func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetConfigKVHandler")
+	ctx := newContext(r, w, "SetConfigKV")
+
+	defer logger.AuditLog(w, r, "SetConfigKV", mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -159,7 +163,9 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 // GetConfigKVHandler - GET /minio/admin/v3/get-config-kv?key={key}
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetConfigKVHandler")
+	ctx := newContext(r, w, "GetConfigKV")
+
+	defer logger.AuditLog(w, r, "GetConfigKV", mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -195,7 +201,9 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 }
 
 func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ClearConfigHistoryKVHandler")
+	ctx := newContext(r, w, "ClearConfigHistoryKV")
+
+	defer logger.AuditLog(w, r, "ClearConfigHistoryKV", mustGetClaimsFromToken(r))
 
 	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -230,7 +238,9 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
 
 // RestoreConfigHistoryKVHandler - restores a config with KV settings for the given KV id.
 func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "RestoreConfigHistoryKVHandler")
+	ctx := newContext(r, w, "RestoreConfigHistoryKV")
+
+	defer logger.AuditLog(w, r, "RestoreConfigHistoryKV", mustGetClaimsFromToken(r))
 
 	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -276,7 +286,9 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 
 // ListConfigHistoryKVHandler - lists all the KV ids.
 func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListConfigHistoryKVHandler")
+	ctx := newContext(r, w, "ListConfigHistoryKV")
+
+	defer logger.AuditLog(w, r, "ListConfigHistoryKV", mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -314,7 +326,9 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 
 // HelpConfigKVHandler - GET /minio/admin/v3/help-config-kv?subSys={subSys}&key={key}
 func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "HelpConfigKVHandler")
+	ctx := newContext(r, w, "HelpConfigKV")
+
+	defer logger.AuditLog(w, r, "HelpHistoryKV", mustGetClaimsFromToken(r))
 
 	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -340,7 +354,9 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
 
 // SetConfigHandler - PUT /minio/admin/v3/config
 func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetConfigHandler")
+	ctx := newContext(r, w, "SetConfig")
+
+	defer logger.AuditLog(w, r, "SetConfig", mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
@@ -395,7 +411,9 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 // GetConfigHandler - GET /minio/admin/v3/config
 // Get config.json of this minio setup.
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetConfigHandler")
+	ctx := newContext(r, w, "GetConfig")
+
+	defer logger.AuditLog(w, r, "GetConfig", mustGetClaimsFromToken(r))
 
 	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {

cmd/admin-handlers-users.go

@@ -55,6 +55,8 @@ func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.R
 func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveUser")
 
+	defer logger.AuditLog(w, r, "RemoveUser", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
 	if objectAPI == nil {
 		return
@@ -91,6 +93,8 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListUsers")
 
+	defer logger.AuditLog(w, r, "ListUsers", mustGetClaimsFromToken(r))
+
 	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.ListUsersAdminAction)
 	if objectAPI == nil {
 		return
@@ -123,6 +127,8 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetUserInfo")
 
+	defer logger.AuditLog(w, r, "GetUserInfo", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetUserAdminAction)
 	if objectAPI == nil {
 		return
@@ -150,6 +156,8 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "UpdateGroupMembers")
 
+	defer logger.AuditLog(w, r, "UpdateGroupMembers", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.AddUserToGroupAdminAction)
 	if objectAPI == nil {
 		return
@@ -193,6 +201,8 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetGroup")
 
+	defer logger.AuditLog(w, r, "GetGroup", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetGroupAdminAction)
 	if objectAPI == nil {
 		return
@@ -220,6 +230,8 @@ func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListGroups")
 
+	defer logger.AuditLog(w, r, "ListGroups", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListGroupsAdminAction)
 	if objectAPI == nil {
 		return
@@ -244,6 +256,8 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetGroupStatus")
 
+	defer logger.AuditLog(w, r, "SetGroupStatus", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.EnableGroupAdminAction)
 	if objectAPI == nil {
 		return
@@ -279,6 +293,8 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetUserStatus")
 
+	defer logger.AuditLog(w, r, "SetUserStatus", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.EnableUserAdminAction)
 	if objectAPI == nil {
 		return
@@ -312,6 +328,8 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddUser")
 
+	defer logger.AuditLog(w, r, "AddUser", mustGetClaimsFromToken(r))
+
 	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.CreateUserAdminAction)
 	if objectAPI == nil {
 		return
@@ -365,6 +383,8 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddServiceAccount")
 
+	defer logger.AuditLog(w, r, "AddServiceAccount", mustGetClaimsFromToken(r))
+
 	// Get current object layer instance.
 	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
@@ -442,6 +462,8 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListServiceAccounts")
 
+	defer logger.AuditLog(w, r, "ListServiceAccounts", mustGetClaimsFromToken(r))
+
 	// Get current object layer instance.
 	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
@@ -495,6 +517,8 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteServiceAccount")
 
+	defer logger.AuditLog(w, r, "DeleteServiceAccount", mustGetClaimsFromToken(r))
+
 	// Get current object layer instance.
 	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
@@ -548,10 +572,118 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 	writeSuccessNoContent(w)
 }
 
+// AccountUsageInfoHandler returns usage
+func (a adminAPIHandlers) AccountUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AccountUsageInfo")
+
+	defer logger.AuditLog(w, r, "AccountUsageInfo", mustGetClaimsFromToken(r))
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	// Set prefix value for "s3:prefix" policy conditionals.
+	r.Header.Set("prefix", "")
+
+	// Set delimiter value for "s3:delimiter" policy conditionals.
+	r.Header.Set("delimiter", SlashSeparator)
+
+	isAllowedAccess := func(bucketName string) (rd, wr bool) {
+		// Use the following trick to filter in place
+		// https://github.com/golang/go/wiki/SliceTricks#filter-in-place
+		if globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     cred.AccessKey,
+			Action:          iampolicy.ListBucketAction,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			IsOwner:         owner,
+			ObjectName:      "",
+			Claims:          claims,
+		}) {
+			rd = true
+		}
+
+		if globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     cred.AccessKey,
+			Action:          iampolicy.PutObjectAction,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			IsOwner:         owner,
+			ObjectName:      "",
+			Claims:          claims,
+		}) {
+			wr = true
+		}
+
+		return rd, wr
+	}
+
+	buckets, err := objectAPI.ListBuckets(ctx)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Load the latest calculated data usage
+	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+	if err != nil {
+		// log the error, continue with the accounting response
+		logger.LogIf(ctx, err)
+	}
+
+	accountName := cred.AccessKey
+	if cred.ParentUser != "" {
+		accountName = cred.ParentUser
+	}
+
+	acctInfo := madmin.AccountUsageInfo{
+		AccountName: accountName,
+	}
+
+	for _, bucket := range buckets {
+		rd, wr := isAllowedAccess(bucket.Name)
+		if rd || wr {
+			var size uint64
+			// Fetch the data usage of the current bucket
+			if !dataUsageInfo.LastUpdate.IsZero() {
+				size = dataUsageInfo.BucketsUsage[bucket.Name].Size
+			}
+			acctInfo.Buckets = append(acctInfo.Buckets, madmin.BucketUsageInfo{
+				Name:    bucket.Name,
+				Created: bucket.Created,
+				Size:    size,
+				Access: madmin.AccountAccess{
+					Read:  rd,
+					Write: wr,
+				},
+			})
+		}
+	}
+
+	usageInfoJSON, err := json.Marshal(acctInfo)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, usageInfoJSON)
+}
+
 // InfoCannedPolicyV2 - GET /minio/admin/v2/info-canned-policy?name={policyName}
 func (a adminAPIHandlers) InfoCannedPolicyV2(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicyV2")
 
+	defer logger.AuditLog(w, r, "InfoCannedPolicyV2", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
 	if objectAPI == nil {
 		return
@@ -577,6 +709,8 @@ func (a adminAPIHandlers) InfoCannedPolicyV2(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicy")
 
+	defer logger.AuditLog(w, r, "InfoCannedPolicy", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
 	if objectAPI == nil {
 		return
@@ -596,6 +730,8 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) ListCannedPoliciesV2(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListCannedPoliciesV2")
 
+	defer logger.AuditLog(w, r, "ListCannedPoliciesV2", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
@@ -628,6 +764,8 @@ func (a adminAPIHandlers) ListCannedPoliciesV2(w http.ResponseWriter, r *http.Re
 func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListCannedPolicies")
 
+	defer logger.AuditLog(w, r, "ListCannedPolicies", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
@@ -660,6 +798,8 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveCannedPolicy")
 
+	defer logger.AuditLog(w, r, "RemoveCannedPolicy", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.DeletePolicyAdminAction)
 	if objectAPI == nil {
 		return
@@ -686,6 +826,8 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddCannedPolicy")
 
+	defer logger.AuditLog(w, r, "AddCannedPolicy", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.CreatePolicyAdminAction)
 	if objectAPI == nil {
 		return
@@ -736,6 +878,8 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetPolicyForUserOrGroup")
 
+	defer logger.AuditLog(w, r, "SetPolicyForUserOrGroup", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
 	if objectAPI == nil {
 		return

cmd/admin-handlers.go

@@ -21,6 +21,7 @@ import (
 	"crypto/subtle"
 	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -33,7 +34,6 @@ import (
 	"strings"
 	"time"
 
-	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
 
 	"github.com/minio/minio/cmd/config"
@@ -52,7 +52,6 @@ import (
 
 const (
 	maxEConfigJSONSize = 262272
-	defaultNetPerfSize = 100 * humanize.MiByte
 )
 
 // Type-safe query params.
@@ -101,6 +100,8 @@ func updateServer(updateURL, sha256Hex string, latestReleaseTime time.Time) (us
 func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerUpdate")
 
+	defer logger.AuditLog(w, r, "ServerUpdate", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerUpdateAdminAction)
 	if objectAPI == nil {
 		return
@@ -174,20 +175,17 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 	}
 }
 
-// ServiceActionHandler - POST /minio/admin/v3/service?action={action}
+// ServiceHandler - POST /minio/admin/v3/service?action={action}
 // ----------
 // restarts/stops minio server gracefully. In a distributed setup,
-func (a adminAPIHandlers) ServiceActionHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ServiceAction")
+func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "Service")
+
+	defer logger.AuditLog(w, r, "Service", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	action := vars["action"]
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, "")
-	if objectAPI == nil {
-		return
-	}
-
 	var serviceSig serviceSignal
 	switch madmin.ServiceAction(action) {
 	case madmin.ServiceActionRestart:
@@ -200,6 +198,16 @@ func (a adminAPIHandlers) ServiceActionHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
+	var objectAPI ObjectLayer
+	if serviceSig == serviceRestart {
+		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceRestartAdminAction)
+	} else {
+		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceStopAdminAction)
+	}
+	if objectAPI == nil {
+		return
+	}
+
 	// Notify all other MinIO peers signal service.
 	for _, nerr := range globalNotificationSys.SignalService(serviceSig) {
 		if nerr.Err != nil {
@@ -268,12 +276,16 @@ type ServerInfo struct {
 // Get server information
 func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StorageInfo")
+
+	defer logger.AuditLog(w, r, "StorageInfo", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.StorageInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
-	storageInfo := objectAPI.StorageInfo(ctx, false)
+	// ignores any errors here.
+	storageInfo, _ := objectAPI.StorageInfo(ctx, false)
 
 	// Marshal API response
 	jsonBytes, err := json.Marshal(storageInfo)
@@ -293,6 +305,9 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 // Get server/cluster data usage info
 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DataUsageInfo")
+
+	defer logger.AuditLog(w, r, "DataUsageInfo", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction)
 	if objectAPI == nil {
 		return
@@ -313,67 +328,6 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
 	writeSuccessResponseJSON(w, dataUsageInfoJSON)
 }
 
-func (a adminAPIHandlers) AccountingUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AccountingUsageInfo")
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AccountingUsageInfoAdminAction)
-	if objectAPI == nil {
-		return
-	}
-
-	var accountingUsageInfo = make(map[string]madmin.BucketAccountingUsage)
-
-	buckets, err := objectAPI.ListBuckets(ctx)
-	if err != nil {
-		// writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	users, err := globalIAMSys.ListUsers()
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Load the latest calculated data usage
-	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
-	if err != nil {
-		logger.LogIf(ctx, err)
-	}
-
-	// Calculate for each bucket, which users are allowed to access to it
-	for _, bucket := range buckets {
-		bucketUsageInfo := madmin.BucketAccountingUsage{}
-
-		// Fetch the data usage of the current bucket
-		if !dataUsageInfo.LastUpdate.IsZero() && dataUsageInfo.BucketsSizes != nil {
-			bucketUsageInfo.Size = dataUsageInfo.BucketsSizes[bucket.Name]
-		}
-
-		for user := range users {
-			rd, wr, custom := globalIAMSys.GetAccountAccess(user, bucket.Name)
-			if rd || wr || custom {
-				bucketUsageInfo.AccessList = append(bucketUsageInfo.AccessList, madmin.AccountAccess{
-					AccountName: user,
-					Read:        rd,
-					Write:       wr,
-					Custom:      custom,
-				})
-			}
-		}
-
-		accountingUsageInfo[bucket.Name] = bucketUsageInfo
-	}
-
-	usageInfoJSON, err := json.Marshal(accountingUsageInfo)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	writeSuccessResponseJSON(w, usageInfoJSON)
-}
-
 func newLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
 	entry := &madmin.LockEntry{
 		Timestamp:  l.Timestamp,
@@ -430,6 +384,8 @@ type PeerLocks struct {
 func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "TopLocks")
 
+	defer logger.AuditLog(w, r, "TopLocks", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.TopLocksAdminAction)
 	if objectAPI == nil {
 		return
@@ -475,6 +431,8 @@ type StartProfilingResult struct {
 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StartProfiling")
 
+	defer logger.AuditLog(w, r, "StartProfiling", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
 	if objectAPI == nil {
 		return
@@ -571,6 +529,8 @@ func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DownloadProfiling")
 
+	defer logger.AuditLog(w, r, "DownloadProfiling", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
 	if objectAPI == nil {
 		return
@@ -663,6 +623,8 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "Heal")
 
+	defer logger.AuditLog(w, r, "Heal", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
 	if objectAPI == nil {
 		return
@@ -746,8 +708,8 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	// find number of disks in the setup
-	info := objectAPI.StorageInfo(ctx, false)
+	// find number of disks in the setup, ignore any errors here.
+	info, _ := objectAPI.StorageInfo(ctx, false)
 	numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum()
 
 	healPath := pathJoin(hip.bucket, hip.objPrefix)
@@ -809,6 +771,8 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HealBackgroundStatus")
 
+	defer logger.AuditLog(w, r, "HealBackgroundStatus", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
 	if objectAPI == nil {
 		return
@@ -935,13 +899,26 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 			HTTPStatusCode: e.StatusCode,
 		}
 	default:
-		if err == errConfigNotFound {
+		switch {
+		case errors.Is(err, errConfigNotFound):
 			apiErr = APIError{
 				Code:           "XMinioConfigError",
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusNotFound,
 			}
-		} else {
+		case errors.Is(err, errIAMActionNotAllowed):
+			apiErr = APIError{
+				Code:           "XMinioIAMActionNotAllowed",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusForbidden,
+			}
+		case errors.Is(err, errIAMNotInitialized):
+			apiErr = APIError{
+				Code:           "XMinioIAMNotInitialized",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusServiceUnavailable,
+			}
+		default:
 			apiErr = errorCodes.ToAPIErrWithErr(toAdminAPIErrCode(ctx, err), err)
 		}
 	}
@@ -970,6 +947,7 @@ func mustTrace(entry interface{}, trcAll, errOnly bool) bool {
 // The handler sends http trace to the connected HTTP client.
 func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HTTPTrace")
+
 	trcAll := r.URL.Query().Get("all") == "true"
 	trcErr := r.URL.Query().Get("err") == "true"
 
@@ -1025,6 +1003,8 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ConsoleLog")
 
+	defer logger.AuditLog(w, r, "ConsoleLog", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConsoleLogAdminAction)
 	if objectAPI == nil {
 		return
@@ -1092,7 +1072,9 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 
 // KMSKeyStatusHandler - GET /minio/admin/v3/kms/key/status?key-id=<master-key-id>
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "KMSKeyStatusHandler")
+	ctx := newContext(r, w, "KMSKeyStatus")
+
+	defer logger.AuditLog(w, r, "KMSKeyStatus", mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
@@ -1164,6 +1146,9 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 // Get server on-board diagnostics
 func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "OBDInfo")
+
+	defer logger.AuditLog(w, r, "OBDInfo", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.OBDInfoAdminAction)
 	if objectAPI == nil {
 		return
@@ -1215,7 +1200,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 		defer close(obdInfoCh)
 
 		if cpu, ok := vars["syscpu"]; ok && cpu == "true" {
-			cpuInfo := getLocalCPUOBDInfo(deadlinedCtx)
+			cpuInfo := getLocalCPUOBDInfo(deadlinedCtx, r)
 
 			obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo)
 			obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, globalNotificationSys.CPUOBDInfo(deadlinedCtx)...)
@@ -1223,7 +1208,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 		}
 
 		if diskHw, ok := vars["sysdiskhw"]; ok && diskHw == "true" {
-			diskHwInfo := getLocalDiskHwOBD(deadlinedCtx)
+			diskHwInfo := getLocalDiskHwOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo)
 			obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, globalNotificationSys.DiskHwOBDInfo(deadlinedCtx)...)
@@ -1231,7 +1216,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 		}
 
 		if osInfo, ok := vars["sysosinfo"]; ok && osInfo == "true" {
-			osInfo := getLocalOsInfoOBD(deadlinedCtx)
+			osInfo := getLocalOsInfoOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo)
 			obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, globalNotificationSys.OsOBDInfo(deadlinedCtx)...)
@@ -1239,7 +1224,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 		}
 
 		if mem, ok := vars["sysmem"]; ok && mem == "true" {
-			memInfo := getLocalMemOBD(deadlinedCtx)
+			memInfo := getLocalMemOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo)
 			obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, globalNotificationSys.MemOBDInfo(deadlinedCtx)...)
@@ -1247,7 +1232,7 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 		}
 
 		if proc, ok := vars["sysprocess"]; ok && proc == "true" {
-			procInfo := getLocalProcOBD(deadlinedCtx)
+			procInfo := getLocalProcOBD(deadlinedCtx, r)
 
 			obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo)
 			obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, globalNotificationSys.ProcOBDInfo(deadlinedCtx)...)
@@ -1281,17 +1266,28 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 				Error:    errStr,
 			}
 			obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBD)
+			partialWrite(obdInfo)
 
 			// Notify all other MinIO peers to report drive obd numbers
-			driveOBDs := globalNotificationSys.DriveOBDInfo(deadlinedCtx)
-			obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBDs...)
-
+			driveOBDs := globalNotificationSys.DriveOBDInfoChan(deadlinedCtx)
+			for obd := range driveOBDs {
+				obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, obd)
+				partialWrite(obdInfo)
+			}
 			partialWrite(obdInfo)
 		}
 
 		if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistXL {
 			obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx))
-			obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.DispatchNetOBDInfo(deadlinedCtx)...)
+			partialWrite(obdInfo)
+
+			netOBDs := globalNotificationSys.DispatchNetOBDChan(deadlinedCtx)
+			for obd := range netOBDs {
+				obdInfo.Perf.Net = append(obdInfo.Perf.Net, obd)
+				partialWrite(obdInfo)
+			}
+			partialWrite(obdInfo)
+
 			obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx)
 			partialWrite(obdInfo)
 		}
@@ -1326,6 +1322,9 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 // Get server information
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerInfo")
+
+	defer logger.AuditLog(w, r, "ServerInfo", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
 	if objectAPI == nil {
 		return
@@ -1344,11 +1343,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
 	if err == nil {
 		buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
-		objects = madmin.Objects{Count: dataUsageInfo.ObjectsCount}
+		objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
 		usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
 	}
 
-	infoMsg := madmin.InfoMessage{}
 	vault := fetchVaultStatus(cfg)
 
 	ldap := madmin.LDAP{}
@@ -1370,8 +1368,8 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	// Get the notification target info
 	notifyTarget := fetchLambdaInfo(cfg)
 
-	// Fetching the Storage information
-	storageInfo := objectAPI.StorageInfo(ctx, false)
+	// Fetching the Storage information, ignore any errors.
+	storageInfo, _ := objectAPI.StorageInfo(ctx, false)
 
 	var OnDisks int
 	var OffDisks int
@@ -1453,7 +1451,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		Notifications: notifyTarget,
 	}
 
-	infoMsg = madmin.InfoMessage{
+	infoMsg := madmin.InfoMessage{
 		Mode:         mode,
 		Domain:       domain,
 		Region:       globalServerRegion,

cmd/admin-handlers_test.go

@@ -68,25 +68,13 @@ func prepareAdminXLTestBed(ctx context.Context) (*adminXLTestBed, error) {
 
 	globalEndpoints = mustGetZoneEndpoints(xlDirs...)
 
-	globalConfigSys = NewConfigSys()
+	newAllSubsystems()
 
-	globalIAMSys = NewIAMSys()
-	globalIAMSys.Init(ctx, objLayer)
-
-	buckets, err := objLayer.ListBuckets(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	globalPolicySys = NewPolicySys()
-	globalPolicySys.Init(buckets, objLayer)
-
-	globalNotificationSys = NewNotificationSys(globalEndpoints)
-	globalNotificationSys.Init(buckets, objLayer)
+	initAllSubsystems(ctx, objLayer)
 
 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()
-	registerAdminRouter(adminRouter, true, true, false)
+	registerAdminRouter(adminRouter, true, true)
 
 	return &adminXLTestBed{
 		xlDirs:   xlDirs,

cmd/admin-heal-ops.go

@@ -221,8 +221,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	// Check if new heal sequence to be started overlaps with any
 	// existing, running sequence
 	for k, hSeq := range ahs.healSeqMap {
-		if !hSeq.hasEnded() && (strings.HasPrefix(k, h.path) ||
-			strings.HasPrefix(h.path, k)) {
+		if !hSeq.hasEnded() && (HasPrefix(k, h.path) || HasPrefix(h.path, k)) {
 
 			errMsg = "The provided heal sequence path overlaps with an existing " +
 				fmt.Sprintf("heal path: %s", k)
@@ -297,8 +296,8 @@ func (ahs *allHealState) PopHealStatusJSON(path string,
 
 // healSource denotes single entity and heal option.
 type healSource struct {
-	path string          // entity path (format, buckets, objects) to heal
-	opts madmin.HealOpts // optional heal option overrides default setting
+	path string           // entity path (format, buckets, objects) to heal
+	opts *madmin.HealOpts // optional heal option overrides default setting
 }
 
 // healSequence - state for each heal sequence initiated on the
@@ -623,8 +622,8 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		opts:       h.settings,
 		responseCh: h.respCh,
 	}
-	if !source.opts.Equal(h.settings) {
-		task.opts = source.opts
+	if source.opts != nil {
+		task.opts = *source.opts
 	}
 	globalBackgroundHealRoutine.queueHealTask(task)
 

cmd/admin-quota-handlers.go

@@ -17,12 +17,13 @@
 package cmd
 
 import (
+	"encoding/json"
 	"io/ioutil"
 	"net/http"
-	"path"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/env"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 )
@@ -38,6 +39,9 @@ const (
 // to enforce total quota for the specified bucket.
 func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketQuotaConfig")
+
+	defer logger.AuditLog(w, r, "PutBucketQuotaConfig", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketQuotaAdminAction)
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
@@ -57,31 +61,22 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	defer r.Body.Close()
+
 	data, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
-	quotaCfg, err := parseBucketQuota(data)
-	if err != nil {
+
+	if _, err = parseBucketQuota(bucket, data); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketQuotaConfigFile)
-	if err = saveConfig(ctx, objectAPI, configFile, data); err != nil {
+
+	if err = globalBucketMetadataSys.Update(bucket, bucketQuotaConfigFile, data); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if quotaCfg.Quota > 0 {
-		globalBucketQuotaSys.Set(bucket, quotaCfg)
-		globalNotificationSys.PutBucketQuotaConfig(ctx, bucket, quotaCfg)
-
-	} else {
-		globalBucketQuotaSys.Remove(bucket)
-		globalNotificationSys.RemoveBucketQuotaConfig(ctx, bucket)
-
-	}
 
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
@@ -91,6 +86,8 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketQuotaConfig")
 
+	defer logger.AuditLog(w, r, "GetBucketQuotaConfig", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketQuotaAdminAction)
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
@@ -103,48 +100,19 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketQuotaConfigFile)
-	configData, err := readConfig(ctx, objectAPI, configFile)
+
+	config, err := globalBucketMetadataSys.GetQuotaConfig(bucket)
 	if err != nil {
-		if err != errConfigNotFound {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, BucketQuotaConfigNotFound{Bucket: bucket}), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+
+	configData, err := json.Marshal(config)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
 	// Write success response.
 	writeSuccessResponseJSON(w, configData)
 }
-
-// RemoveBucketQuotaConfigHandler - removes Bucket quota configuration.
-// ----------
-// Removes quota configuration on the specified bucket.
-func (a adminAPIHandlers) RemoveBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "RemoveBucketQuotaConfig")
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketQuotaAdminAction)
-	if objectAPI == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return
-	}
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketQuotaConfigFile)
-	if err := deleteConfig(ctx, objectAPI, configFile); err != nil {
-		if err != errConfigNotFound {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, BucketQuotaConfigNotFound{Bucket: bucket}), r.URL)
-		return
-	}
-	globalBucketQuotaSys.Remove(bucket)
-	globalNotificationSys.RemoveBucketQuotaConfig(ctx, bucket)
-	// Write success response.
-	writeSuccessNoContent(w)
-}

cmd/admin-router.go

@@ -20,6 +20,8 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -35,7 +37,7 @@ const (
 type adminAPIHandlers struct{}
 
 // registerAdminRouter - Add handler functions for each service REST API routes.
-func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps, enableBucketQuotaOps bool) {
+func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool) {
 
 	adminAPI := adminAPIHandlers{}
 	// Admin router
@@ -50,7 +52,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps, enab
 
 	for _, adminVersion := range adminVersions {
 		// Restart and stop MinIO service.
-		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(httpTraceAll(adminAPI.ServiceActionHandler)).Queries("action", "{action:.*}")
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(httpTraceAll(adminAPI.ServiceHandler)).Queries("action", "{action:.*}")
 		// Update MinIO servers.
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(httpTraceAll(adminAPI.ServerUpdateHandler)).Queries("updateURL", "{updateURL:.*}")
 
@@ -62,8 +64,6 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps, enab
 		// DataUsageInfo operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(httpTraceAll(adminAPI.DataUsageInfoHandler))
 
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountingusageinfo").HandlerFunc(httpTraceAll(adminAPI.AccountingUsageInfoHandler))
-
 		if globalIsDistXL || globalIsXL {
 			/// Heal operations
 
@@ -115,6 +115,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps, enab
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
 
 			// Add user IAM
+
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountusageinfo").HandlerFunc(httpTraceAll(adminAPI.AccountUsageInfoHandler))
+
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
 
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
@@ -169,16 +172,15 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps, enab
 		}
 
 		// Quota operations
-		if enableConfigOps && enableBucketQuotaOps {
-			// GetBucketQuotaConfig
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
-				httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
-			// PutBucketQuotaConfig
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
-				httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
-			// RemoveBucketQuotaConfig
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-bucket-quota").HandlerFunc(
-				httpTraceHdrs(adminAPI.RemoveBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
+		if globalIsXL || globalIsDistXL {
+			if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
+				// GetBucketQuotaConfig
+				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
+					httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
+				// PutBucketQuotaConfig
+				adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
+					httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
+			}
 		}
 
 		// -- Top APIs --

cmd/api-errors.go

@@ -100,6 +100,9 @@ const (
 	ErrNoSuchBucketLifecycle
 	ErrNoSuchLifecycleConfiguration
 	ErrNoSuchBucketSSEConfig
+	ErrNoSuchCORSConfiguration
+	ErrNoSuchWebsiteConfiguration
+	ErrReplicationConfigurationNotFoundError
 	ErrNoSuchKey
 	ErrNoSuchUpload
 	ErrNoSuchVersion
@@ -456,7 +459,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrInvalidAccessKeyID: {
 		Code:           "InvalidAccessKeyId",
-		Description:    "The access key ID you provided does not exist in our records.",
+		Description:    "The Access Key Id you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrInvalidBucketName: {
@@ -793,6 +796,21 @@ var errorCodes = errorCodeMap{
 		Description:    "Object Lock configuration cannot be enabled on existing buckets",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrNoSuchCORSConfiguration: {
+		Code:           "NoSuchCORSConfiguration",
+		Description:    "The CORS configuration does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchWebsiteConfiguration: {
+		Code:           "NoSuchWebsiteConfiguration",
+		Description:    "The specified bucket does not have a website configuration",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrReplicationConfigurationNotFoundError: {
+		Code:           "ReplicationConfigurationNotFoundError",
+		Description:    "The replication configuration was not found",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrNoSuchObjectLockConfiguration: {
 		Code:           "NoSuchObjectLockConfiguration",
 		Description:    "The specified object does not have a ObjectLock configuration",
@@ -1120,14 +1138,14 @@ var errorCodes = errorCodeMap{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrOperationTimedOut: {
-		Code:           "XMinioServerTimedOut",
-		Description:    "A timeout occurred while trying to lock a resource",
-		HTTPStatusCode: http.StatusRequestTimeout,
+		Code:           "RequestTimeout",
+		Description:    "A timeout occurred while trying to lock a resource, please reduce your request rate",
+		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
 	ErrOperationMaxedOut: {
-		Code:           "XMinioServerTimedOut",
-		Description:    "A timeout exceeded while waiting to proceed with the request",
-		HTTPStatusCode: http.StatusRequestTimeout,
+		Code:           "SlowDown",
+		Description:    "A timeout exceeded while waiting to proceed with the request, please reduce your request rate",
+		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
 	ErrUnsupportedMetadata: {
 		Code:           "InvalidArgument",
@@ -1649,7 +1667,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	if err == nil {
 		return ErrNone
 	}
-	// Verify if the underlying error is signature mismatch.
+
 	switch err {
 	case errInvalidArgument:
 		apiErr = ErrAdminInvalidArgument
@@ -1808,6 +1826,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrNoSuchLifecycleConfiguration
 	case BucketSSEConfigNotFound:
 		apiErr = ErrNoSuchBucketSSEConfig
+	case BucketTaggingNotFound:
+		apiErr = ErrBucketTaggingNotFound
+	case BucketObjectLockConfigNotFound:
+		apiErr = ErrObjectLockConfigurationNotFound
 	case BucketQuotaConfigNotFound:
 		apiErr = ErrAdminNoSuchQuotaConfiguration
 	case BucketQuotaExceeded:

cmd/api-headers.go

@@ -68,6 +68,13 @@ func encodeResponseJSON(response interface{}) []byte {
 	return bytesBuffer.Bytes()
 }
 
+// Write parts count
+func setPartsCountHeaders(w http.ResponseWriter, objInfo ObjectInfo) {
+	if strings.Contains(objInfo.ETag, "-") && len(objInfo.Parts) > 0 {
+		w.Header()[xhttp.AmzMpPartsCount] = []string{strconv.Itoa(len(objInfo.Parts))}
+	}
+}
+
 // Write object header
 func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSpec) (err error) {
 	// set common headers
@@ -82,10 +89,6 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
 	}
 
-	if strings.Contains(objInfo.ETag, "-") && len(objInfo.Parts) > 0 {
-		w.Header().Set(xhttp.AmzMpPartsCount, strconv.Itoa(len(objInfo.Parts)))
-	}
-
 	if objInfo.ContentType != "" {
 		w.Header().Set(xhttp.ContentType, objInfo.ContentType)
 	}
@@ -106,34 +109,34 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	// Set tag count if object has tags
 	tags, _ := url.ParseQuery(objInfo.UserTags)
 	tagCount := len(tags)
-	if tagCount != 0 {
-		w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount))
+	if tagCount > 0 {
+		w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(tagCount)}
 	}
 
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
-		if HasPrefix(k, ReservedMetadataPrefix) {
+		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 			// Do not need to send any internal metadata
 			// values to client.
 			continue
 		}
-		w.Header().Set(k, v)
+		var isSet bool
+		for _, userMetadataPrefix := range userMetadataKeyPrefixes {
+			if !strings.HasPrefix(k, userMetadataPrefix) {
+				continue
+			}
+			w.Header()[strings.ToLower(k)] = []string{v}
+			isSet = true
+			break
+		}
+		if !isSet {
+			w.Header().Set(k, v)
+		}
 	}
 
-	var totalObjectSize int64
-	switch {
-	case crypto.IsEncrypted(objInfo.UserDefined):
-		totalObjectSize, err = objInfo.DecryptedSize()
-		if err != nil {
-			return err
-		}
-	case objInfo.IsCompressed():
-		totalObjectSize = objInfo.GetActualSize()
-		if totalObjectSize < 0 {
-			return errInvalidDecompressedSize
-		}
-	default:
-		totalObjectSize = objInfo.Size
+	totalObjectSize, err := objInfo.GetActualSize()
+	if err != nil {
+		return err
 	}
 
 	// for providing ranged content

cmd/api-response.go

@@ -33,8 +33,10 @@ import (
 )
 
 const (
-	timeFormatAMZLong = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
-	maxObjectList     = 10000                      // Limit number of objects in a listObjectsResponse.
+	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
+	iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
+	maxObjectList     = 50000                      // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
+	maxDeleteList     = 10000                      // Limit number of objects deleted in a delete call.
 	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
 	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.
 )
@@ -400,7 +402,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	for _, bucket := range buckets {
 		var listbucket = Bucket{}
 		listbucket.Name = bucket.Name
-		listbucket.CreationDate = bucket.Created.UTC().Format(timeFormatAMZLong)
+		listbucket.CreationDate = bucket.Created.UTC().Format(iso8601TimeFormat)
 		listbuckets = append(listbuckets, listbucket)
 	}
 
@@ -424,7 +426,7 @@ func generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingTyp
 			continue
 		}
 		content.Key = s3EncodeName(object.Name, encodingType)
-		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
+		content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
 		if object.ETag != "" {
 			content.ETag = "\"" + object.ETag + "\""
 		}
@@ -440,9 +442,9 @@ func generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingTyp
 		content.IsLatest = true
 		versions = append(versions, content)
 	}
+
 	data.Name = bucket
 	data.Versions = versions
-
 	data.EncodingType = encodingType
 	data.Prefix = s3EncodeName(prefix, encodingType)
 	data.KeyMarker = s3EncodeName(marker, encodingType)
@@ -475,7 +477,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 			continue
 		}
 		content.Key = s3EncodeName(object.Name, encodingType)
-		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
+		content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
 		if object.ETag != "" {
 			content.ETag = "\"" + object.ETag + "\""
 		}
@@ -525,7 +527,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 			continue
 		}
 		content.Key = s3EncodeName(object.Name, encodingType)
-		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
+		content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
 		if object.ETag != "" {
 			content.ETag = "\"" + object.ETag + "\""
 		}
@@ -539,7 +541,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 		if metadata {
 			content.UserMetadata = make(StringMap)
 			for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) {
-				if HasPrefix(k, ReservedMetadataPrefix) {
+				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 					// Do not need to send any internal metadata
 					// values to client.
 					continue
@@ -574,7 +576,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
 	return CopyObjectResponse{
 		ETag:         "\"" + etag + "\"",
-		LastModified: lastModified.UTC().Format(timeFormatAMZLong),
+		LastModified: lastModified.UTC().Format(iso8601TimeFormat),
 	}
 }
 
@@ -582,7 +584,7 @@ func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectR
 func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse {
 	return CopyObjectPartResponse{
 		ETag:         "\"" + etag + "\"",
-		LastModified: lastModified.UTC().Format(timeFormatAMZLong),
+		LastModified: lastModified.UTC().Format(iso8601TimeFormat),
 	}
 }
 
@@ -627,7 +629,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
 		newPart.PartNumber = part.PartNumber
 		newPart.ETag = "\"" + part.ETag + "\""
 		newPart.Size = part.Size
-		newPart.LastModified = part.LastModified.UTC().Format(timeFormatAMZLong)
+		newPart.LastModified = part.LastModified.UTC().Format(iso8601TimeFormat)
 		listPartsResponse.Parts[index] = newPart
 	}
 	return listPartsResponse
@@ -657,7 +659,7 @@ func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMult
 		newUpload := Upload{}
 		newUpload.UploadID = upload.UploadID
 		newUpload.Key = s3EncodeName(upload.Object, encodingType)
-		newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZLong)
+		newUpload.Initiated = upload.Initiated.UTC().Format(iso8601TimeFormat)
 		listMultipartUploadsResponse.Uploads[index] = newUpload
 	}
 	return listMultipartUploadsResponse
@@ -767,17 +769,6 @@ func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIE
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }
 
-// writeVersionMismatchResponse - writes custom error responses for version mismatches.
-func writeVersionMismatchResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL, isJSON bool) {
-	if isJSON {
-		// Generate error response.
-		errorResponse := getAPIErrorResponse(ctx, err, reqURL.String(), w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
-		writeResponse(w, err.HTTPStatusCode, encodeResponseJSON(errorResponse), mimeJSON)
-	} else {
-		writeResponse(w, err.HTTPStatusCode, []byte(err.Description), mimeNone)
-	}
-}
-
 // writeCustomErrorResponseJSON - similar to writeErrorResponseJSON,
 // but accepts the error message directly (this allows messages to be
 // dynamically generated.)

cmd/api-router.go

@@ -285,6 +285,11 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
 		maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler))))
 
+	// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
+	// than failing with UnknownAPIRequest we simply handle it for now.
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
+		maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler))))
+
 	// If none of the routes match add default error handler routes
 	apiRouter.NotFoundHandler = http.HandlerFunc(collectAPIStats("notfound", httpTraceAll(errorResponseHandler)))
 	apiRouter.MethodNotAllowedHandler = http.HandlerFunc(collectAPIStats("methodnotallowed", httpTraceAll(errorResponseHandler)))

cmd/auth-handler.go

@@ -423,7 +423,7 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 	if err != nil {
 		return toAPIErrorCode(ctx, err)
 	}
-	r.Body = ioutil.NopCloser(reader)
+	r.Body = reader
 	return ErrNone
 }
 

cmd/background-heal-ops.go

@@ -114,8 +114,9 @@ func startBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 	go globalBackgroundHealRoutine.run(ctx, objAPI)
 
 	// Launch the background healer sequence to track
-	// background healing operations
-	info := objAPI.StorageInfo(ctx, false)
+	// background healing operations, ignore errors
+	// errors are handled into offline disks already.
+	info, _ := objAPI.StorageInfo(ctx, false)
 	numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum()
 	nh := newBgHealSequence(numDisks)
 	globalBackgroundHealState.LaunchNewHealSequence(nh)

cmd/background-newdisks-heal-ops.go

@@ -115,7 +115,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 			// Heal all erasure sets that need
 			for i, erasureSetToHeal := range erasureSetInZoneToHeal {
 				for _, setIndex := range erasureSetToHeal {
-					err := healErasureSet(ctx, setIndex, z.zones[i].sets[setIndex])
+					err := healErasureSet(ctx, setIndex, z.zones[i].sets[setIndex], z.zones[i].drivesPerSet)
 					if err != nil {
 						logger.LogIf(ctx, err)
 					}

cmd/benchmark-utils_test.go

@@ -35,7 +35,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err = obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -76,7 +76,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	object := getRandomObjectName()
 
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err = obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -181,7 +181,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -278,7 +278,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -322,7 +322,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
 	if err != nil {
 		b.Fatal(err)
 	}

cmd/bitrot_test.go

@@ -34,7 +34,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	volume := "testvol"
 	filePath := "testfile"
 
-	disk, err := newPosix(tmpDir)
+	disk, err := newPosix(tmpDir, "")
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/bucket-encryption-handlers.go

@@ -23,9 +23,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/policy"
 )
 
@@ -47,15 +45,14 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 		return
 	}
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-
-	// PutBucketEncyrption API requires Content-Md5
-	if _, ok := r.Header[xhttp.ContentMD5]; !ok {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
+	if !objAPI.IsEncryptionSupported() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketEncryptionAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
@@ -85,17 +82,17 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// Store the bucket encryption configuration in the object layer
-	if err = objAPI.SetBucketSSEConfig(ctx, bucket, encConfig); err != nil {
+	configData, err := xml.Marshal(encConfig)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	// Update the in-memory bucket encryption cache
-	globalBucketSSEConfigSys.Set(bucket, encConfig)
-
-	// Update peer MinIO servers of the updated bucket encryption config
-	globalNotificationSys.SetBucketSSEConfig(ctx, bucket, encConfig)
+	// Store the bucket encryption configuration in the object layer
+	if err = globalBucketMetadataSys.Update(bucket, bucketSSEConfig, configData); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
 
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -128,21 +125,20 @@ func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// Fetch bucket encryption configuration from object layer
-	var encConfig *bucketsse.BucketSSEConfig
-	if encConfig, err = objAPI.GetBucketSSEConfig(ctx, bucket); err != nil {
+	config, err := globalBucketMetadataSys.GetSSEConfig(bucket)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	var encConfigData []byte
-	if encConfigData, err = xml.Marshal(encConfig); err != nil {
+	configData, err := xml.Marshal(config)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Write bucket encryption configuration to client
-	writeSuccessResponseXML(w, encConfigData)
+	writeSuccessResponseXML(w, configData)
 }
 
 // DeleteBucketEncryptionHandler - Removes bucket encryption configuration
@@ -173,16 +169,10 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 	}
 
 	// Delete bucket encryption config from object layer
-	if err = objAPI.DeleteBucketSSEConfig(ctx, bucket); err != nil {
+	if err = globalBucketMetadataSys.Update(bucket, bucketSSEConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	// Remove entry from the in-memory bucket encryption cache
-	globalBucketSSEConfigSys.Remove(bucket)
-
-	// Update peer MinIO servers of the updated bucket encryption config
-	globalNotificationSys.RemoveBucketSSEConfig(ctx, bucket)
-
 	writeSuccessNoContent(w)
 }

cmd/bucket-encryption.go

@@ -17,143 +17,32 @@
 package cmd
 
 import (
-	"bytes"
-	"context"
-	"encoding/xml"
 	"errors"
 	"io"
-	"path"
-	"sync"
 
 	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 )
 
 // BucketSSEConfigSys - in-memory cache of bucket encryption config
-type BucketSSEConfigSys struct {
-	sync.RWMutex
-	bucketSSEConfigMap map[string]*bucketsse.BucketSSEConfig
-}
+type BucketSSEConfigSys struct{}
 
 // NewBucketSSEConfigSys - Creates an empty in-memory bucket encryption configuration cache
 func NewBucketSSEConfigSys() *BucketSSEConfigSys {
-	return &BucketSSEConfigSys{
-		bucketSSEConfigMap: make(map[string]*bucketsse.BucketSSEConfig),
-	}
-}
-
-// load - Loads the bucket encryption configuration for the given list of buckets
-func (sys *BucketSSEConfigSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
-	for _, bucket := range buckets {
-		config, err := objAPI.GetBucketSSEConfig(GlobalContext, bucket.Name)
-		if err != nil {
-			if _, ok := err.(BucketSSEConfigNotFound); ok {
-				continue
-			}
-			// Quorum errors should be returned.
-			return err
-		}
-		sys.Set(bucket.Name, config)
-	}
-
-	return nil
-}
-
-// Init - Initializes in-memory bucket encryption config cache for the given list of buckets
-func (sys *BucketSSEConfigSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	// We don't cache bucket encryption config in gateway mode, nothing to do.
-	if globalIsGateway {
-		return nil
-	}
-
-	// Load bucket encryption config cache once during boot.
-	return sys.load(buckets, objAPI)
+	return &BucketSSEConfigSys{}
 }
 
 // Get - gets bucket encryption config for the given bucket.
-func (sys *BucketSSEConfigSys) Get(bucket string) (config *bucketsse.BucketSSEConfig, ok bool) {
-	// We don't cache bucket encryption config in gateway mode.
+func (sys *BucketSSEConfigSys) Get(bucket string) (*bucketsse.BucketSSEConfig, error) {
 	if globalIsGateway {
 		objAPI := newObjectLayerWithoutSafeModeFn()
 		if objAPI == nil {
-			return
+			return nil, errServerNotInitialized
 		}
 
-		cfg, err := objAPI.GetBucketSSEConfig(GlobalContext, bucket)
-		if err != nil {
-			return
-		}
-		return cfg, true
+		return nil, BucketSSEConfigNotFound{Bucket: bucket}
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
-	config, ok = sys.bucketSSEConfigMap[bucket]
-	return
-}
-
-// Set - sets bucket encryption config to given bucket name.
-func (sys *BucketSSEConfigSys) Set(bucket string, config *bucketsse.BucketSSEConfig) {
-	// We don't cache bucket encryption config in gateway mode.
-	if globalIsGateway {
-		return
-	}
-
-	sys.Lock()
-	defer sys.Unlock()
-	sys.bucketSSEConfigMap[bucket] = config
-}
-
-// Remove - removes bucket encryption config for given bucket.
-func (sys *BucketSSEConfigSys) Remove(bucket string) {
-	sys.Lock()
-	defer sys.Unlock()
-
-	delete(sys.bucketSSEConfigMap, bucket)
-}
-
-// saveBucketSSEConfig - save bucket encryption config for given bucket.
-func saveBucketSSEConfig(ctx context.Context, objAPI ObjectLayer, bucket string, config *bucketsse.BucketSSEConfig) error {
-	data, err := xml.Marshal(config)
-	if err != nil {
-		return err
-	}
-
-	// Path to store bucket encryption config for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
-	return saveConfig(ctx, objAPI, configFile, data)
-}
-
-// getBucketSSEConfig - get bucket encryption config for given bucket.
-func getBucketSSEConfig(objAPI ObjectLayer, bucket string) (*bucketsse.BucketSSEConfig, error) {
-	// Path to bucket-encryption.xml for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
-	configData, err := readConfig(GlobalContext, objAPI, configFile)
-	if err != nil {
-		if err == errConfigNotFound {
-			err = BucketSSEConfigNotFound{Bucket: bucket}
-		}
-		return nil, err
-	}
-
-	return bucketsse.ParseBucketSSEConfig(bytes.NewReader(configData))
-}
-
-// removeBucketSSEConfig - removes bucket encryption config for given bucket.
-func removeBucketSSEConfig(ctx context.Context, objAPI ObjectLayer, bucket string) error {
-	// Path to bucket-encryption.xml for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
-
-	if err := deleteConfig(ctx, objAPI, configFile); err != nil {
-		if err == errConfigNotFound {
-			return BucketSSEConfigNotFound{Bucket: bucket}
-		}
-		return err
-	}
-	return nil
+	return globalBucketMetadataSys.GetSSEConfig(bucket)
 }
 
 // validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO.

cmd/bucket-handlers.go

@@ -47,7 +47,7 @@ import (
 const (
 	getBucketVersioningResponse = `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`
 	objectLockConfig            = "object-lock.xml"
-	bucketTaggingConfigFile     = "tagging.xml"
+	bucketTaggingConfig         = "tagging.xml"
 )
 
 // Check if there are buckets on server without corresponding entry in etcd backend and
@@ -292,16 +292,6 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-
-		for i := range bucketsInfo {
-			meta, err := loadBucketMetadata(ctx, objectAPI, bucketsInfo[i].Name)
-			if err == nil {
-				bucketsInfo[i].Created = meta.Created
-			}
-			if err != errMetaDataConverted {
-				logger.LogIf(ctx, err)
-			}
-		}
 	}
 
 	if s3Error == ErrAccessDenied {
@@ -409,9 +399,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			continue
 		}
 
-		if _, ok := globalBucketObjectLockConfig.Get(bucket); ok {
-			if err := enforceRetentionBypassForDelete(ctx, r, bucket, object.ObjectName, getObjectInfoFn); err != ErrNone {
-				dErrs[index] = err
+		if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
+			if apiErr := enforceRetentionBypassForDelete(ctx, r, bucket, object.ObjectName, getObjectInfoFn); apiErr != ErrNone {
+				dErrs[index] = apiErr
 				continue
 			}
 		}
@@ -537,14 +527,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		if err != nil {
 			if err == dns.ErrNoEntriesFound {
 				// Proceed to creating a bucket.
-				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
-					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-					return
-				}
-
-				meta := newBucketMetadata(bucket)
-				meta.LockEnabled = objectLockEnabled
-				if err := meta.save(ctx, objectAPI); err != nil {
+				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location, objectLockEnabled); err != nil {
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 					return
 				}
@@ -555,6 +538,9 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 					return
 				}
 
+				// Load updated bucket metadata into memory.
+				globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
+
 				// Make sure to add Location information here only for bucket
 				w.Header().Set(xhttp.Location,
 					getObjectLocation(r, globalDomainNames, bucket, ""))
@@ -579,25 +565,14 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Proceed to creating a bucket.
-	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
+	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location, objectLockEnabled)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	if !globalIsGateway {
-		meta := newBucketMetadata(bucket)
-		meta.LockEnabled = objectLockEnabled
-		if err := meta.save(ctx, objectAPI); err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-		if objectLockEnabled {
-			ret := &objectlock.Retention{}
-			globalBucketObjectLockConfig.Set(bucket, ret)
-			globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, ret)
-		}
-	}
+	// Load updated bucket metadata into memory.
+	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 
 	// Make sure to add Location information here only for bucket
 	w.Header().Set(xhttp.Location, path.Clean(r.URL.Path)) // Clean any trailing slashes.
@@ -771,11 +746,13 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	var objectEncryptionKey crypto.ObjectKey
 
 	// Check if bucket encryption is enabled
-	_, encEnabled := globalBucketSSEConfigSys.Get(bucket)
-	// This request header needs to be set prior to setting ObjectOptions
-	if (globalAutoEncryption || encEnabled) && !crypto.SSEC.IsRequested(r.Header) {
-		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
+	if _, err = globalBucketSSEConfigSys.Get(bucket); err == nil || globalAutoEncryption {
+		// This request header needs to be set prior to setting ObjectOptions
+		if !crypto.SSEC.IsRequested(r.Header) {
+			r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
+		}
 	}
+
 	// get gateway encryption options
 	var opts ObjectOptions
 	opts, err = putOpts(ctx, r, bucket, object, metadata)
@@ -933,9 +910,11 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		}
 	}
 
-	if _, ok := globalBucketObjectLockConfig.Get(bucket); ok && forceDelete {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
-		return
+	if forceDelete {
+		if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
+			return
+		}
 	}
 
 	deleteBucket := objectAPI.DeleteBucket
@@ -949,16 +928,13 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	if globalDNSConfig != nil {
 		if err := globalDNSConfig.Delete(bucket); err != nil {
 			// Deleting DNS entry failed, attempt to create the bucket again.
-			objectAPI.MakeBucketWithLocation(ctx, bucket, "")
+			objectAPI.MakeBucketWithLocation(ctx, bucket, "", false)
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
-	// Delete metadata, only log errors.
-	logger.LogIf(ctx, newBucketMetadata(bucket).delete(ctx, objectAPI))
-
-	globalNotificationSys.DeleteBucket(ctx, bucket)
+	globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
 
 	// Write success response.
 	writeSuccessNoContent(w)
@@ -1041,6 +1017,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	config, err := objectlock.ParseObjectLockConfig(r.Body)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
@@ -1049,34 +1026,22 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		return
 	}
 
-	meta, err := loadBucketMetadata(ctx, objectAPI, bucket)
+	configData, err := xml.Marshal(config)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if !meta.LockEnabled {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectLockConfigurationNotAllowed), r.URL, guessIsBrowserReq(r))
-		return
-	}
 
-	data, err := xml.Marshal(config)
-	if err != nil {
+	// Deny object locking configuration settings on existing buckets without object lock enabled.
+	if _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	configFile := path.Join(bucketConfigPrefix, bucket, objectLockConfig)
-	if err = saveConfig(ctx, objectAPI, configFile, data); err != nil {
+
+	if err = globalBucketMetadataSys.Update(bucket, objectLockConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if config.Rule != nil {
-		retention := config.ToRetention()
-		globalBucketObjectLockConfig.Set(bucket, retention)
-		globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, retention)
-	} else {
-		globalBucketObjectLockConfig.Set(bucket, &objectlock.Retention{})
-		globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, &objectlock.Retention{})
-	}
 
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
@@ -1100,34 +1065,23 @@ func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWri
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	// check if user has permissions to perform this operation
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketObjectLockConfigurationAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	meta, err := loadBucketMetadata(ctx, objectAPI, bucket)
-	if err != nil && err != errMetaDataConverted {
+	config, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if !meta.LockEnabled {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectLockConfigurationNotAllowed), r.URL, guessIsBrowserReq(r))
-		return
-	}
 
-	configFile := path.Join(bucketConfigPrefix, bucket, objectLockConfig)
-	configData, err := readConfig(ctx, objectAPI, configFile)
+	configData, err := xml.Marshal(config)
 	if err != nil {
-		if err != errConfigNotFound {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-
-		if configData, err = xml.Marshal(objectlock.NewObjectLockConfig()); err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
 	}
 
 	// Write success response.
@@ -1154,6 +1108,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	tags, err := tags.ParseBucketXML(io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
@@ -1161,13 +1116,14 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 		writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
 		return
 	}
-	data, err := xml.Marshal(tags)
+
+	configData, err := xml.Marshal(tags)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketTaggingConfigFile)
-	if err = saveConfig(ctx, objectAPI, configFile, data); err != nil {
+
+	if err = globalBucketMetadataSys.Update(bucket, bucketTaggingConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -1191,21 +1147,22 @@ func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *h
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	// check if user has permissions to perform this operation
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketTaggingAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketTaggingConfigFile)
-	configData, err := readConfig(ctx, objectAPI, configFile)
+
+	config, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
 	if err != nil {
-		var aerr APIError
-		if err == errConfigNotFound {
-			aerr = errorCodes.ToAPIErr(ErrBucketTaggingNotFound)
-		} else {
-			aerr = toAPIError(ctx, err)
-		}
-		writeErrorResponse(ctx, w, aerr, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	configData, err := xml.Marshal(config)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -1234,8 +1191,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketTaggingConfigFile)
-	if err := deleteConfig(ctx, objectAPI, configFile); err != nil && err != errConfigNotFound {
+	if err := globalBucketMetadataSys.Update(bucket, bucketTaggingConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}

cmd/bucket-handlers_test.go

@@ -115,7 +115,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 			errorResponse: APIErrorResponse{
 				Resource: SlashSeparator + bucketName + SlashSeparator,
 				Code:     "InvalidAccessKeyId",
-				Message:  "The access key ID you provided does not exist in our records.",
+				Message:  "The Access Key Id you provided does not exist in our records.",
 			},
 			shouldPass: false,
 		},

cmd/bucket-lifecycle-handlers.go

@@ -72,13 +72,22 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err = objAPI.SetBucketLifecycle(ctx, bucket, bucketLifecycle); err != nil {
+	// Validate the received bucket policy document
+	if err = bucketLifecycle.Validate(); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	globalLifecycleSys.Set(bucket, bucketLifecycle)
-	globalNotificationSys.SetBucketLifecycle(ctx, bucket, bucketLifecycle)
+	configData, err := xml.Marshal(bucketLifecycle)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if err = globalBucketMetadataSys.Update(bucket, bucketLifecycleConfig, configData); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
 
 	// Success.
 	writeSuccessResponseHeadersOnly(w)
@@ -110,21 +119,20 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// Read bucket access lifecycle.
-	bucketLifecycle, err := objAPI.GetBucketLifecycle(ctx, bucket)
+	config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	lifecycleData, err := xml.Marshal(bucketLifecycle)
+	configData, err := xml.Marshal(config)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Write lifecycle configuration to client.
-	writeSuccessResponseXML(w, lifecycleData)
+	writeSuccessResponseXML(w, configData)
 }
 
 // DeleteBucketLifecycleHandler - This HTTP handler removes bucket lifecycle configuration.
@@ -153,14 +161,11 @@ func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter,
 		return
 	}
 
-	if err := objAPI.DeleteBucketLifecycle(ctx, bucket); err != nil {
+	if err := globalBucketMetadataSys.Update(bucket, bucketLifecycleConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	globalLifecycleSys.Remove(bucket)
-	globalNotificationSys.RemoveBucketLifecycle(ctx, bucket)
-
 	// Success.
 	writeSuccessNoContent(w)
 }

cmd/bucket-lifecycle-handlers_test.go

@@ -72,7 +72,7 @@ func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType,
 			errorResponse: APIErrorResponse{
 				Resource: SlashSeparator + bucketName + SlashSeparator,
 				Code:     "InvalidAccessKeyId",
-				Message:  "The access key ID you provided does not exist in our records.",
+				Message:  "The Access Key Id you provided does not exist in our records.",
 			},
 			shouldPass: false,
 		},
@@ -102,7 +102,7 @@ func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType,
 			errorResponse: APIErrorResponse{
 				Resource: SlashSeparator + bucketName + SlashSeparator,
 				Code:     "InvalidAccessKeyId",
-				Message:  "The access key ID you provided does not exist in our records.",
+				Message:  "The Access Key Id you provided does not exist in our records.",
 			},
 			shouldPass: false,
 		},
@@ -132,7 +132,7 @@ func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType,
 			errorResponse: APIErrorResponse{
 				Resource: SlashSeparator + bucketName + SlashSeparator,
 				Code:     "InvalidAccessKeyId",
-				Message:  "The access key ID you provided does not exist in our records.",
+				Message:  "The Access Key Id you provided does not exist in our records.",
 			},
 			shouldPass: false,
 		},

cmd/bucket-lifecycle.go

@@ -17,12 +17,6 @@
 package cmd
 
 import (
-	"bytes"
-	"context"
-	"encoding/xml"
-	"path"
-	"sync"
-
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 )
 
@@ -33,131 +27,23 @@ const (
 )
 
 // LifecycleSys - Bucket lifecycle subsystem.
-type LifecycleSys struct {
-	sync.RWMutex
-	bucketLifecycleMap map[string]*lifecycle.Lifecycle
-}
-
-// Set - sets lifecycle config to given bucket name.
-func (sys *LifecycleSys) Set(bucketName string, lifecycle *lifecycle.Lifecycle) {
-	if globalIsGateway {
-		// no-op
-		return
-	}
-
-	sys.Lock()
-	defer sys.Unlock()
-
-	sys.bucketLifecycleMap[bucketName] = lifecycle
-}
+type LifecycleSys struct{}
 
 // Get - gets lifecycle config associated to a given bucket name.
-func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, ok bool) {
+func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) {
 	if globalIsGateway {
-		// When gateway is enabled, no cached value
-		// is used to validate life cycle policies.
 		objAPI := newObjectLayerWithoutSafeModeFn()
 		if objAPI == nil {
-			return
+			return nil, errServerNotInitialized
 		}
 
-		l, err := objAPI.GetBucketLifecycle(GlobalContext, bucketName)
-		if err != nil {
-			return
-		}
-		return l, true
+		return nil, BucketLifecycleNotFound{Bucket: bucketName}
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
-	lc, ok = sys.bucketLifecycleMap[bucketName]
-	return
-}
-
-func saveLifecycleConfig(ctx context.Context, objAPI ObjectLayer, bucketName string, bucketLifecycle *lifecycle.Lifecycle) error {
-	data, err := xml.Marshal(bucketLifecycle)
-	if err != nil {
-		return err
-	}
-
-	// Construct path to lifecycle.xml for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucketName, bucketLifecycleConfig)
-	return saveConfig(ctx, objAPI, configFile, data)
-}
-
-// getLifecycleConfig - get lifecycle config for given bucket name.
-func getLifecycleConfig(objAPI ObjectLayer, bucketName string) (*lifecycle.Lifecycle, error) {
-	// Construct path to lifecycle.xml for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucketName, bucketLifecycleConfig)
-	configData, err := readConfig(GlobalContext, objAPI, configFile)
-	if err != nil {
-		if err == errConfigNotFound {
-			err = BucketLifecycleNotFound{Bucket: bucketName}
-		}
-		return nil, err
-	}
-
-	return lifecycle.ParseLifecycleConfig(bytes.NewReader(configData))
-}
-
-func removeLifecycleConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) error {
-	// Construct path to lifecycle.xml for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucketName, bucketLifecycleConfig)
-
-	if err := deleteConfig(ctx, objAPI, configFile); err != nil {
-		if err == errConfigNotFound {
-			return BucketLifecycleNotFound{Bucket: bucketName}
-		}
-		return err
-	}
-	return nil
+	return globalBucketMetadataSys.GetLifecycleConfig(bucketName)
 }
 
 // NewLifecycleSys - creates new lifecycle system.
 func NewLifecycleSys() *LifecycleSys {
-	return &LifecycleSys{
-		bucketLifecycleMap: make(map[string]*lifecycle.Lifecycle),
-	}
-}
-
-// Init - initializes lifecycle system from lifecycle.xml of all buckets.
-func (sys *LifecycleSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	// In gateway mode, we always fetch the bucket lifecycle configuration from the gateway backend.
-	// So, this is a no-op for gateway servers.
-	if globalIsGateway {
-		return nil
-	}
-
-	// Load LifecycleSys once during boot.
-	return sys.load(buckets, objAPI)
-}
-
-// Loads lifecycle policies for all buckets into LifecycleSys.
-func (sys *LifecycleSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
-	for _, bucket := range buckets {
-		config, err := getLifecycleConfig(objAPI, bucket.Name)
-		if err != nil {
-			if _, ok := err.(BucketLifecycleNotFound); ok {
-				continue
-			}
-			// Quorum errors should be returned.
-			return err
-		}
-
-		sys.Set(bucket.Name, config)
-	}
-
-	return nil
-}
-
-// Remove - removes lifecycle config for given bucket name.
-func (sys *LifecycleSys) Remove(bucketName string) {
-	sys.Lock()
-	defer sys.Unlock()
-
-	delete(sys.bucketLifecycleMap, bucketName)
+	return &LifecycleSys{}
 }

cmd/bucket-listobjects-handlers.go

@@ -99,24 +99,13 @@ func (api objectAPIHandlers) ListBucketObjectVersionsHandler(w http.ResponseWrit
 	}
 
 	for i := range listObjectsInfo.Objects {
-		var actualSize int64
-		if listObjectsInfo.Objects[i].IsCompressed() {
-			// Read the decompressed size from the meta.json.
-			actualSize = listObjectsInfo.Objects[i].GetActualSize()
-			if actualSize < 0 {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize),
-					r.URL, guessIsBrowserReq(r))
-				return
-			}
-			// Set the info.Size to the actualSize.
-			listObjectsInfo.Objects[i].Size = actualSize
-		} else if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
+		if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
 			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
-			listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
-			if err != nil {
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-				return
-			}
+		}
+		listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].GetActualSize()
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
 		}
 	}
 
@@ -181,23 +170,13 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 	}
 
 	for i := range listObjectsV2Info.Objects {
-		var actualSize int64
-		if listObjectsV2Info.Objects[i].IsCompressed() {
-			// Read the decompressed size from the meta.json.
-			actualSize = listObjectsV2Info.Objects[i].GetActualSize()
-			if actualSize < 0 {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
-				return
-			}
-			// Set the info.Size to the actualSize.
-			listObjectsV2Info.Objects[i].Size = actualSize
-		} else if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
+		if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
 			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
-			listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
-			if err != nil {
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-				return
-			}
+		}
+		listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].GetActualSize()
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
 		}
 	}
 
@@ -265,23 +244,13 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	}
 
 	for i := range listObjectsV2Info.Objects {
-		var actualSize int64
-		if listObjectsV2Info.Objects[i].IsCompressed() {
-			// Read the decompressed size from the meta.json.
-			actualSize = listObjectsV2Info.Objects[i].GetActualSize()
-			if actualSize < 0 {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
-				return
-			}
-			// Set the info.Size to the actualSize.
-			listObjectsV2Info.Objects[i].Size = actualSize
-		} else if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
+		if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
 			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
-			listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
-			if err != nil {
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-				return
-			}
+		}
+		listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].GetActualSize()
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
 		}
 	}
 
@@ -344,25 +313,16 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 
 	for i := range listObjectsInfo.Objects {
-		var actualSize int64
-		if listObjectsInfo.Objects[i].IsCompressed() {
-			// Read the decompressed size from the meta.json.
-			actualSize = listObjectsInfo.Objects[i].GetActualSize()
-			if actualSize < 0 {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
-				return
-			}
-			// Set the info.Size to the actualSize.
-			listObjectsInfo.Objects[i].Size = actualSize
-		} else if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
+		if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
 			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
-			listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
-			if err != nil {
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-				return
-			}
+		}
+		listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].GetActualSize()
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
 		}
 	}
+
 	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo)
 
 	// Write success response.

cmd/bucket-meta.go

@@ -1,167 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"path"
-	"time"
-
-	"github.com/minio/minio/cmd/logger"
-)
-
-const (
-	legacyBucketObjectLockEnabledConfigFile = "object-lock-enabled.json"
-	legacyBucketObjectLockEnabledConfig     = `{"x-amz-bucket-object-lock-enabled":true}`
-
-	bucketMetadataFile    = ".metadata.bin"
-	bucketMetadataFormat  = 1
-	bucketMetadataVersion = 1
-)
-
-//go:generate msgp -file $GOFILE -unexported
-
-// bucketMetadata contains bucket metadata.
-// When adding/removing fields, regenerate the marshal code using the go generate above.
-// Only changing meaning of fields requires a version bump.
-// bucketMetadataFormat refers to the format.
-// bucketMetadataVersion can be used to track a rolling upgrade of a field.
-type bucketMetadata struct {
-	Name        string
-	Created     time.Time
-	LockEnabled bool
-}
-
-// newBucketMetadata creates bucketMetadata with the supplied name and Created to Now.
-func newBucketMetadata(name string) bucketMetadata {
-	return bucketMetadata{
-		Name:    name,
-		Created: UTCNow(),
-	}
-}
-
-// loadBucketMeta loads the metadata of bucket by name from ObjectLayer o.
-// If an error is returned the returned metadata will be default initialized.
-func loadBucketMeta(ctx context.Context, o ObjectLayer, name string) (bucketMetadata, error) {
-	b := newBucketMetadata(name)
-	configFile := path.Join(bucketConfigPrefix, name, bucketMetadataFile)
-	data, err := readConfig(ctx, o, configFile)
-	if err != nil {
-		return b, err
-	}
-	if len(data) <= 4 {
-		return b, fmt.Errorf("loadBucketMetadata: no data")
-	}
-	// Read header
-	switch binary.LittleEndian.Uint16(data[0:2]) {
-	case bucketMetadataFormat:
-	default:
-		return b, fmt.Errorf("loadBucketMetadata: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
-	}
-	switch binary.LittleEndian.Uint16(data[2:4]) {
-	case bucketMetadataVersion:
-	default:
-		return b, fmt.Errorf("loadBucketMetadata: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
-	}
-
-	// OK, parse data.
-	_, err = b.UnmarshalMsg(data[4:])
-	return b, err
-}
-
-var errMetaDataConverted = errors.New("metadata converted")
-
-// loadBucketMetadata loads and migrates to bucket metadata.
-func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (bucketMetadata, error) {
-	meta, err := loadBucketMeta(ctx, objectAPI, bucket)
-	if err == nil {
-		return meta, nil
-	}
-
-	if err != errConfigNotFound {
-		return meta, err
-	}
-
-	// Control here means old bucket without bucket metadata. Hence we migrate existing settings.
-	if err = meta.convertLegacyLockconfig(ctx, objectAPI); err != nil {
-		return meta, err
-	}
-
-	return meta, errMetaDataConverted
-}
-
-func (b *bucketMetadata) convertLegacyLockconfig(ctx context.Context, objectAPI ObjectLayer) error {
-	configFile := path.Join(bucketConfigPrefix, b.Name, legacyBucketObjectLockEnabledConfigFile)
-
-	save := func() error {
-		if err := b.save(ctx, objectAPI); err != nil {
-			return err
-		}
-
-		if err := deleteConfig(ctx, objectAPI, configFile); err != nil && !errors.Is(err, errConfigNotFound) {
-			logger.LogIf(ctx, err)
-		}
-		return nil
-	}
-
-	configData, err := readConfig(ctx, objectAPI, configFile)
-	if err != nil {
-		if !errors.Is(err, errConfigNotFound) {
-			return err
-		}
-
-		return save()
-	}
-
-	if string(configData) != legacyBucketObjectLockEnabledConfig {
-		return fmt.Errorf("content mismatch in config file %v", configFile)
-	}
-
-	b.LockEnabled = true
-	return save()
-}
-
-// save config to supplied ObjectLayer o.
-func (b *bucketMetadata) save(ctx context.Context, o ObjectLayer) error {
-	data := make([]byte, 4, b.Msgsize()+4)
-	// Put header
-	binary.LittleEndian.PutUint16(data[0:2], bucketMetadataFormat)
-	binary.LittleEndian.PutUint16(data[2:4], bucketMetadataVersion)
-
-	// Add data
-	data, err := b.MarshalMsg(data)
-	if err != nil {
-		return err
-	}
-	configFile := path.Join(bucketConfigPrefix, b.Name, bucketMetadataFile)
-	return saveConfig(ctx, o, configFile, data)
-}
-
-// delete the config metadata.
-// If config does not exist no error is returned.
-func (b bucketMetadata) delete(ctx context.Context, o ObjectLayer) error {
-	configFile := path.Join(bucketConfigPrefix, b.Name, bucketMetadataFile)
-	err := deleteConfig(ctx, o, configFile)
-	if err == errConfigNotFound {
-		// We don't care
-		err = nil
-	}
-	return err
-}

cmd/bucket-meta_gen.go

@@ -1,160 +0,0 @@
-package cmd
-
-// Code generated by github.com/tinylib/msgp DO NOT EDIT.
-
-import (
-	"github.com/tinylib/msgp/msgp"
-)
-
-// DecodeMsg implements msgp.Decodable
-func (z *bucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Name":
-			z.Name, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "Name")
-				return
-			}
-		case "Created":
-			z.Created, err = dc.ReadTime()
-			if err != nil {
-				err = msgp.WrapError(err, "Created")
-				return
-			}
-		case "LockEnabled":
-			z.LockEnabled, err = dc.ReadBool()
-			if err != nil {
-				err = msgp.WrapError(err, "LockEnabled")
-				return
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z bucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 3
-	// write "Name"
-	err = en.Append(0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Name)
-	if err != nil {
-		err = msgp.WrapError(err, "Name")
-		return
-	}
-	// write "Created"
-	err = en.Append(0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
-	if err != nil {
-		return
-	}
-	err = en.WriteTime(z.Created)
-	if err != nil {
-		err = msgp.WrapError(err, "Created")
-		return
-	}
-	// write "LockEnabled"
-	err = en.Append(0xab, 0x4c, 0x6f, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64)
-	if err != nil {
-		return
-	}
-	err = en.WriteBool(z.LockEnabled)
-	if err != nil {
-		err = msgp.WrapError(err, "LockEnabled")
-		return
-	}
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z bucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	// map header, size 3
-	// string "Name"
-	o = append(o, 0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
-	o = msgp.AppendString(o, z.Name)
-	// string "Created"
-	o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
-	o = msgp.AppendTime(o, z.Created)
-	// string "LockEnabled"
-	o = append(o, 0xab, 0x4c, 0x6f, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64)
-	o = msgp.AppendBool(o, z.LockEnabled)
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *bucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Name":
-			z.Name, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Name")
-				return
-			}
-		case "Created":
-			z.Created, bts, err = msgp.ReadTimeBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Created")
-				return
-			}
-		case "LockEnabled":
-			z.LockEnabled, bts, err = msgp.ReadBoolBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "LockEnabled")
-				return
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z bucketMetadata) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize
-	return
-}

cmd/bucket-metadata-sys.go

@@ -0,0 +1,399 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/minio/minio-go/v6/pkg/tags"
+	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
+	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/sync/errgroup"
+)
+
+// BucketMetadataSys captures all bucket metadata for a given cluster.
+type BucketMetadataSys struct {
+	sync.RWMutex
+	metadataMap map[string]BucketMetadata
+}
+
+// Remove bucket metadata from memory.
+func (sys *BucketMetadataSys) Remove(bucket string) {
+	if globalIsGateway {
+		return
+	}
+	sys.Lock()
+	delete(sys.metadataMap, bucket)
+	sys.Unlock()
+}
+
+// Set - sets a new metadata in-memory.
+// Only a shallow copy is saved and fields with references
+// cannot be modified without causing a race condition,
+// so they should be replaced atomically and not appended to, etc.
+// Data is not persisted to disk.
+func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
+	if globalIsGateway {
+		return
+	}
+
+	if bucket != minioMetaBucket {
+		sys.Lock()
+		sys.metadataMap[bucket] = meta
+		sys.Unlock()
+	}
+}
+
+// Update update bucket metadata for the specified config file.
+// The configData data should not be modified after being sent here.
+func (sys *BucketMetadataSys) Update(bucket string, configFile string, configData []byte) error {
+	objAPI := newObjectLayerWithoutSafeModeFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	if globalIsGateway {
+		// This code is needed only for gateway implementations.
+		switch configFile {
+		case bucketSSEConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.EncryptionConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketLifecycleConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.LifecycleConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketTaggingConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.TaggingConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketNotificationConfig:
+			if globalGatewayName == "nas" {
+				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+				if err != nil {
+					return err
+				}
+				meta.NotificationConfigXML = configData
+				return meta.Save(GlobalContext, objAPI)
+			}
+		case bucketPolicyConfig:
+			if configData == nil {
+				return objAPI.DeleteBucketPolicy(GlobalContext, bucket)
+			}
+			config, err := policy.ParseConfig(bytes.NewReader(configData), bucket)
+			if err != nil {
+				return err
+			}
+			return objAPI.SetBucketPolicy(GlobalContext, bucket, config)
+		}
+		return NotImplemented{}
+	}
+
+	if bucket == minioMetaBucket {
+		return errInvalidArgument
+	}
+
+	meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+	if err != nil {
+		return err
+	}
+
+	switch configFile {
+	case bucketPolicyConfig:
+		meta.PolicyConfigJSON = configData
+	case bucketNotificationConfig:
+		meta.NotificationConfigXML = configData
+	case bucketLifecycleConfig:
+		meta.LifecycleConfigXML = configData
+	case bucketSSEConfig:
+		meta.EncryptionConfigXML = configData
+	case bucketTaggingConfig:
+		meta.TaggingConfigXML = configData
+	case objectLockConfig:
+		meta.ObjectLockConfigXML = configData
+	case bucketQuotaConfigFile:
+		meta.QuotaConfigJSON = configData
+	default:
+		return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
+	}
+
+	if err := meta.Save(GlobalContext, objAPI); err != nil {
+		return err
+	}
+
+	sys.Set(bucket, meta)
+	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
+
+	return nil
+}
+
+// Get metadata for a bucket.
+// If no metadata exists errConfigNotFound is returned and a new metadata is returned.
+// Only a shallow copy is returned, so referenced data should not be modified,
+// but can be replaced atomically.
+func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
+	if globalIsGateway || bucket == minioMetaBucket {
+		return newBucketMetadata(bucket), errConfigNotFound
+	}
+
+	sys.RLock()
+	defer sys.RUnlock()
+
+	meta, ok := sys.metadataMap[bucket]
+	if !ok {
+		return newBucketMetadata(bucket), errConfigNotFound
+	}
+
+	return meta, nil
+}
+
+// GetTaggingConfig returns configured tagging config
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, error) {
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, BucketTaggingNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+	if meta.taggingConfig == nil {
+		return nil, BucketTaggingNotFound{Bucket: bucket}
+	}
+	return meta.taggingConfig, nil
+}
+
+// GetObjectLockConfig returns configured object lock config
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Config, error) {
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, BucketObjectLockConfigNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+	if meta.objectLockConfig == nil {
+		return nil, BucketObjectLockConfigNotFound{Bucket: bucket}
+	}
+	return meta.objectLockConfig, nil
+}
+
+// GetLifecycleConfig returns configured lifecycle config
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, error) {
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, BucketLifecycleNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+	if meta.lifecycleConfig == nil {
+		return nil, BucketLifecycleNotFound{Bucket: bucket}
+	}
+	return meta.lifecycleConfig, nil
+}
+
+// GetNotificationConfig returns configured notification config
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
+	if globalIsGateway && globalGatewayName == "nas" {
+		// Only needed in case of NAS gateway.
+		objAPI := newObjectLayerWithoutSafeModeFn()
+		if objAPI == nil {
+			return nil, errServerNotInitialized
+		}
+		meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+		if err != nil {
+			return nil, err
+		}
+		return meta.notificationConfig, nil
+	}
+
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		return nil, err
+	}
+	return meta.notificationConfig, nil
+}
+
+// GetSSEConfig returns configured SSE config
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEConfig, error) {
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, BucketSSEConfigNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+	if meta.sseConfig == nil {
+		return nil, BucketSSEConfigNotFound{Bucket: bucket}
+	}
+	return meta.sseConfig, nil
+}
+
+// GetPolicyConfig returns configured bucket policy
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, error) {
+	if globalIsGateway {
+		objAPI := newObjectLayerWithoutSafeModeFn()
+		if objAPI == nil {
+			return nil, errServerNotInitialized
+		}
+		return objAPI.GetBucketPolicy(GlobalContext, bucket)
+	}
+
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, BucketPolicyNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+	if meta.policyConfig == nil {
+		return nil, BucketPolicyNotFound{Bucket: bucket}
+	}
+	return meta.policyConfig, nil
+}
+
+// GetQuotaConfig returns configured bucket quota
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetQuotaConfig(bucket string) (*madmin.BucketQuota, error) {
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		return nil, err
+	}
+	return meta.quotaConfig, nil
+}
+
+// GetConfig returns the current bucket metadata
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetConfig(bucket string) (BucketMetadata, error) {
+	objAPI := newObjectLayerWithoutSafeModeFn()
+	if objAPI == nil {
+		return newBucketMetadata(bucket), errServerNotInitialized
+	}
+
+	if globalIsGateway {
+		return newBucketMetadata(bucket), NotImplemented{}
+	}
+
+	if bucket == minioMetaBucket {
+		return newBucketMetadata(bucket), errInvalidArgument
+	}
+
+	sys.RLock()
+	meta, ok := sys.metadataMap[bucket]
+	sys.RUnlock()
+	if ok {
+		return meta, nil
+	}
+	meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+	if err != nil {
+		return meta, err
+	}
+	sys.Lock()
+	sys.metadataMap[bucket] = meta
+	sys.Unlock()
+	return meta, nil
+}
+
+// Init - initializes bucket metadata system for all buckets.
+func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	// In gateway mode, we don't need to load the policies
+	// from the backend.
+	if globalIsGateway {
+		return nil
+	}
+
+	// Load PolicySys once during boot.
+	return sys.load(ctx, buckets, objAPI)
+}
+
+// concurrently load bucket metadata to speed up loading bucket metadata.
+func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+	g := errgroup.WithNErrs(len(buckets))
+	for index := range buckets {
+		index := index
+		g.Go(func() error {
+			meta, err := loadBucketMetadata(ctx, objAPI, buckets[index].Name)
+			if err != nil {
+				return err
+			}
+			sys.Lock()
+			sys.metadataMap[buckets[index].Name] = meta
+			sys.Unlock()
+			return nil
+		}, index)
+	}
+	for _, err := range g.Wait() {
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Loads bucket metadata for all buckets into BucketMetadataSys.
+func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+	count := 100 // load 100 bucket metadata at a time.
+	for {
+		if len(buckets) < count {
+			return sys.concurrentLoad(ctx, buckets, objAPI)
+		}
+		if err := sys.concurrentLoad(ctx, buckets[:count], objAPI); err != nil {
+			return err
+		}
+		buckets = buckets[count:]
+	}
+}
+
+// NewBucketMetadataSys - creates new policy system.
+func NewBucketMetadataSys() *BucketMetadataSys {
+	return &BucketMetadataSys{
+		metadataMap: make(map[string]BucketMetadata),
+	}
+}

cmd/bucket-metadata.go

@@ -0,0 +1,311 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"path"
+	"time"
+
+	"github.com/minio/minio-go/v6/pkg/tags"
+	"github.com/minio/minio/cmd/logger"
+	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
+	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+const (
+	legacyBucketObjectLockEnabledConfigFile = "object-lock-enabled.json"
+	legacyBucketObjectLockEnabledConfig     = `{"x-amz-bucket-object-lock-enabled":true}`
+
+	bucketMetadataFile    = ".metadata.bin"
+	bucketMetadataFormat  = 1
+	bucketMetadataVersion = 1
+)
+
+var (
+	enabledBucketObjectLockConfig = []byte(`<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled></ObjectLockConfiguration>`)
+)
+
+//go:generate msgp -file $GOFILE
+
+// BucketMetadata contains bucket metadata.
+// When adding/removing fields, regenerate the marshal code using the go generate above.
+// Only changing meaning of fields requires a version bump.
+// bucketMetadataFormat refers to the format.
+// bucketMetadataVersion can be used to track a rolling upgrade of a field.
+type BucketMetadata struct {
+	Name                  string
+	Created               time.Time
+	LockEnabled           bool // legacy not used anymore.
+	PolicyConfigJSON      []byte
+	NotificationConfigXML []byte
+	LifecycleConfigXML    []byte
+	ObjectLockConfigXML   []byte
+	EncryptionConfigXML   []byte
+	TaggingConfigXML      []byte
+	QuotaConfigJSON       []byte
+
+	// Unexported fields. Must be updated atomically.
+	policyConfig       *policy.Policy
+	notificationConfig *event.Config
+	lifecycleConfig    *lifecycle.Lifecycle
+	objectLockConfig   *objectlock.Config
+	sseConfig          *bucketsse.BucketSSEConfig
+	taggingConfig      *tags.Tags
+	quotaConfig        *madmin.BucketQuota
+}
+
+// newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
+func newBucketMetadata(name string) BucketMetadata {
+	return BucketMetadata{
+		Name:    name,
+		Created: UTCNow(),
+		notificationConfig: &event.Config{
+			XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
+		},
+		quotaConfig: &madmin.BucketQuota{},
+	}
+}
+
+// Load - loads the metadata of bucket by name from ObjectLayer api.
+// If an error is returned the returned metadata will be default initialized.
+func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
+	configFile := path.Join(bucketConfigPrefix, name, bucketMetadataFile)
+	data, err := readConfig(ctx, api, configFile)
+	if err != nil {
+		return err
+	}
+	if len(data) <= 4 {
+		return fmt.Errorf("loadBucketMetadata: no data")
+	}
+	// Read header
+	switch binary.LittleEndian.Uint16(data[0:2]) {
+	case bucketMetadataFormat:
+	default:
+		return fmt.Errorf("loadBucketMetadata: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+	}
+	switch binary.LittleEndian.Uint16(data[2:4]) {
+	case bucketMetadataVersion:
+	default:
+		return fmt.Errorf("loadBucketMetadata: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+	}
+
+	// OK, parse data.
+	_, err = b.UnmarshalMsg(data[4:])
+	return err
+}
+
+// loadBucketMetadata loads and migrates to bucket metadata.
+func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
+	b := newBucketMetadata(bucket)
+	err := b.Load(ctx, objectAPI, bucket)
+	if err == nil {
+		return b, b.convertLegacyConfigs(ctx, objectAPI)
+	}
+
+	if err != errConfigNotFound {
+		return b, err
+	}
+
+	// Old bucket without bucket metadata. Hence we migrate existing settings.
+	return b, b.convertLegacyConfigs(ctx, objectAPI)
+}
+
+// parseAllConfigs will parse all configs and populate the private fields.
+// The first error encountered is returned.
+func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLayer) (err error) {
+	if len(b.PolicyConfigJSON) != 0 {
+		b.policyConfig, err = policy.ParseConfig(bytes.NewReader(b.PolicyConfigJSON), b.Name)
+		if err != nil {
+			return err
+		}
+	} else {
+		b.policyConfig = nil
+	}
+
+	if len(b.NotificationConfigXML) != 0 {
+		if err = xml.Unmarshal(b.NotificationConfigXML, b.notificationConfig); err != nil {
+			return err
+		}
+	}
+
+	if len(b.LifecycleConfigXML) != 0 {
+		b.lifecycleConfig, err = lifecycle.ParseLifecycleConfig(bytes.NewReader(b.LifecycleConfigXML))
+		if err != nil {
+			return err
+		}
+	} else {
+		b.lifecycleConfig = nil
+	}
+
+	if len(b.EncryptionConfigXML) != 0 {
+		b.sseConfig, err = bucketsse.ParseBucketSSEConfig(bytes.NewReader(b.EncryptionConfigXML))
+		if err != nil {
+			return err
+		}
+	} else {
+		b.sseConfig = nil
+	}
+
+	if len(b.TaggingConfigXML) != 0 {
+		b.taggingConfig, err = tags.ParseBucketXML(bytes.NewReader(b.TaggingConfigXML))
+		if err != nil {
+			return err
+		}
+	} else {
+		b.taggingConfig = nil
+	}
+
+	if len(b.ObjectLockConfigXML) != 0 {
+		b.objectLockConfig, err = objectlock.ParseObjectLockConfig(bytes.NewReader(b.ObjectLockConfigXML))
+		if err != nil {
+			return err
+		}
+	} else {
+		b.objectLockConfig = nil
+	}
+
+	if len(b.QuotaConfigJSON) != 0 {
+		b.quotaConfig, err = parseBucketQuota(b.Name, b.QuotaConfigJSON)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer) error {
+	legacyConfigs := []string{
+		legacyBucketObjectLockEnabledConfigFile,
+		bucketPolicyConfig,
+		bucketNotificationConfig,
+		bucketLifecycleConfig,
+		bucketQuotaConfigFile,
+		bucketSSEConfig,
+		bucketTaggingConfig,
+		objectLockConfig,
+	}
+
+	configs := make(map[string][]byte)
+
+	// Handle migration from lockEnabled to newer format.
+	if b.LockEnabled {
+		configs[objectLockConfig] = enabledBucketObjectLockConfig
+		b.LockEnabled = false // legacy value unset it
+		// we are only interested in b.ObjectLockConfigXML or objectLockConfig value
+	}
+
+	for _, legacyFile := range legacyConfigs {
+		configFile := path.Join(bucketConfigPrefix, b.Name, legacyFile)
+
+		configData, err := readConfig(ctx, objectAPI, configFile)
+		if err != nil {
+			if errors.Is(err, errConfigNotFound) {
+				// legacy file config not found, proceed to look for new metadata.
+				continue
+			}
+
+			return err
+		}
+		configs[legacyFile] = configData
+	}
+
+	if len(configs) == 0 {
+		// nothing to update, return right away.
+		return b.parseAllConfigs(ctx, objectAPI)
+	}
+
+	for legacyFile, configData := range configs {
+		switch legacyFile {
+		case legacyBucketObjectLockEnabledConfigFile:
+			if string(configData) == legacyBucketObjectLockEnabledConfig {
+				b.ObjectLockConfigXML = enabledBucketObjectLockConfig
+				b.LockEnabled = false // legacy value unset it
+				// we are only interested in b.ObjectLockConfigXML
+			}
+		case bucketPolicyConfig:
+			b.PolicyConfigJSON = configData
+		case bucketNotificationConfig:
+			b.NotificationConfigXML = configData
+		case bucketLifecycleConfig:
+			b.LifecycleConfigXML = configData
+		case bucketSSEConfig:
+			b.EncryptionConfigXML = configData
+		case bucketTaggingConfig:
+			b.TaggingConfigXML = configData
+		case objectLockConfig:
+			b.ObjectLockConfigXML = configData
+		case bucketQuotaConfigFile:
+			b.QuotaConfigJSON = configData
+		}
+	}
+
+	if err := b.Save(ctx, objectAPI); err != nil {
+		return err
+	}
+
+	for legacyFile := range configs {
+		configFile := path.Join(bucketConfigPrefix, b.Name, legacyFile)
+		if err := deleteConfig(ctx, objectAPI, configFile); err != nil && !errors.Is(err, errConfigNotFound) {
+			logger.LogIf(ctx, err)
+		}
+	}
+
+	return nil
+}
+
+// Save config to supplied ObjectLayer api.
+func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
+	if err := b.parseAllConfigs(ctx, api); err != nil {
+		return err
+	}
+
+	data := make([]byte, 4, b.Msgsize()+4)
+
+	// Initialize the header.
+	binary.LittleEndian.PutUint16(data[0:2], bucketMetadataFormat)
+	binary.LittleEndian.PutUint16(data[2:4], bucketMetadataVersion)
+
+	// Marshal the bucket metadata
+	data, err := b.MarshalMsg(data)
+	if err != nil {
+		return err
+	}
+
+	configFile := path.Join(bucketConfigPrefix, b.Name, bucketMetadataFile)
+	return saveConfig(ctx, api, configFile, data)
+}
+
+// deleteBucketMetadata deletes bucket metadata
+// If config does not exist no error is returned.
+func deleteBucketMetadata(ctx context.Context, obj ObjectLayer, bucket string) error {
+	configFile := path.Join(bucketConfigPrefix, bucket, bucketMetadataFile)
+	if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
+		return err
+	}
+	return nil
+}

cmd/bucket-metadata_gen.go

@@ -0,0 +1,335 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Name":
+			z.Name, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "Created":
+			z.Created, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "Created")
+				return
+			}
+		case "LockEnabled":
+			z.LockEnabled, err = dc.ReadBool()
+			if err != nil {
+				err = msgp.WrapError(err, "LockEnabled")
+				return
+			}
+		case "PolicyConfigJSON":
+			z.PolicyConfigJSON, err = dc.ReadBytes(z.PolicyConfigJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "PolicyConfigJSON")
+				return
+			}
+		case "NotificationConfigXML":
+			z.NotificationConfigXML, err = dc.ReadBytes(z.NotificationConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "NotificationConfigXML")
+				return
+			}
+		case "LifecycleConfigXML":
+			z.LifecycleConfigXML, err = dc.ReadBytes(z.LifecycleConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "LifecycleConfigXML")
+				return
+			}
+		case "ObjectLockConfigXML":
+			z.ObjectLockConfigXML, err = dc.ReadBytes(z.ObjectLockConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "ObjectLockConfigXML")
+				return
+			}
+		case "EncryptionConfigXML":
+			z.EncryptionConfigXML, err = dc.ReadBytes(z.EncryptionConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "EncryptionConfigXML")
+				return
+			}
+		case "TaggingConfigXML":
+			z.TaggingConfigXML, err = dc.ReadBytes(z.TaggingConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "TaggingConfigXML")
+				return
+			}
+		case "QuotaConfigJSON":
+			z.QuotaConfigJSON, err = dc.ReadBytes(z.QuotaConfigJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "QuotaConfigJSON")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 10
+	// write "Name"
+	err = en.Append(0x8a, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Name)
+	if err != nil {
+		err = msgp.WrapError(err, "Name")
+		return
+	}
+	// write "Created"
+	err = en.Append(0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.Created)
+	if err != nil {
+		err = msgp.WrapError(err, "Created")
+		return
+	}
+	// write "LockEnabled"
+	err = en.Append(0xab, 0x4c, 0x6f, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteBool(z.LockEnabled)
+	if err != nil {
+		err = msgp.WrapError(err, "LockEnabled")
+		return
+	}
+	// write "PolicyConfigJSON"
+	err = en.Append(0xb0, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.PolicyConfigJSON)
+	if err != nil {
+		err = msgp.WrapError(err, "PolicyConfigJSON")
+		return
+	}
+	// write "NotificationConfigXML"
+	err = en.Append(0xb5, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.NotificationConfigXML)
+	if err != nil {
+		err = msgp.WrapError(err, "NotificationConfigXML")
+		return
+	}
+	// write "LifecycleConfigXML"
+	err = en.Append(0xb2, 0x4c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.LifecycleConfigXML)
+	if err != nil {
+		err = msgp.WrapError(err, "LifecycleConfigXML")
+		return
+	}
+	// write "ObjectLockConfigXML"
+	err = en.Append(0xb3, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.ObjectLockConfigXML)
+	if err != nil {
+		err = msgp.WrapError(err, "ObjectLockConfigXML")
+		return
+	}
+	// write "EncryptionConfigXML"
+	err = en.Append(0xb3, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.EncryptionConfigXML)
+	if err != nil {
+		err = msgp.WrapError(err, "EncryptionConfigXML")
+		return
+	}
+	// write "TaggingConfigXML"
+	err = en.Append(0xb0, 0x54, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.TaggingConfigXML)
+	if err != nil {
+		err = msgp.WrapError(err, "TaggingConfigXML")
+		return
+	}
+	// write "QuotaConfigJSON"
+	err = en.Append(0xaf, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.QuotaConfigJSON)
+	if err != nil {
+		err = msgp.WrapError(err, "QuotaConfigJSON")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 10
+	// string "Name"
+	o = append(o, 0x8a, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = msgp.AppendString(o, z.Name)
+	// string "Created"
+	o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
+	o = msgp.AppendTime(o, z.Created)
+	// string "LockEnabled"
+	o = append(o, 0xab, 0x4c, 0x6f, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64)
+	o = msgp.AppendBool(o, z.LockEnabled)
+	// string "PolicyConfigJSON"
+	o = append(o, 0xb0, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
+	o = msgp.AppendBytes(o, z.PolicyConfigJSON)
+	// string "NotificationConfigXML"
+	o = append(o, 0xb5, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	o = msgp.AppendBytes(o, z.NotificationConfigXML)
+	// string "LifecycleConfigXML"
+	o = append(o, 0xb2, 0x4c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	o = msgp.AppendBytes(o, z.LifecycleConfigXML)
+	// string "ObjectLockConfigXML"
+	o = append(o, 0xb3, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	o = msgp.AppendBytes(o, z.ObjectLockConfigXML)
+	// string "EncryptionConfigXML"
+	o = append(o, 0xb3, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	o = msgp.AppendBytes(o, z.EncryptionConfigXML)
+	// string "TaggingConfigXML"
+	o = append(o, 0xb0, 0x54, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	o = msgp.AppendBytes(o, z.TaggingConfigXML)
+	// string "QuotaConfigJSON"
+	o = append(o, 0xaf, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
+	o = msgp.AppendBytes(o, z.QuotaConfigJSON)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Name":
+			z.Name, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "Created":
+			z.Created, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Created")
+				return
+			}
+		case "LockEnabled":
+			z.LockEnabled, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "LockEnabled")
+				return
+			}
+		case "PolicyConfigJSON":
+			z.PolicyConfigJSON, bts, err = msgp.ReadBytesBytes(bts, z.PolicyConfigJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "PolicyConfigJSON")
+				return
+			}
+		case "NotificationConfigXML":
+			z.NotificationConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.NotificationConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "NotificationConfigXML")
+				return
+			}
+		case "LifecycleConfigXML":
+			z.LifecycleConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.LifecycleConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "LifecycleConfigXML")
+				return
+			}
+		case "ObjectLockConfigXML":
+			z.ObjectLockConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.ObjectLockConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "ObjectLockConfigXML")
+				return
+			}
+		case "EncryptionConfigXML":
+			z.EncryptionConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.EncryptionConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "EncryptionConfigXML")
+				return
+			}
+		case "TaggingConfigXML":
+			z.TaggingConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.TaggingConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "TaggingConfigXML")
+				return
+			}
+		case "QuotaConfigJSON":
+			z.QuotaConfigJSON, bts, err = msgp.ReadBytesBytes(bts, z.QuotaConfigJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "QuotaConfigJSON")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *BucketMetadata) Msgsize() (s int) {
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON)
+	return
+}

cmd/bucket-metadata_gen_test.go

@@ -9,8 +9,8 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
-func TestMarshalUnmarshalbucketMetadata(t *testing.T) {
-	v := bucketMetadata{}
+func TestMarshalUnmarshalBucketMetadata(t *testing.T) {
+	v := BucketMetadata{}
 	bts, err := v.MarshalMsg(nil)
 	if err != nil {
 		t.Fatal(err)
@@ -32,8 +32,8 @@ func TestMarshalUnmarshalbucketMetadata(t *testing.T) {
 	}
 }
 
-func BenchmarkMarshalMsgbucketMetadata(b *testing.B) {
-	v := bucketMetadata{}
+func BenchmarkMarshalMsgBucketMetadata(b *testing.B) {
+	v := BucketMetadata{}
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -41,8 +41,8 @@ func BenchmarkMarshalMsgbucketMetadata(b *testing.B) {
 	}
 }
 
-func BenchmarkAppendMsgbucketMetadata(b *testing.B) {
-	v := bucketMetadata{}
+func BenchmarkAppendMsgBucketMetadata(b *testing.B) {
+	v := BucketMetadata{}
 	bts := make([]byte, 0, v.Msgsize())
 	bts, _ = v.MarshalMsg(bts[0:0])
 	b.SetBytes(int64(len(bts)))
@@ -53,8 +53,8 @@ func BenchmarkAppendMsgbucketMetadata(b *testing.B) {
 	}
 }
 
-func BenchmarkUnmarshalbucketMetadata(b *testing.B) {
-	v := bucketMetadata{}
+func BenchmarkUnmarshalBucketMetadata(b *testing.B) {
+	v := BucketMetadata{}
 	bts, _ := v.MarshalMsg(nil)
 	b.ReportAllocs()
 	b.SetBytes(int64(len(bts)))
@@ -67,17 +67,17 @@ func BenchmarkUnmarshalbucketMetadata(b *testing.B) {
 	}
 }
 
-func TestEncodeDecodebucketMetadata(t *testing.T) {
-	v := bucketMetadata{}
+func TestEncodeDecodeBucketMetadata(t *testing.T) {
+	v := BucketMetadata{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 
 	m := v.Msgsize()
 	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodebucketMetadata Msgsize() is inaccurate")
+		t.Log("WARNING: TestEncodeDecodeBucketMetadata Msgsize() is inaccurate")
 	}
 
-	vn := bucketMetadata{}
+	vn := BucketMetadata{}
 	err := msgp.Decode(&buf, &vn)
 	if err != nil {
 		t.Error(err)
@@ -91,8 +91,8 @@ func TestEncodeDecodebucketMetadata(t *testing.T) {
 	}
 }
 
-func BenchmarkEncodebucketMetadata(b *testing.B) {
-	v := bucketMetadata{}
+func BenchmarkEncodeBucketMetadata(b *testing.B) {
+	v := BucketMetadata{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
@@ -105,8 +105,8 @@ func BenchmarkEncodebucketMetadata(b *testing.B) {
 	en.Flush()
 }
 
-func BenchmarkDecodebucketMetadata(b *testing.B) {
-	v := bucketMetadata{}
+func BenchmarkDecodeBucketMetadata(b *testing.B) {
+	v := BucketMetadata{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))

cmd/bucket-notification-handlers.go

@@ -19,10 +19,8 @@ package cmd
 import (
 	"encoding/json"
 	"encoding/xml"
-	"errors"
 	"io"
 	"net/http"
-	"path"
 	"reflect"
 	"time"
 
@@ -38,8 +36,6 @@ const (
 	bucketNotificationConfig = "notification.xml"
 )
 
-var errNoSuchNotifications = errors.New("The specified bucket does not have bucket notifications")
-
 // GetBucketNotificationHandler - This HTTP handler returns event notification configuration
 // as per http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html.
 // It returns empty configuration if its not set.
@@ -73,71 +69,44 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	// Construct path to notification.xml for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucketName, bucketNotificationConfig)
-
-	var config = event.Config{}
-	configData, err := readConfig(ctx, objAPI, configFile)
+	config, err := globalBucketMetadataSys.GetNotificationConfig(bucketName)
 	if err != nil {
-		if err != errConfigNotFound {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-		config.SetRegion(globalServerRegion)
-		config.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
-		notificationBytes, err := xml.Marshal(config)
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-
-		writeSuccessResponseXML(w, notificationBytes)
-		return
-	}
-
-	config.SetRegion(globalServerRegion)
-
-	if err = xml.Unmarshal(configData, &config); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-
+	config.SetRegion(globalServerRegion)
 	if err = config.Validate(globalServerRegion, globalNotificationSys.targetList); err != nil {
 		arnErr, ok := err.(*event.ErrARNNotFound)
-		if !ok {
+		if ok {
+			for i, queue := range config.QueueList {
+				// Remove ARN not found queues, because we previously allowed
+				// adding unexpected entries into the config.
+				//
+				// With newer config disallowing changing / turning off
+				// notification targets without removing ARN in notification
+				// configuration we won't see this problem anymore.
+				if reflect.DeepEqual(queue.ARN, arnErr.ARN) && i < len(config.QueueList) {
+					config.QueueList = append(config.QueueList[:i],
+						config.QueueList[i+1:]...)
+				}
+				// This is a one time activity we shall do this
+				// here and allow stale ARN to be removed. We shall
+				// never reach a stage where we will have stale
+				// notification configs.
+			}
+		} else {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		for i, queue := range config.QueueList {
-			// Remove ARN not found queues, because we previously allowed
-			// adding unexpected entries into the config.
-			//
-			// With newer config disallowing changing / turning off
-			// notification targets without removing ARN in notification
-			// configuration we won't see this problem anymore.
-			if reflect.DeepEqual(queue.ARN, arnErr.ARN) && i < len(config.QueueList) {
-				config.QueueList = append(config.QueueList[:i],
-					config.QueueList[i+1:]...)
-			}
-			// This is a one time activity we shall do this
-			// here and allow stale ARN to be removed. We shall
-			// never reach a stage where we will have stale
-			// notification configs.
-		}
 	}
 
-	// If xml namespace is empty, set a default value before returning.
-	if config.XMLNS == "" {
-		config.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
-	}
-
-	notificationBytes, err := xml.Marshal(config)
+	configData, err := xml.Marshal(config)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	writeSuccessResponseXML(w, notificationBytes)
+	writeSuccessResponseXML(w, configData)
 }
 
 // PutBucketNotificationHandler - This HTTP handler stores given notification configuration as per
@@ -178,8 +147,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	lreader := io.LimitReader(r.Body, r.ContentLength)
-	config, err := event.ParseConfig(lreader, globalServerRegion, globalNotificationSys.targetList)
+	config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalServerRegion, globalNotificationSys.targetList)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		if event.IsEventError(err) {
@@ -189,14 +157,19 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
+	configData, err := xml.Marshal(config)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if err = globalBucketMetadataSys.Update(bucketName, bucketNotificationConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	rulesMap := config.ToRulesMap()
 	globalNotificationSys.AddRulesMap(bucketName, rulesMap)
-	globalNotificationSys.PutBucketNotification(ctx, bucketName, rulesMap)
 
 	writeSuccessResponseHeadersOnly(w)
 }

cmd/bucket-object-lock.go

@@ -17,12 +17,9 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
-	"errors"
 	"math"
 	"net/http"
-	"path"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
@@ -30,6 +27,31 @@ import (
 	"github.com/minio/minio/pkg/bucket/policy"
 )
 
+// BucketObjectLockSys - map of bucket and retention configuration.
+type BucketObjectLockSys struct{}
+
+// Get - Get retention configuration.
+func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention, err error) {
+	if globalIsGateway {
+		objAPI := newObjectLayerWithoutSafeModeFn()
+		if objAPI == nil {
+			return r, errServerNotInitialized
+		}
+
+		return r, nil
+	}
+
+	config, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName)
+	if err != nil {
+		if _, ok := err.(BucketObjectLockConfigNotFound); ok {
+			return r, nil
+		}
+		return r, err
+
+	}
+	return config.ToRetention(), nil
+}
+
 // Similar to enforceRetentionBypassForDelete but for WebUI
 func enforceRetentionBypassForDeleteWeb(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, govBypassPerms bool) APIErrorCode {
 	opts, err := getOpts(ctx, r, bucket, object)
@@ -295,11 +317,16 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 	retentionRequested := objectlock.IsObjectLockRetentionRequested(r.Header)
 	legalHoldRequested := objectlock.IsObjectLockLegalHoldRequested(r.Header)
 
-	retentionCfg, isWORMBucket := globalBucketObjectLockConfig.Get(bucket)
-	if !isWORMBucket {
+	retentionCfg, err := globalBucketObjectLockSys.Get(bucket)
+	if err != nil {
+		return mode, retainDate, legalHold, ErrInvalidBucketObjectLockConfiguration
+	}
+
+	if !retentionCfg.LockEnabled {
 		if legalHoldRequested || retentionRequested {
 			return mode, retainDate, legalHold, ErrInvalidBucketObjectLockConfiguration
 		}
+
 		// If this not a WORM enabled bucket, we should return right here.
 		return mode, retainDate, legalHold, ErrNone
 	}
@@ -356,7 +383,7 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 		return rMode, rDate, legalHold, ErrNone
 	}
 
-	if !retentionRequested && isWORMBucket {
+	if !retentionRequested && retentionCfg.Validity > 0 {
 		if retentionPermErr != ErrNone {
 			return mode, retainDate, legalHold, retentionPermErr
 		}
@@ -369,7 +396,7 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 		if objExists && retainDate.After(t) {
 			return mode, retainDate, legalHold, ErrObjectLocked
 		}
-		if !legalHoldRequested && !retentionCfg.IsEmpty() {
+		if !legalHoldRequested {
 			// inherit retention from bucket configuration
 			return retentionCfg.Mode, objectlock.RetentionDate{Time: t.Add(retentionCfg.Validity)}, legalHold, ErrNone
 		}
@@ -378,41 +405,7 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 	return mode, retainDate, legalHold, ErrNone
 }
 
-func initBucketObjectLockConfig(buckets []BucketInfo, objAPI ObjectLayer) error {
-	for _, bucket := range buckets {
-		ctx := logger.SetReqInfo(GlobalContext, &logger.ReqInfo{BucketName: bucket.Name})
-		meta, err := loadBucketMetadata(ctx, objAPI, bucket.Name)
-		if err != nil {
-			if err != errMetaDataConverted {
-				return err
-			}
-
-			meta.Created = bucket.Created
-			logger.LogIf(ctx, meta.save(ctx, objAPI))
-		}
-		if !meta.LockEnabled {
-			continue
-		}
-
-		configFile := path.Join(bucketConfigPrefix, bucket.Name, objectLockConfig)
-		configData, err := readConfig(ctx, objAPI, configFile)
-		if err != nil {
-			if errors.Is(err, errConfigNotFound) {
-				globalBucketObjectLockConfig.Set(bucket.Name, &objectlock.Retention{})
-				continue
-			}
-			return err
-		}
-
-		config, err := objectlock.ParseObjectLockConfig(bytes.NewReader(configData))
-		if err != nil {
-			return err
-		}
-		retention := &objectlock.Retention{}
-		if config.Rule != nil {
-			retention = config.ToRetention()
-		}
-		globalBucketObjectLockConfig.Set(bucket.Name, retention)
-	}
-	return nil
+// NewBucketObjectLockSys returns initialized BucketObjectLockSys
+func NewBucketObjectLockSys() *BucketObjectLockSys {
+	return &BucketObjectLockSys{}
 }

cmd/bucket-policy-handlers.go

@@ -87,13 +87,16 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if err = objAPI.SetBucketPolicy(ctx, bucket, bucketPolicy); err != nil {
+	configData, err := json.Marshal(bucketPolicy)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	globalPolicySys.Set(bucket, bucketPolicy)
-	globalNotificationSys.SetBucketPolicy(ctx, bucket, bucketPolicy)
+	if err = globalBucketMetadataSys.Update(bucket, bucketPolicyConfig, configData); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
 
 	// Success.
 	writeSuccessNoContent(w)
@@ -125,14 +128,11 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err := objAPI.DeleteBucketPolicy(ctx, bucket); err != nil {
+	if err := globalBucketMetadataSys.Update(bucket, bucketPolicyConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	globalPolicySys.Remove(bucket)
-	globalNotificationSys.RemoveBucketPolicy(ctx, bucket)
-
 	// Success.
 	writeSuccessNoContent(w)
 }
@@ -164,18 +164,18 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Read bucket access policy.
-	bucketPolicy, err := objAPI.GetBucketPolicy(ctx, bucket)
+	config, err := globalBucketMetadataSys.GetPolicyConfig(bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	policyData, err := json.Marshal(bucketPolicy)
+	configData, err := json.Marshal(config)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Write to client.
-	w.Write(policyData)
+	writeSuccessResponseJSON(w, configData)
 }

cmd/bucket-policy-handlers_test.go

@@ -102,7 +102,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	credentials auth.Credentials, t *testing.T) {
 
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, ""); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, "", false); err != nil {
 		t.Fatal(err)
 	}
 

cmd/bucket-policy.go

@@ -17,15 +17,11 @@
 package cmd
 
 import (
-	"bytes"
-	"context"
 	"encoding/json"
 	"net/http"
 	"net/url"
-	"path"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
@@ -37,56 +33,23 @@ import (
 )
 
 // PolicySys - policy subsystem.
-type PolicySys struct {
-	sync.RWMutex
-	bucketPolicyMap map[string]*policy.Policy
-}
+type PolicySys struct{}
 
-// Set - sets policy to given bucket name.  If policy is empty, existing policy is removed.
-func (sys *PolicySys) Set(bucketName string, policy *policy.Policy) {
-	if globalIsGateway {
-		// Set policy is a non-op under gateway mode.
-		return
-	}
-
-	sys.Lock()
-	defer sys.Unlock()
-
-	if policy.IsEmpty() {
-		delete(sys.bucketPolicyMap, bucketName)
-	} else {
-		sys.bucketPolicyMap[bucketName] = policy
-	}
-}
-
-// Remove - removes policy for given bucket name.
-func (sys *PolicySys) Remove(bucketName string) {
-	sys.Lock()
-	defer sys.Unlock()
-
-	delete(sys.bucketPolicyMap, bucketName)
+// Get returns stored bucket policy
+func (sys *PolicySys) Get(bucket string) (*policy.Policy, error) {
+	return globalBucketMetadataSys.GetPolicyConfig(bucket)
 }
 
 // IsAllowed - checks given policy args is allowed to continue the Rest API.
 func (sys *PolicySys) IsAllowed(args policy.Args) bool {
-	if globalIsGateway {
-		// When gateway is enabled, no cached value
-		// is used to validate bucket policies.
-		objAPI := newObjectLayerFn()
-		if objAPI != nil {
-			config, err := objAPI.GetBucketPolicy(GlobalContext, args.BucketName)
-			if err == nil {
-				return config.IsAllowed(args)
-			}
-		}
-	} else {
-		sys.RLock()
-		defer sys.RUnlock()
+	p, err := sys.Get(args.BucketName)
+	if err == nil {
+		return p.IsAllowed(args)
+	}
 
-		// If policy is available for given bucket, check the policy.
-		if p, found := sys.bucketPolicyMap[args.BucketName]; found {
-			return p.IsAllowed(args)
-		}
+	// Log unhandled errors.
+	if _, ok := err.(BucketPolicyNotFound); !ok {
+		logger.LogIf(GlobalContext, err)
 	}
 
 	// As policy is not available for given bucket name, returns IsOwner i.e.
@@ -94,56 +57,9 @@ func (sys *PolicySys) IsAllowed(args policy.Args) bool {
 	return args.IsOwner
 }
 
-// Loads policies for all buckets into PolicySys.
-func (sys *PolicySys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
-	for _, bucket := range buckets {
-		config, err := getPolicyConfig(objAPI, bucket.Name)
-		if err != nil {
-			if _, ok := err.(BucketPolicyNotFound); ok {
-				continue
-			}
-			return err
-		}
-		// This part is specifically written to handle migration
-		// when the Version string is empty, this was allowed
-		// in all previous minio releases but we need to migrate
-		// those policies by properly setting the Version string
-		// from now on.
-		if config.Version == "" {
-			logger.Info("Found in-consistent bucket policies, Migrating them for Bucket: (%s)", bucket.Name)
-			config.Version = policy.DefaultVersion
-
-			if err = savePolicyConfig(GlobalContext, objAPI, bucket.Name, config); err != nil {
-				logger.LogIf(GlobalContext, err)
-				return err
-			}
-		}
-		sys.Set(bucket.Name, config)
-	}
-	return nil
-}
-
-// Init - initializes policy system from policy.json of all buckets.
-func (sys *PolicySys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	// In gateway mode, we don't need to load the policies
-	// from the backend.
-	if globalIsGateway {
-		return nil
-	}
-
-	// Load PolicySys once during boot.
-	return sys.load(buckets, objAPI)
-}
-
 // NewPolicySys - creates new policy system.
 func NewPolicySys() *PolicySys {
-	return &PolicySys{
-		bucketPolicyMap: make(map[string]*policy.Policy),
-	}
+	return &PolicySys{}
 }
 
 func getConditionValues(r *http.Request, lc string, username string, claims map[string]interface{}) map[string][]string {
@@ -226,50 +142,6 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 	return args
 }
 
-// getPolicyConfig - get policy config for given bucket name.
-func getPolicyConfig(objAPI ObjectLayer, bucketName string) (*policy.Policy, error) {
-	// Construct path to policy.json for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
-
-	configData, err := readConfig(GlobalContext, objAPI, configFile)
-	if err != nil {
-		if err == errConfigNotFound {
-			err = BucketPolicyNotFound{Bucket: bucketName}
-		}
-
-		return nil, err
-	}
-
-	return policy.ParseConfig(bytes.NewReader(configData), bucketName)
-}
-
-func savePolicyConfig(ctx context.Context, objAPI ObjectLayer, bucketName string, bucketPolicy *policy.Policy) error {
-	data, err := json.Marshal(bucketPolicy)
-	if err != nil {
-		return err
-	}
-
-	// Construct path to policy.json for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
-
-	return saveConfig(ctx, objAPI, configFile, data)
-}
-
-func removePolicyConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) error {
-	// Construct path to policy.json for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
-
-	if err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile); err != nil {
-		if _, ok := err.(ObjectNotFound); ok {
-			return BucketPolicyNotFound{Bucket: bucketName}
-		}
-
-		return err
-	}
-
-	return nil
-}
-
 // PolicyToBucketAccessPolicy - converts policy.Policy to minio-go/policy.BucketAccessPolicy.
 func PolicyToBucketAccessPolicy(bucketPolicy *policy.Policy) (*miniogopolicy.BucketAccessPolicy, error) {
 	// Return empty BucketAccessPolicy for empty bucket policy.

cmd/bucket-quota.go

@@ -19,10 +19,7 @@ package cmd
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"fmt"
-	"path"
-	"sync"
 	"time"
 
 	"github.com/minio/minio/cmd/config"
@@ -34,128 +31,93 @@ import (
 
 // BucketQuotaSys - map of bucket and quota configuration.
 type BucketQuotaSys struct {
-	sync.RWMutex
-	quotaMap map[string]madmin.BucketQuota
-}
-
-// Set - set quota configuration.
-func (sys *BucketQuotaSys) Set(bucketName string, q madmin.BucketQuota) {
-	sys.Lock()
-	sys.quotaMap[bucketName] = q
-	sys.Unlock()
+	bucketStorageCache timedValue
 }
 
 // Get - Get quota configuration.
-func (sys *BucketQuotaSys) Get(bucketName string) (q madmin.BucketQuota, ok bool) {
-	sys.RLock()
-	defer sys.RUnlock()
-	q, ok = sys.quotaMap[bucketName]
-	return
-}
-
-// Remove - removes quota configuration.
-func (sys *BucketQuotaSys) Remove(bucketName string) {
-	sys.Lock()
-	delete(sys.quotaMap, bucketName)
-	sys.Unlock()
-}
-
-// Buckets - list of buckets with quota configuration
-func (sys *BucketQuotaSys) Buckets() []string {
-	sys.RLock()
-	defer sys.RUnlock()
-	var buckets []string
-	for k := range sys.quotaMap {
-		buckets = append(buckets, k)
-	}
-	return buckets
-}
-
-// Init initialize bucket quota sys configuration with all buckets.
-func (sys *BucketQuotaSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	// In gateway mode, we do not support bucket quota.
-	// So, this is a no-op for gateway servers.
+func (sys *BucketQuotaSys) Get(bucketName string) (*madmin.BucketQuota, error) {
 	if globalIsGateway {
-		return nil
+		objAPI := newObjectLayerFn()
+		if objAPI == nil {
+			return nil, errServerNotInitialized
+		}
+		return &madmin.BucketQuota{}, nil
 	}
 
-	return sys.load(buckets, objAPI)
-}
-
-func (sys *BucketQuotaSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
-	for _, bucket := range buckets {
-		ctx := logger.SetReqInfo(GlobalContext, &logger.ReqInfo{BucketName: bucket.Name})
-		configFile := path.Join(bucketConfigPrefix, bucket.Name, bucketQuotaConfigFile)
-		configData, err := readConfig(ctx, objAPI, configFile)
-		if err != nil {
-			if errors.Is(err, errConfigNotFound) {
-				continue
-			}
-			return err
-		}
-		quotaCfg, err := parseBucketQuota(configData)
-		if err != nil {
-			return err
-		}
-		globalBucketQuotaSys.Set(bucket.Name, quotaCfg)
-	}
-	return nil
+	return globalBucketMetadataSys.GetQuotaConfig(bucketName)
 }
 
 // NewBucketQuotaSys returns initialized BucketQuotaSys
 func NewBucketQuotaSys() *BucketQuotaSys {
-	return &BucketQuotaSys{quotaMap: map[string]madmin.BucketQuota{}}
+	return &BucketQuotaSys{}
 }
 
 // parseBucketQuota parses BucketQuota from json
-func parseBucketQuota(data []byte) (quotaCfg madmin.BucketQuota, err error) {
-	err = json.Unmarshal(data, &quotaCfg)
-	if err != nil {
-		return
+func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota, err error) {
+	quotaCfg = &madmin.BucketQuota{}
+	if err = json.Unmarshal(data, quotaCfg); err != nil {
+		return quotaCfg, err
 	}
-	if !quotaCfg.Type.IsValid() {
-		return quotaCfg, fmt.Errorf("Invalid quota type %s", quotaCfg.Type)
+	if !quotaCfg.IsValid() {
+		return quotaCfg, fmt.Errorf("Invalid quota config %#v", quotaCfg)
 	}
 	return
 }
 
-type bucketStorageCache struct {
-	bucketsSizes map[string]uint64
-	lastUpdate   time.Time
-	mu           sync.Mutex
-}
-
-func (b *bucketStorageCache) check(ctx context.Context, q madmin.BucketQuota, bucket string, size int64) error {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	if time.Since(b.lastUpdate) > 10*time.Second {
-		dui, err := loadDataUsageFromBackend(ctx, newObjectLayerWithoutSafeModeFn())
-		if err != nil {
-			return err
-		}
-		b.lastUpdate = time.Now()
-		b.bucketsSizes = dui.BucketsSizes
+func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64) error {
+	objAPI := newObjectLayerWithoutSafeModeFn()
+	if objAPI == nil {
+		return errServerNotInitialized
 	}
-	currUsage := b.bucketsSizes[bucket]
-	if (currUsage + uint64(size)) > q.Quota {
+
+	q, err := sys.Get(bucket)
+	if err != nil {
+		return nil
+	}
+
+	if q.Type == madmin.FIFOQuota {
+		return nil
+	}
+
+	if q.Quota == 0 {
+		// No quota set return quickly.
+		return nil
+	}
+
+	sys.bucketStorageCache.Once.Do(func() {
+		sys.bucketStorageCache.TTL = 10 * time.Second
+		sys.bucketStorageCache.Update = func() (interface{}, error) {
+			return loadDataUsageFromBackend(ctx, objAPI)
+		}
+	})
+
+	v, err := sys.bucketStorageCache.Get()
+	if err != nil {
+		return err
+	}
+
+	dui := v.(DataUsageInfo)
+
+	bui, ok := dui.BucketsUsage[bucket]
+	if !ok {
+		// bucket not found, cannot enforce quota
+		// call will fail anyways later.
+		return nil
+	}
+
+	if (bui.Size + uint64(size)) > q.Quota {
 		return BucketQuotaExceeded{Bucket: bucket}
 	}
+
 	return nil
 }
+
 func enforceBucketQuota(ctx context.Context, bucket string, size int64) error {
 	if size < 0 {
 		return nil
 	}
-	q, ok := globalBucketQuotaSys.Get(bucket)
-	if !ok {
-		return nil
-	}
 
-	return globalBucketStorageCache.check(ctx, q, bucket, size)
+	return globalBucketQuotaSys.check(ctx, bucket, size)
 }
 
 const (
@@ -189,28 +151,46 @@ func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) error {
 	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
 		return nil
 	}
-	for _, bucket := range globalBucketQuotaSys.Buckets() {
-		// Check if the current bucket has quota restrictions, if not skip it
-		cfg, ok := globalBucketQuotaSys.Get(bucket)
+
+	buckets, err := objectAPI.ListBuckets(ctx)
+	if err != nil {
+		return err
+	}
+
+	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+	if err != nil {
+		return err
+	}
+
+	for _, binfo := range buckets {
+		bucket := binfo.Name
+
+		bui, ok := dataUsageInfo.BucketsUsage[bucket]
 		if !ok {
+			// bucket doesn't exist anymore, or we
+			// do not have any information to proceed.
 			continue
 		}
+
+		// Check if the current bucket has quota restrictions, if not skip it
+		cfg, err := globalBucketQuotaSys.Get(bucket)
+		if err != nil {
+			continue
+		}
+
 		if cfg.Type != madmin.FIFOQuota {
 			continue
 		}
-		_, bucketHasLockConfig := globalBucketObjectLockConfig.Get(bucket)
 
-		dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
-		if err != nil {
-			return err
-		}
 		var toFree uint64
-		if dataUsageInfo.BucketsSizes[bucket] > cfg.Quota {
-			toFree = dataUsageInfo.BucketsSizes[bucket] - cfg.Quota
+		if bui.Size > cfg.Quota && cfg.Quota > 0 {
+			toFree = bui.Size - cfg.Quota
 		}
-		if toFree <= 0 {
+
+		if toFree == 0 {
 			continue
 		}
+
 		// Allocate new results channel to receive ObjectInfo.
 		objInfoCh := make(chan ObjectInfo)
 
@@ -218,6 +198,7 @@ func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) error {
 		if err := objectAPI.Walk(ctx, bucket, "", objInfoCh); err != nil {
 			return err
 		}
+
 		// reuse the fileScorer used by disk cache to score entries by
 		// ModTime to find the oldest objects in bucket to delete. In
 		// the context of bucket quota enforcement - number of hits are
@@ -227,9 +208,11 @@ func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) error {
 			return err
 		}
 
+		rcfg, _ := globalBucketObjectLockSys.Get(bucket)
+
 		for obj := range objInfoCh {
 			// skip objects currently under retention
-			if bucketHasLockConfig && enforceRetentionForDeletion(ctx, obj) {
+			if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
 				continue
 			}
 			scorer.addFile(obj.Name, obj.ModTime, obj.Size, 1)
@@ -238,7 +221,7 @@ func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) error {
 		numKeys := len(scorer.fileNames())
 		for i, key := range scorer.fileNames() {
 			objects = append(objects, key)
-			if len(objects) < maxObjectList && (i < numKeys-1) {
+			if len(objects) < maxDeleteList && (i < numKeys-1) {
 				// skip deletion until maxObjectList or end of slice
 				continue
 			}

cmd/common-main.go

@@ -145,7 +145,7 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 
 	// Fetch address option
 	globalCLIContext.Addr = ctx.GlobalString("address")
-	if globalCLIContext.Addr == "" || globalCLIContext.Addr == ":"+globalMinioDefaultPort {
+	if globalCLIContext.Addr == "" || globalCLIContext.Addr == ":"+GlobalMinioDefaultPort {
 		globalCLIContext.Addr = ctx.String("address")
 	}
 
@@ -186,6 +186,11 @@ func handleCommonEnvVars() {
 		logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
 	}
 
+	globalFSOSync, err = config.ParseBool(env.Get(config.EnvFSOSync, config.EnableOff))
+	if err != nil {
+		logger.Fatal(config.ErrInvalidFSOSyncValue(err), "Invalid MINIO_FS_OSYNC value in environment variable")
+	}
+
 	domains := env.Get(config.EnvDomain, "")
 	if len(domains) != 0 {
 		for _, domainName := range strings.Split(domains, config.ValueSeparator) {

cmd/config-current.go

@@ -325,13 +325,37 @@ func lookupConfigs(s config.Config) {
 
 	etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+		if globalIsGateway {
+			logger.FatalIf(err, "Unable to initialize etcd config")
+		} else {
+			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+		}
 	}
 
 	if etcdCfg.Enabled {
 		globalEtcdClient, err = etcd.New(etcdCfg)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+			if globalIsGateway {
+				logger.FatalIf(err, "Unable to initialize etcd config")
+			} else {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+			}
+		}
+		if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
+			globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
+				dns.DomainNames(globalDomainNames),
+				dns.DomainIPs(globalDomainIPs),
+				dns.DomainPort(globalMinioPort),
+				dns.CoreDNSPath(etcdCfg.CoreDNSPath),
+			)
+			if err != nil {
+				if globalIsGateway {
+					logger.FatalIf(err, "Unable to initialize DNS config")
+				} else {
+					logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
+						globalDomainNames, err))
+				}
+			}
 		}
 	}
 
@@ -342,19 +366,6 @@ func lookupConfigs(s config.Config) {
 	// but not federation.
 	globalBucketFederation = etcdCfg.PathPrefix == "" && etcdCfg.Enabled
 
-	if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
-		globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
-			dns.DomainNames(globalDomainNames),
-			dns.DomainIPs(globalDomainIPs),
-			dns.DomainPort(globalMinioPort),
-			dns.CoreDNSPath(etcdCfg.CoreDNSPath),
-		)
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
-				globalDomainNames, err))
-		}
-	}
-
 	globalServerRegion, err = config.LookupRegion(s[config.RegionSubSys][config.Default])
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Invalid region configuration: %w", err))
@@ -365,12 +376,7 @@ func lookupConfigs(s config.Config) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
 	}
 
-	apiRequestsMax := apiConfig.APIRequestsMax
-	if len(globalEndpoints.Hosts()) > 0 {
-		apiRequestsMax /= len(globalEndpoints.Hosts())
-	}
-
-	globalAPIThrottling.init(apiRequestsMax, apiConfig.APIRequestsDeadline)
+	globalAPIConfig.init(apiConfig)
 
 	if globalIsXL {
 		globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],
@@ -382,7 +388,11 @@ func lookupConfigs(s config.Config) {
 
 	globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
+		if globalIsGateway {
+			logger.FatalIf(err, "Unable to setup cache")
+		} else {
+			logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
+		}
 	}
 
 	if globalCacheConfig.Enabled {
@@ -560,6 +570,8 @@ func newServerConfig() config.Config {
 	return config.New()
 }
 
+var lookupConfigOnce sync.Once
+
 // newSrvConfig - initialize a new server config, saves env parameters if
 // found, otherwise use default parameters
 func newSrvConfig(objAPI ObjectLayer) error {
@@ -567,7 +579,9 @@ func newSrvConfig(objAPI ObjectLayer) error {
 	srvCfg := newServerConfig()
 
 	// Override any values from ENVs.
-	lookupConfigs(srvCfg)
+	lookupConfigOnce.Do(func() {
+		lookupConfigs(srvCfg)
+	})
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()
@@ -591,7 +605,9 @@ func loadConfig(objAPI ObjectLayer) error {
 	}
 
 	// Override any values from ENVs.
-	lookupConfigs(srvCfg)
+	lookupConfigOnce.Do(func() {
+		lookupConfigs(srvCfg)
+	})
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()

cmd/config.go

@@ -40,9 +40,6 @@ const (
 
 	// MinIO configuration file.
 	minioConfigFile = "config.json"
-
-	// MinIO configuration backup file
-	minioConfigBackupFile = minioConfigFile + ".backup"
 )
 
 func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData bool, count int) (
@@ -168,13 +165,14 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 		}
 	}
 
-	var config = config.New()
+	var srvCfg = config.New()
 	var json = jsoniter.ConfigCompatibleWithStandardLibrary
-	if err = json.Unmarshal(configData, &config); err != nil {
+	if err = json.Unmarshal(configData, &srvCfg); err != nil {
 		return nil, err
 	}
 
-	return config, nil
+	// Add any missing entries
+	return srvCfg.Merge(), nil
 }
 
 // ConfigSys - config system.

cmd/config/api/api.go

@@ -20,15 +20,24 @@ import (
 	"encoding/json"
 	"errors"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/env"
 )
 
+// API sub-system constants
 const (
 	apiRequestsMax      = "requests_max"
 	apiRequestsDeadline = "requests_deadline"
+	apiReadyDeadline    = "ready_deadline"
+	apiCorsAllowOrigin  = "cors_allow_origin"
+
+	EnvAPIRequestsMax      = "MINIO_API_REQUESTS_MAX"
+	EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE"
+	EnvAPIReadyDeadline    = "MINIO_API_READY_DEADLINE"
+	EnvAPICorsAllowOrigin  = "MINIO_API_CORS_ALLOW_ORIGIN"
 )
 
 // DefaultKVS - default storage class config
@@ -42,6 +51,14 @@ var (
 			Key:   apiRequestsDeadline,
 			Value: "10s",
 		},
+		config.KV{
+			Key:   apiReadyDeadline,
+			Value: "10s",
+		},
+		config.KV{
+			Key:   apiCorsAllowOrigin,
+			Value: "*",
+		},
 	}
 )
 
@@ -49,6 +66,8 @@ var (
 type Config struct {
 	APIRequestsMax      int           `json:"requests_max"`
 	APIRequestsDeadline time.Duration `json:"requests_deadline"`
+	APIReadyDeadline    time.Duration `json:"ready_deadline"`
+	APICorsAllowOrigin  []string      `json:"cors_allow_origin"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -69,7 +88,7 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	}
 
 	// Check environment variables parameters
-	requestsMax, err := strconv.Atoi(env.Get(config.EnvAPIRequestsMax, kvs.Get(apiRequestsMax)))
+	requestsMax, err := strconv.Atoi(env.Get(EnvAPIRequestsMax, kvs.Get(apiRequestsMax)))
 	if err != nil {
 		return cfg, err
 	}
@@ -78,14 +97,21 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, errors.New("invalid API max requests value")
 	}
 
-	requestsDeadline, err := time.ParseDuration(env.Get(config.EnvAPIRequestsDeadline, kvs.Get(apiRequestsDeadline)))
+	requestsDeadline, err := time.ParseDuration(env.Get(EnvAPIRequestsDeadline, kvs.Get(apiRequestsDeadline)))
 	if err != nil {
 		return cfg, err
 	}
 
-	cfg = Config{
+	readyDeadline, err := time.ParseDuration(env.Get(EnvAPIReadyDeadline, kvs.Get(apiReadyDeadline)))
+	if err != nil {
+		return cfg, err
+	}
+
+	corsAllowOrigin := strings.Split(env.Get(EnvAPICorsAllowOrigin, kvs.Get(apiCorsAllowOrigin)), ",")
+	return Config{
 		APIRequestsMax:      requestsMax,
 		APIRequestsDeadline: requestsDeadline,
-	}
-	return cfg, nil
+		APIReadyDeadline:    readyDeadline,
+		APICorsAllowOrigin:  corsAllowOrigin,
+	}, nil
 }

cmd/config/api/help.go

@@ -33,5 +33,17 @@ var (
 			Optional:    true,
 			Type:        "duration",
 		},
+		config.HelpKV{
+			Key:         apiReadyDeadline,
+			Description: `set the deadline for health check API /minio/health/ready e.g. "1m"`,
+			Optional:    true,
+			Type:        "duration",
+		},
+		config.HelpKV{
+			Key:         apiCorsAllowOrigin,
+			Description: `set comma separated list of origins allowed for CORS requests e.g. "https://example1.com,https://example2.com"`,
+			Optional:    true,
+			Type:        "csv",
+		},
 	}
 )

cmd/config/config.go

@@ -434,6 +434,26 @@ func LookupWorm() (bool, error) {
 	return ParseBool(env.Get(EnvWorm, EnableOff))
 }
 
+// Merge - merges a new config with all the
+// missing values for default configs,
+// returns a config.
+func (c Config) Merge() Config {
+	cp := New()
+	for subSys, tgtKV := range c {
+		for tgt := range tgtKV {
+			ckvs := c[subSys][tgt]
+			for _, kv := range cp[subSys][Default] {
+				_, ok := c[subSys][tgt].Lookup(kv.Key)
+				if !ok {
+					ckvs.Set(kv.Key, kv.Value)
+				}
+			}
+			cp[subSys][tgt] = ckvs
+		}
+	}
+	return cp
+}
+
 // New - initialize a new server config.
 func New() Config {
 	srvCfg := make(Config)

cmd/config/constants.go

@@ -32,10 +32,7 @@ const (
 	EnvRegionName   = "MINIO_REGION_NAME"
 	EnvPublicIPs    = "MINIO_PUBLIC_IPS"
 	EnvEndpoints    = "MINIO_ENDPOINTS"
-
-	// API sub-system
-	EnvAPIRequestsMax      = "MINIO_API_REQUESTS_MAX"
-	EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE"
+	EnvFSOSync      = "MINIO_FS_OSYNC"
 
 	EnvUpdate = "MINIO_UPDATE"
 

cmd/config/errors.go

@@ -24,6 +24,12 @@ var (
 		"Browser can only accept `on` and `off` values. To disable web browser access, set this value to `off`",
 	)
 
+	ErrInvalidFSOSyncValue = newErrFn(
+		"Invalid O_SYNC value",
+		"Please check the passed value",
+		"Can only accept `on` and `off` values. To enable O_SYNC for fs backend, set this value to `on`",
+	)
+
 	ErrInvalidDomainValue = newErrFn(
 		"Invalid domain value",
 		"Please check the passed value",
@@ -169,6 +175,18 @@ Example 1:
    $ minio server /data/minio/`,
 	)
 
+	ErrUnsupportedBackend = newErrFn(
+		"Unable to write to the backend",
+		"Please ensure your disk supports O_DIRECT",
+		"",
+	)
+
+	ErrCorruptedBackend = newErrFn(
+		"Unable to use the specified backend, pre-existing content detected",
+		"Please ensure your disk mount does not have any pre-existing content",
+		"",
+	)
+
 	ErrUnableToWriteInBackend = newErrFn(
 		"Unable to write to the backend",
 		"Please ensure MinIO binary has write permissions for the backend",

cmd/config/etcd/dns/types.go

@@ -23,7 +23,6 @@ import (
 
 const (
 	defaultTTL            = 30
-	defaultPrefixPath     = "/skydns"
 	defaultContextTimeout = 5 * time.Minute
 )
 

cmd/config/identity/ldap/config.go

@@ -58,6 +58,7 @@ type Config struct {
 	stsExpiryDuration time.Duration // contains converted value
 	tlsSkipVerify     bool          // allows skipping TLS verification
 	serverInsecure    bool          // allows plain text connection to LDAP Server
+	serverStartTLS    bool          // allows plain text connection to LDAP Server
 	rootCAs           *x509.CertPool
 }
 
@@ -73,11 +74,13 @@ const (
 	GroupSearchBaseDN    = "group_search_base_dn"
 	TLSSkipVerify        = "tls_skip_verify"
 	ServerInsecure       = "server_insecure"
+	ServerStartTLS       = "server_starttls"
 
 	EnvServerAddr           = "MINIO_IDENTITY_LDAP_SERVER_ADDR"
 	EnvSTSExpiry            = "MINIO_IDENTITY_LDAP_STS_EXPIRY"
 	EnvTLSSkipVerify        = "MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY"
 	EnvServerInsecure       = "MINIO_IDENTITY_LDAP_SERVER_INSECURE"
+	EnvServerStartTLS       = "MINIO_IDENTITY_LDAP_SERVER_STARTTLS"
 	EnvUsernameFormat       = "MINIO_IDENTITY_LDAP_USERNAME_FORMAT"
 	EnvUsernameSearchFilter = "MINIO_IDENTITY_LDAP_USERNAME_SEARCH_FILTER"
 	EnvUsernameSearchBaseDN = "MINIO_IDENTITY_LDAP_USERNAME_SEARCH_BASE_DN"
@@ -129,6 +132,10 @@ var (
 			Key:   ServerInsecure,
 			Value: config.EnableOff,
 		},
+		config.KV{
+			Key:   ServerStartTLS,
+			Value: config.EnableOff,
+		},
 	}
 )
 
@@ -257,6 +264,18 @@ func (l *Config) Connect() (ldapConn *ldap.Conn, err error) {
 		return ldap.Dial("tcp", l.ServerAddr)
 	}
 
+	if l.serverStartTLS {
+		conn, err := ldap.Dial("tcp", l.ServerAddr)
+		if err != nil {
+			return nil, err
+		}
+		err = conn.StartTLS(&tls.Config{
+			InsecureSkipVerify: l.tlsSkipVerify,
+			RootCAs:            l.rootCAs,
+		})
+		return conn, err
+	}
+
 	return ldap.DialTLS("tcp", l.ServerAddr, &tls.Config{
 		InsecureSkipVerify: l.tlsSkipVerify,
 		RootCAs:            l.rootCAs,
@@ -303,6 +322,12 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
 			return l, err
 		}
 	}
+	if v := env.Get(EnvServerStartTLS, kvs.Get(ServerStartTLS)); v != "" {
+		l.serverStartTLS, err = config.ParseBool(v)
+		if err != nil {
+			return l, err
+		}
+	}
 	if v := env.Get(EnvTLSSkipVerify, kvs.Get(TLSSkipVerify)); v != "" {
 		l.tlsSkipVerify, err = config.ParseBool(v)
 		if err != nil {

cmd/config/identity/openid/ecdsa-sha3.go

@@ -18,7 +18,9 @@ package openid
 
 import (
 	"crypto"
+
 	"github.com/dgrijalva/jwt-go"
+
 	// Needed for SHA3 to work - See: https://golang.org/src/crypto/crypto.go?s=1034:1288
 	_ "golang.org/x/crypto/sha3"
 )

cmd/config/identity/openid/jwt.go

@@ -30,7 +30,7 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/env"
-	"github.com/minio/minio/pkg/iam/policy"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	xnet "github.com/minio/minio/pkg/net"
 )
 

cmd/config/identity/openid/rsa-sha3.go

@@ -18,7 +18,9 @@ package openid
 
 import (
 	"crypto"
+
 	"github.com/dgrijalva/jwt-go"
+
 	// Needed for SHA3 to work - See: https://golang.org/src/crypto/crypto.go?s=1034:1288
 	_ "golang.org/x/crypto/sha3"
 )

cmd/config/notify/help.go

@@ -59,6 +59,18 @@ var (
 			Optional:    true,
 			Type:        "sentence",
 		},
+		config.HelpKV{
+			Key:         target.WebhookClientCert,
+			Description: "client cert for Webhook mTLS auth",
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         target.WebhookClientKey,
+			Description: "client cert key for Webhook mTLS auth",
+			Optional:    true,
+			Type:        "string",
+		},
 	}
 
 	HelpAMQP = config.HelpKVS{

cmd/config/notify/legacy.go

@@ -281,6 +281,14 @@ func SetNotifyWebhook(s config.Config, whName string, cfg target.WebhookArgs) er
 			Key:   target.WebhookQueueLimit,
 			Value: strconv.Itoa(int(cfg.QueueLimit)),
 		},
+		config.KV{
+			Key:   target.WebhookClientCert,
+			Value: cfg.ClientCert,
+		},
+		config.KV{
+			Key:   target.WebhookClientKey,
+			Value: cfg.ClientKey,
+		},
 	}
 
 	return nil

cmd/config/notify/parse.go

@@ -36,7 +36,6 @@ import (
 
 const (
 	formatNamespace = "namespace"
-	formatAccess    = "access"
 )
 
 // ErrTargetsOffline - Indicates single/multiple target failures.
@@ -1429,6 +1428,14 @@ var (
 			Key:   target.WebhookQueueDir,
 			Value: "",
 		},
+		config.KV{
+			Key:   target.WebhookClientCert,
+			Value: "",
+		},
+		config.KV{
+			Key:   target.WebhookClientKey,
+			Value: "",
+		},
 	}
 )
 
@@ -1472,6 +1479,15 @@ func GetNotifyWebhook(webhookKVS map[string]config.KVS, transport *http.Transpor
 		if k != config.Default {
 			authEnv = authEnv + config.Default + k
 		}
+		clientCertEnv := target.EnvWebhookClientCert
+		if k != config.Default {
+			clientCertEnv = clientCertEnv + config.Default + k
+		}
+
+		clientKeyEnv := target.EnvWebhookClientKey
+		if k != config.Default {
+			clientKeyEnv = clientKeyEnv + config.Default + k
+		}
 
 		webhookArgs := target.WebhookArgs{
 			Enable:     enabled,
@@ -1480,6 +1496,8 @@ func GetNotifyWebhook(webhookKVS map[string]config.KVS, transport *http.Transpor
 			AuthToken:  env.Get(authEnv, kv.Get(target.WebhookAuthToken)),
 			QueueDir:   env.Get(queueDirEnv, kv.Get(target.WebhookQueueDir)),
 			QueueLimit: uint64(queueLimit),
+			ClientCert: env.Get(clientCertEnv, kv.Get(target.WebhookClientCert)),
+			ClientKey:  env.Get(clientKeyEnv, kv.Get(target.WebhookClientKey)),
 		}
 		if err = webhookArgs.Validate(); err != nil {
 			return nil, err

cmd/crypto/kes.go

@@ -34,6 +34,10 @@ import (
 	xnet "github.com/minio/minio/pkg/net"
 )
 
+// ErrKESKeyNotFound is the error returned a KES server
+// when a master key does not exist.
+var ErrKESKeyNotFound = NewKESError(http.StatusNotFound, "key does not exist")
+
 // KesConfig contains the configuration required
 // to initialize and connect to a kes server.
 type KesConfig struct {
@@ -154,6 +158,12 @@ func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 
 	var plainKey []byte
 	plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes())
+	if err == ErrKESKeyNotFound { // Try to create the key if it does not exist.
+		if err = kes.client.CreateKey(keyID); err != nil {
+			return key, nil, err
+		}
+		plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes())
+	}
 	if err != nil {
 		return key, nil, err
 	}
@@ -214,16 +224,19 @@ type kesClient struct {
 	httpClient http.Client
 }
 
-// Response KES response struct
-type response struct {
-	Plaintext  []byte `json:"plaintext"`
-	Ciphertext []byte `json:"ciphertext,omitempty"`
-}
-
-// Request KES request struct
-type request struct {
-	Ciphertext []byte `json:"ciphertext,omitempty"`
-	Context    []byte `json:"context"`
+// CreateKey tries to create a new cryptographic key with
+// the specified name.
+//
+// The key will be generated by the server. The client
+// application does not have the cryptographic key at
+// any point in time.
+func (c *kesClient) CreateKey(name string) error {
+	url := fmt.Sprintf("%s/v1/key/create/%s", c.addr, url.PathEscape(name))
+	_, err := c.postRetry(url, nil, 0) // No request body and no response expected
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
 // GenerateDataKey requests a new data key from the KES server.
@@ -235,48 +248,155 @@ type request struct {
 // such that you have to provide the same context when decrypting
 // the data key.
 func (c *kesClient) GenerateDataKey(name string, context []byte) ([]byte, []byte, error) {
-	body, err := json.Marshal(request{
+	type Request struct {
+		Context []byte `json:"context"`
+	}
+	type Response struct {
+		Plaintext  []byte `json:"plaintext"`
+		Ciphertext []byte `json:"ciphertext"`
+	}
+
+	body, err := json.Marshal(Request{
 		Context: context,
 	})
 	if err != nil {
 		return nil, nil, err
 	}
 
-	url := fmt.Sprintf("%s/v1/key/generate/%s", c.addr, url.PathEscape(name))
-
 	const limit = 1 << 20 // A plaintext/ciphertext key pair will never be larger than 1 MB
+	url := fmt.Sprintf("%s/v1/key/generate/%s", c.addr, url.PathEscape(name))
 	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	return resp.Plaintext, resp.Ciphertext, nil
+	var response Response
+	if err = json.NewDecoder(resp).Decode(&response); err != nil {
+		return nil, nil, err
+	}
+	return response.Plaintext, response.Ciphertext, nil
 }
 
-func (c *kesClient) post(url string, body io.Reader, limit int64) (*response, error) {
-	resp, err := c.httpClient.Post(url, "application/json", body)
+// GenerateDataKey decrypts an encrypted data key with the key
+// specified by name by talking to the KES server.
+// On success, the KES server will respond with the plaintext key.
+//
+// The optional context must match the value you provided when
+// generating the data key.
+func (c *kesClient) DecryptDataKey(name string, ciphertext, context []byte) ([]byte, error) {
+	type Request struct {
+		Ciphertext []byte `json:"ciphertext"`
+		Context    []byte `json:"context,omitempty"`
+	}
+	type Response struct {
+		Plaintext []byte `json:"plaintext"`
+	}
+
+	body, err := json.Marshal(Request{
+		Ciphertext: ciphertext,
+		Context:    context,
+	})
 	if err != nil {
 		return nil, err
 	}
 
+	const limit = 1 << 20 // A data key will never be larger than 1 MiB
+	url := fmt.Sprintf("%s/v1/key/decrypt/%s", c.addr, url.PathEscape(name))
+	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
+	if err != nil {
+		return nil, err
+	}
+
+	var response Response
+	if err = json.NewDecoder(resp).Decode(&response); err != nil {
+		return nil, err
+	}
+	return response.Plaintext, nil
+}
+
+// NewKESError returns a new KES API error with the given
+// HTTP status code and error message.
+//
+// Two errors with the same status code and
+// error message are equal:
+//   e1 == e2 // true.
+func NewKESError(code int, text string) error {
+	return kesError{
+		code:    code,
+		message: text,
+	}
+}
+
+type kesError struct {
+	code    int
+	message string
+}
+
+// Status returns the HTTP status code of the error.
+func (e kesError) Status() int { return e.code }
+
+// Status returns the error message of the error.
+func (e kesError) Error() string { return e.message }
+
+func parseErrorResponse(resp *http.Response) error {
+	if resp == nil || resp.StatusCode < 400 {
+		return nil
+	}
+	if resp.Body == nil {
+		return NewKESError(resp.StatusCode, "")
+	}
+	defer resp.Body.Close()
+
+	const MaxBodySize = 1 << 20
+	var size = resp.ContentLength
+	if size < 0 || size > MaxBodySize {
+		size = MaxBodySize
+	}
+
+	contentType := strings.TrimSpace(resp.Header.Get("Content-Type"))
+	if strings.HasPrefix(contentType, "application/json") {
+		type Response struct {
+			Message string `json:"message"`
+		}
+		var response Response
+		if err := json.NewDecoder(io.LimitReader(resp.Body, size)).Decode(&response); err != nil {
+			return err
+		}
+		return NewKESError(resp.StatusCode, response.Message)
+	}
+
+	var sb strings.Builder
+	if _, err := io.Copy(&sb, io.LimitReader(resp.Body, size)); err != nil {
+		return err
+	}
+	return NewKESError(resp.StatusCode, sb.String())
+}
+
+func (c *kesClient) post(url string, body io.Reader, limit int64) (io.Reader, error) {
+	resp, err := c.httpClient.Post(url, "application/json", body)
+	if err != nil {
+		return nil, err
+	}
 	// Drain the entire body to make sure we have re-use connections
 	defer xhttp.DrainBody(resp.Body)
 
 	if resp.StatusCode != http.StatusOK {
-		return nil, c.parseErrorResponse(resp)
+		return nil, parseErrorResponse(resp)
 	}
 
-	response := &response{}
-	if err = json.NewDecoder(io.LimitReader(resp.Body, limit)).Decode(response); err != nil {
+	// We have to copy the response body due to draining.
+	var respBody bytes.Buffer
+	if _, err = io.Copy(&respBody, io.LimitReader(resp.Body, limit)); err != nil {
 		return nil, err
 	}
-	return response, nil
+	return &respBody, nil
 }
 
-func (c *kesClient) postRetry(url string, body io.ReadSeeker, limit int64) (*response, error) {
+func (c *kesClient) postRetry(url string, body io.ReadSeeker, limit int64) (io.Reader, error) {
 	for i := 0; ; i++ {
-		body.Seek(0, io.SeekStart) // seek to the beginning of the body.
-
+		if body != nil {
+			body.Seek(0, io.SeekStart) // seek to the beginning of the body.
+		}
 		response, err := c.post(url, body, limit)
 		if err == nil {
 			return response, nil
@@ -296,45 +416,6 @@ func (c *kesClient) postRetry(url string, body io.ReadSeeker, limit int64) (*res
 	}
 }
 
-// GenerateDataKey decrypts an encrypted data key with the key
-// specified by name by talking to the KES server.
-// On success, the KES server will respond with the plaintext key.
-//
-// The optional context must match the value you provided when
-// generating the data key.
-func (c *kesClient) DecryptDataKey(name string, ciphertext, context []byte) ([]byte, error) {
-	body, err := json.Marshal(request{
-		Ciphertext: ciphertext,
-		Context:    context,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	url := fmt.Sprintf("%s/v1/key/decrypt/%s", c.addr, url.PathEscape(name))
-
-	const limit = 32 * 1024 // A data key will never be larger than 32 KB
-	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
-	if err != nil {
-		return nil, err
-	}
-	return resp.Plaintext, nil
-}
-
-func (c *kesClient) parseErrorResponse(resp *http.Response) error {
-	if resp.Body == nil {
-		return Errorf("%s: no body", http.StatusText(resp.StatusCode))
-	}
-
-	const limit = 32 * 1024 // A (valid) error response will not be greater than 32 KB
-	var errMsg strings.Builder
-	if _, err := io.Copy(&errMsg, io.LimitReader(resp.Body, limit)); err != nil {
-		return Errorf("%s: %s", http.StatusText(resp.StatusCode), err)
-	}
-
-	return Errorf("%s: %s", http.StatusText(resp.StatusCode), errMsg.String())
-}
-
 // loadCACertificates returns a new CertPool
 // that contains all system root CA certificates
 // and any PEM-encoded certificate(s) found at

cmd/crypto/metadata.go

@@ -44,6 +44,8 @@ func RemoveSensitiveEntries(metadata map[string]string) { // The functions is te
 // header entries from the metadata map.
 func RemoveSSEHeaders(metadata map[string]string) {
 	delete(metadata, SSEHeader)
+	delete(metadata, SSEKmsID)
+	delete(metadata, SSEKmsContext)
 	delete(metadata, SSECKeyMD5)
 	delete(metadata, SSECAlgorithm)
 }

cmd/daily-lifecycle-ops.go

@@ -1,131 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"time"
-
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/bucket/lifecycle"
-	"github.com/minio/minio/pkg/event"
-)
-
-const (
-	bgLifecycleInterval = 24 * time.Hour
-	bgLifecycleTick     = time.Hour
-)
-
-// initDailyLifecycle starts the routine that receives the daily
-// listing of all objects and applies any matching bucket lifecycle
-// rules.
-func initDailyLifecycle(ctx context.Context, objAPI ObjectLayer) {
-	go startDailyLifecycle(ctx, objAPI)
-}
-
-func startDailyLifecycle(ctx context.Context, objAPI ObjectLayer) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.NewTimer(bgLifecycleInterval).C:
-			// Perform one lifecycle operation
-			logger.LogIf(ctx, lifecycleRound(ctx, objAPI))
-		}
-
-	}
-}
-
-func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
-	buckets, err := objAPI.ListBuckets(ctx)
-	if err != nil {
-		return err
-	}
-
-	for _, bucket := range buckets {
-		// Check if the current bucket has a configured lifecycle policy, skip otherwise
-		l, ok := globalLifecycleSys.Get(bucket.Name)
-		if !ok {
-			continue
-		}
-
-		_, bucketHasLockConfig := globalBucketObjectLockConfig.Get(bucket.Name)
-
-		// Calculate the common prefix of all lifecycle rules
-		var prefixes []string
-		for _, rule := range l.Rules {
-			prefixes = append(prefixes, rule.Prefix())
-		}
-		commonPrefix := lcp(prefixes)
-
-		// Allocate new results channel to receive ObjectInfo.
-		objInfoCh := make(chan ObjectInfo)
-
-		// Walk through all objects
-		if err := objAPI.Walk(ctx, bucket.Name, commonPrefix, objInfoCh); err != nil {
-			return err
-		}
-
-		for {
-			var objects []string
-			for obj := range objInfoCh {
-				if len(objects) == maxObjectList {
-					// Reached maximum delete requests, attempt a delete for now.
-					break
-				}
-				// Find the action that need to be executed
-				if l.ComputeAction(obj.Name, obj.UserTags, obj.ModTime) == lifecycle.DeleteAction {
-					if bucketHasLockConfig && enforceRetentionForDeletion(ctx, obj) {
-						continue
-					}
-					objects = append(objects, obj.Name)
-				}
-			}
-
-			// Nothing to do.
-			if len(objects) == 0 {
-				break
-			}
-
-			waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()))
-
-			// Deletes a list of objects.
-			deleteErrs, err := objAPI.DeleteObjects(ctx, bucket.Name, objects)
-			if err != nil {
-				logger.LogIf(ctx, err)
-			} else {
-				for i := range deleteErrs {
-					if deleteErrs[i] != nil {
-						logger.LogIf(ctx, deleteErrs[i])
-						continue
-					}
-					// Notify object deleted event.
-					sendEvent(eventArgs{
-						EventName:  event.ObjectRemovedDelete,
-						BucketName: bucket.Name,
-						Object: ObjectInfo{
-							Name: objects[i],
-						},
-						Host: "Internal: [ILM-EXPIRY]",
-					})
-				}
-			}
-		}
-	}
-
-	return nil
-}

cmd/data-crawler.go

@@ -0,0 +1,600 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"errors"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/minio/minio/cmd/config"
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/hash"
+	"github.com/willf/bloom"
+)
+
+const (
+	dataCrawlSleepPerFolder  = time.Millisecond // Time to wait between folders.
+	dataCrawlSleepDefMult    = 10.0             // Default multiplier for waits between operations.
+	dataCrawlStartDelay      = 5 * time.Minute  // Time to wait on startup and between cycles.
+	dataUsageUpdateDirCycles = 16               // Visit all folders every n cycles.
+
+)
+
+// initDataCrawler will start the crawler unless disabled.
+func initDataCrawler(ctx context.Context, objAPI ObjectLayer) {
+	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
+		go runDataCrawler(ctx, objAPI)
+	}
+}
+
+// runDataCrawler will start a data crawler.
+// The function will block until the context is canceled.
+// There should only ever be one crawler running per cluster.
+func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
+	// Load current bloom cycle
+	nextBloomCycle := intDataUpdateTracker.current() + 1
+	var buf bytes.Buffer
+	err := objAPI.GetObject(ctx, dataUsageBucket, dataUsageBloomName, 0, -1, &buf, "", ObjectOptions{})
+	if err != nil {
+		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) {
+			logger.LogIf(ctx, err)
+		}
+	} else {
+		if buf.Len() == 8 {
+			nextBloomCycle = binary.LittleEndian.Uint64(buf.Bytes())
+		}
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.NewTimer(dataCrawlStartDelay).C:
+			// Wait before starting next cycle and wait on startup.
+			results := make(chan DataUsageInfo, 1)
+			go storeDataUsageInBackend(ctx, objAPI, results)
+			bf, err := globalNotificationSys.updateBloomFilter(ctx, nextBloomCycle)
+			logger.LogIf(ctx, err)
+			err = objAPI.CrawlAndGetDataUsage(ctx, bf, results)
+			close(results)
+			logger.LogIf(ctx, err)
+			if err == nil {
+				// Store new cycle...
+				nextBloomCycle++
+				if nextBloomCycle%dataUpdateTrackerResetEvery == 0 {
+					if intDataUpdateTracker.debug {
+						logger.Info(color.Green("runDataCrawler:") + " Resetting bloom filter for next runs.")
+					}
+					nextBloomCycle++
+				}
+				var tmp [8]byte
+				binary.LittleEndian.PutUint64(tmp[:], nextBloomCycle)
+				r, err := hash.NewReader(bytes.NewReader(tmp[:]), int64(len(tmp)), "", "", int64(len(tmp)), false)
+				if err != nil {
+					logger.LogIf(ctx, err)
+					continue
+				}
+
+				_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageBloomName, NewPutObjReader(r, nil, nil), ObjectOptions{})
+				if !isErrBucketNotFound(err) {
+					logger.LogIf(ctx, err)
+				}
+			}
+		}
+	}
+}
+
+type cachedFolder struct {
+	name   string
+	parent *dataUsageHash
+}
+
+type folderScanner struct {
+	root               string
+	getSize            getSizeFn
+	oldCache           dataUsageCache
+	newCache           dataUsageCache
+	withFilter         *bloomFilter
+	waitForLowActiveIO func()
+
+	dataUsageCrawlMult  float64
+	dataUsageCrawlDebug bool
+
+	newFolders      []cachedFolder
+	existingFolders []cachedFolder
+}
+
+// crawlDataFolder will crawl the basepath+cache.Info.Name and return an updated cache.
+// The returned cache will always be valid, but may not be updated from the existing.
+// Before each operation waitForLowActiveIO is called which can be used to temporarily halt the crawler.
+// If the supplied context is canceled the function will return at the first chance.
+func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache, waitForLowActiveIO func(), getSize getSizeFn) (dataUsageCache, error) {
+	t := UTCNow()
+
+	logPrefix := color.Green("data-usage: ")
+	logSuffix := color.Blue(" - %v + %v", basePath, cache.Info.Name)
+	if intDataUpdateTracker.debug {
+		defer func() {
+			logger.Info(logPrefix+" Crawl time: %v"+logSuffix, time.Since(t))
+		}()
+
+	}
+
+	switch cache.Info.Name {
+	case "", dataUsageRoot:
+		return cache, errors.New("internal error: root scan attempted")
+	}
+
+	delayMult, err := strconv.ParseFloat(env.Get(envDataUsageCrawlDelay, "10.0"), 64)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		delayMult = dataCrawlSleepDefMult
+	}
+
+	s := folderScanner{
+		root:                basePath,
+		getSize:             getSize,
+		oldCache:            cache,
+		newCache:            dataUsageCache{Info: cache.Info},
+		waitForLowActiveIO:  waitForLowActiveIO,
+		newFolders:          nil,
+		existingFolders:     nil,
+		dataUsageCrawlMult:  delayMult,
+		dataUsageCrawlDebug: intDataUpdateTracker.debug,
+	}
+
+	if len(cache.Info.BloomFilter) > 0 {
+		s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
+		_, err := s.withFilter.ReadFrom(bytes.NewBuffer(cache.Info.BloomFilter))
+		if err != nil {
+			logger.LogIf(ctx, err, logPrefix+"Error reading bloom filter")
+			s.withFilter = nil
+		}
+	}
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Start crawling. Bloom filter: %v"+logSuffix, s.withFilter != nil)
+	}
+
+	done := ctx.Done()
+	var flattenLevels = 2
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Cycle: %v, Entries: %v"+logSuffix, cache.Info.NextCycle, len(cache.Cache))
+	}
+
+	// Always scan flattenLevels deep. Cache root is level 0.
+	todo := []cachedFolder{{name: cache.Info.Name}}
+	for i := 0; i < flattenLevels; i++ {
+		if s.dataUsageCrawlDebug {
+			logger.Info(logPrefix+"Level %v, scanning %v directories."+logSuffix, i, len(todo))
+		}
+		select {
+		case <-done:
+			return cache, ctx.Err()
+		default:
+		}
+		var err error
+		todo, err = s.scanQueuedLevels(ctx, todo, i == flattenLevels-1)
+		if err != nil {
+			// No useful information...
+			return cache, err
+		}
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"New folders: %v"+logSuffix, s.newFolders)
+	}
+
+	// Add new folders first
+	for _, folder := range s.newFolders {
+		select {
+		case <-done:
+			return s.newCache, ctx.Err()
+		default:
+		}
+		du, err := s.deepScanFolder(ctx, folder.name)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		if du == nil {
+			logger.Info(logPrefix + "no disk usage provided" + logSuffix)
+			continue
+		}
+
+		s.newCache.replace(folder.name, "", *du)
+		// Add to parent manually
+		if folder.parent != nil {
+			parent := s.newCache.Cache[folder.parent.Key()]
+			parent.addChildString(folder.name)
+		}
+	}
+
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Existing folders: %v"+logSuffix, len(s.existingFolders))
+	}
+
+	// Do selective scanning of existing folders.
+	for _, folder := range s.existingFolders {
+		select {
+		case <-done:
+			return s.newCache, ctx.Err()
+		default:
+		}
+		h := hashPath(folder.name)
+		if !h.mod(s.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
+			s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h.Key()])
+			continue
+		}
+
+		if s.withFilter != nil {
+			_, prefix := path2BucketObjectWithBasePath(basePath, folder.name)
+			if s.oldCache.Info.lifeCycle == nil || !s.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
+				// If folder isn't in filter, skip it completely.
+				if !s.withFilter.containsDir(folder.name) {
+					if s.dataUsageCrawlDebug {
+						logger.Info(logPrefix+"Skipping non-updated folder: %v"+logSuffix, folder)
+					}
+					s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h.Key()])
+					continue
+				}
+			}
+		}
+
+		// Update on this cycle...
+		du, err := s.deepScanFolder(ctx, folder.name)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		if du == nil {
+			logger.LogIf(ctx, errors.New("data-usage: no disk usage provided"))
+			continue
+		}
+		s.newCache.replaceHashed(h, folder.parent, *du)
+	}
+	if s.dataUsageCrawlDebug {
+		logger.Info(logPrefix+"Finished crawl, %v entries"+logSuffix, len(s.newCache.Cache))
+	}
+	s.newCache.Info.LastUpdate = UTCNow()
+	s.newCache.Info.NextCycle++
+	return s.newCache, nil
+}
+
+// scanQueuedLevels will scan the provided folders.
+// Files found in the folders will be added to f.newCache.
+// If final is provided folders will be put into f.newFolders or f.existingFolders.
+// If final is not provided the folders found are returned from the function.
+func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool) ([]cachedFolder, error) {
+	var nextFolders []cachedFolder
+	done := ctx.Done()
+	for _, folder := range folders {
+		select {
+		case <-done:
+			return nil, ctx.Err()
+		default:
+		}
+		thisHash := hashPath(folder.name)
+
+		// If there are lifecycle rules for the prefix, remove the filter.
+		filter := f.withFilter
+		var activeLifeCycle *lifecycle.Lifecycle
+		if f.oldCache.Info.lifeCycle != nil && filter != nil {
+			_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
+			if f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" Prefix %q has active rules", prefix)
+				}
+				activeLifeCycle = f.oldCache.Info.lifeCycle
+				filter = nil
+			}
+		}
+
+		if _, ok := f.oldCache.Cache[thisHash.Key()]; filter != nil && ok {
+			// If folder isn't in filter and we have data, skip it completely.
+			if folder.name != dataUsageRoot && !filter.containsDir(folder.name) {
+				f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" Skipping non-updated folder: %v", folder.name)
+				}
+				continue
+			}
+		}
+		f.waitForLowActiveIO()
+		sleepDuration(dataCrawlSleepPerFolder, f.dataUsageCrawlMult)
+
+		cache := dataUsageEntry{}
+
+		err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error {
+			// Parse
+			entName = path.Clean(path.Join(folder.name, entName))
+			bucket, prefix := path2BucketObjectWithBasePath(f.root, entName)
+			if bucket == "" {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" no bucket (%s,%s)", f.root, entName)
+				}
+				return nil
+			}
+
+			if isReservedOrInvalidBucket(bucket, false) {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" invalid bucket: %v, entry: %v", bucket, entName)
+				}
+				return nil
+			}
+
+			select {
+			case <-done:
+				return ctx.Err()
+			default:
+			}
+
+			if typ&os.ModeDir != 0 {
+				h := hashPath(entName)
+				_, exists := f.oldCache.Cache[h.Key()]
+				cache.addChildString(entName)
+
+				this := cachedFolder{name: entName, parent: &thisHash}
+				cache.addChild(h)
+				if final {
+					if exists {
+						f.existingFolders = append(f.existingFolders, this)
+					} else {
+						f.newFolders = append(f.newFolders, this)
+					}
+				} else {
+					nextFolders = append(nextFolders, this)
+				}
+				return nil
+			}
+			f.waitForLowActiveIO()
+			// Dynamic time delay.
+			t := UTCNow()
+
+			// Get file size, ignore errors.
+			item := crawlItem{
+				Path:       path.Join(f.root, entName),
+				Typ:        typ,
+				bucket:     bucket,
+				prefix:     path.Dir(prefix),
+				objectName: path.Base(entName),
+				debug:      f.dataUsageCrawlDebug,
+				lifeCycle:  activeLifeCycle,
+			}
+			size, err := f.getSize(item)
+
+			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+			if err == errSkipFile || err == errFileNotFound {
+				return nil
+			}
+			logger.LogIf(ctx, err)
+			cache.Size += size
+			cache.Objects++
+			cache.ObjSizes.add(size)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+		f.newCache.replaceHashed(thisHash, folder.parent, cache)
+	}
+	return nextFolders, nil
+}
+
+// deepScanFolder will deep scan a folder and return the size if no error occurs.
+func (f *folderScanner) deepScanFolder(ctx context.Context, folder string) (*dataUsageEntry, error) {
+	var cache dataUsageEntry
+
+	done := ctx.Done()
+
+	var addDir func(entName string, typ os.FileMode) error
+	var dirStack = []string{f.root, folder}
+
+	addDir = func(entName string, typ os.FileMode) error {
+		select {
+		case <-done:
+			return ctx.Err()
+		default:
+		}
+
+		f.waitForLowActiveIO()
+		if typ&os.ModeDir != 0 {
+			dirStack = append(dirStack, entName)
+			err := readDirFn(path.Join(dirStack...), addDir)
+			dirStack = dirStack[:len(dirStack)-1]
+			sleepDuration(dataCrawlSleepPerFolder, f.dataUsageCrawlMult)
+			return err
+		}
+
+		// Dynamic time delay.
+		t := UTCNow()
+
+		// Get file size, ignore errors.
+		dirStack = append(dirStack, entName)
+		fileName := path.Join(dirStack...)
+		dirStack = dirStack[:len(dirStack)-1]
+
+		bucket, prefix := path2BucketObjectWithBasePath(f.root, fileName)
+		var activeLifeCycle *lifecycle.Lifecycle
+		if f.oldCache.Info.lifeCycle != nil {
+			if f.oldCache.Info.lifeCycle.HasActiveRules(prefix, false) {
+				if f.dataUsageCrawlDebug {
+					logger.Info(color.Green("data-usage:")+" Prefix %q has active rules", prefix)
+				}
+				activeLifeCycle = f.oldCache.Info.lifeCycle
+			}
+		}
+
+		size, err := f.getSize(
+			crawlItem{
+				Path:       fileName,
+				Typ:        typ,
+				bucket:     bucket,
+				prefix:     path.Dir(prefix),
+				objectName: path.Base(entName),
+				debug:      f.dataUsageCrawlDebug,
+				lifeCycle:  activeLifeCycle,
+			})
+
+		// Don't sleep for really small amount of time
+		sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+
+		if err == errSkipFile {
+			return nil
+		}
+		logger.LogIf(ctx, err)
+		cache.Size += size
+		cache.Objects++
+		cache.ObjSizes.add(size)
+		return nil
+	}
+	err := readDirFn(path.Join(dirStack...), addDir)
+	if err != nil {
+		return nil, err
+	}
+	return &cache, nil
+}
+
+// crawlItem represents each file while walking.
+type crawlItem struct {
+	Path string
+	Typ  os.FileMode
+
+	bucket     string // Bucket.
+	prefix     string // Only the prefix if any, does not have final object name.
+	objectName string // Only the object name without prefixes.
+	lifeCycle  *lifecycle.Lifecycle
+	debug      bool
+}
+
+type getSizeFn func(item crawlItem) (int64, error)
+
+// transformMetaDir will transform a directory to prefix/file.ext
+func (i *crawlItem) transformMetaDir() {
+	split := strings.Split(i.prefix, SlashSeparator)
+	if len(split) > 1 {
+		i.prefix = path.Join(split[:len(split)-1]...)
+	} else {
+		i.prefix = ""
+	}
+	// Object name is last element
+	i.objectName = split[len(split)-1]
+}
+
+// actionMeta contains information used to apply actions.
+type actionMeta struct {
+	oi      ObjectInfo
+	trustOI bool // Set true if oi can be trusted and has been read with quorum.
+	meta    map[string]string
+}
+
+// applyActions will apply lifecycle checks on to a scanned item.
+// The resulting size on disk will always be returned.
+// The metadata will be compared to consensus on the object layer before any changes are applied.
+// If no metadata is supplied, -1 is returned if no action is taken.
+func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta actionMeta) (size int64) {
+	size, err := meta.oi.GetActualSize()
+	if i.debug {
+		logger.LogIf(ctx, err)
+	}
+	if i.lifeCycle == nil {
+		return size
+	}
+
+	action := i.lifeCycle.ComputeAction(i.objectPath(), meta.meta[xhttp.AmzObjectTagging], meta.oi.ModTime)
+	if i.debug {
+		logger.Info(color.Green("applyActions:")+" lifecycle: %q, Initial scan: %v", i.objectPath(), action)
+	}
+	switch action {
+	case lifecycle.DeleteAction:
+	default:
+		// No action.
+		return size
+	}
+
+	// These (expensive) operations should only run on items we are likely to delete.
+	// Load to ensure that we have the correct version and not an unsynced version.
+	if !meta.trustOI {
+		obj, err := o.GetObjectInfo(ctx, i.bucket, i.objectPath(), ObjectOptions{})
+		if err != nil {
+			// Do nothing - heal in the future.
+			logger.LogIf(ctx, err)
+			return size
+		}
+		size = obj.Size
+
+		// Recalculate action.
+		action = i.lifeCycle.ComputeAction(i.objectPath(), obj.UserTags, obj.ModTime)
+		if i.debug {
+			logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action)
+		}
+		switch action {
+		case lifecycle.DeleteAction:
+		default:
+			// No action.
+			return size
+		}
+	}
+
+	err = o.DeleteObject(ctx, i.bucket, i.objectPath())
+	if err != nil {
+		// Assume it is still there.
+		logger.LogIf(ctx, err)
+		return size
+	}
+
+	// Notify object deleted event.
+	sendEvent(eventArgs{
+		EventName:  event.ObjectRemovedDelete,
+		BucketName: i.bucket,
+		Object: ObjectInfo{
+			Name: i.objectPath(),
+		},
+		Host: "Internal: [ILM-EXPIRY]",
+	})
+	return 0
+}
+
+// objectPath returns the prefix and object name.
+func (i *crawlItem) objectPath() string {
+	return path.Join(i.prefix, i.objectName)
+}
+
+// sleepDuration multiplies the duration d by x and sleeps if is more than 100 micro seconds.
+// sleep is limited to max 1 second.
+func sleepDuration(d time.Duration, x float64) {
+	// Don't sleep for really small amount of time
+	if d := time.Duration(float64(d) * x); d > time.Microsecond*100 {
+		if d > time.Second {
+			d = time.Second
+		}
+		time.Sleep(d)
+	}
+}

cmd/data-update-tracker.go

@@ -46,7 +46,7 @@ const (
 	dataUpdateTrackerQueueSize = 10000
 
 	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
-	dataUpdateTrackerVersion      = 1
+	dataUpdateTrackerVersion      = 2
 	dataUpdateTrackerSaveInterval = 5 * time.Minute
 
 	// Reset bloom filters every n cycle
@@ -69,6 +69,7 @@ type dataUpdateTracker struct {
 	save       chan struct{}
 	debug      bool
 	saveExited chan struct{}
+	dirty      bool
 
 	Current dataUpdateFilter
 	History dataUpdateTrackerHistory
@@ -87,6 +88,7 @@ func newDataUpdateTracker() *dataUpdateTracker {
 		saveExited: make(chan struct{}),
 	}
 	d.Current.bf = d.newBloomFilter()
+	d.dirty = true
 	return d
 }
 
@@ -114,9 +116,7 @@ func (b bloomFilter) containsDir(in string) bool {
 	if len(split) == 0 {
 		return false
 	}
-	var tmp [dataUsageHashLen]byte
-	hashPath(path.Join(split...)).bytes(tmp[:])
-	return b.Test(tmp[:])
+	return b.TestString(hashPath(path.Join(split...)).String())
 }
 
 // bytes returns the bloom filter serialized as a byte slice.
@@ -235,11 +235,16 @@ func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Durati
 		}
 		buf.Reset()
 		d.mu.Lock()
+		if !d.dirty {
+			d.mu.Unlock()
+			return
+		}
 		d.Saved = UTCNow()
 		err := d.serialize(&buf)
 		if d.debug {
 			logger.Info(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v", buf.Len(), d.Current.idx)
 		}
+		d.dirty = false
 		d.mu.Unlock()
 		if err != nil {
 			logger.LogIf(ctx, err, "Error serializing usage tracker data")
@@ -359,6 +364,9 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 		return err
 	}
 	switch tmp[0] {
+	case 1:
+		logger.Info(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
+		return nil
 	case dataUpdateTrackerVersion:
 	default:
 		return errors.New("dataUpdateTracker: Unknown data version")
@@ -428,7 +436,6 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 // start a collector that picks up entries from objectUpdatedCh
 // and adds them  to the current bloom filter.
 func (d *dataUpdateTracker) startCollector(ctx context.Context) {
-	var tmp [dataUsageHashLen]byte
 	for {
 		select {
 		case <-ctx.Done():
@@ -456,9 +463,9 @@ func (d *dataUpdateTracker) startCollector(ctx context.Context) {
 				if d.debug && false {
 					logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
 				}
-				hashPath(path.Join(split[:i+1]...)).bytes(tmp[:])
-				d.Current.bf.Add(tmp[:])
+				d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
 			}
+			d.dirty = d.dirty || len(split) > 0
 			d.mu.Unlock()
 		}
 	}
@@ -527,7 +534,6 @@ func (d *dataUpdateTracker) filterFrom(ctx context.Context, oldest, newest uint6
 func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uint64) (*bloomFilterResponse, error) {
 	d.mu.Lock()
 	defer d.mu.Unlock()
-
 	if current == 0 {
 		if len(d.History) == 0 {
 			return d.filterFrom(ctx, d.Current.idx, d.Current.idx), nil
@@ -538,6 +544,7 @@ func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uin
 
 	// Move current to history if new one requested
 	if d.Current.idx != current {
+		d.dirty = true
 		if d.debug {
 			logger.Info(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v", d.Current.idx, current)
 		}

cmd/data-usage-cache.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
@@ -29,17 +28,17 @@ import (
 	"time"
 
 	"github.com/cespare/xxhash/v2"
+	"github.com/klauspost/compress/zstd"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/tinylib/msgp/msgp"
 )
 
-const dataUsageHashLen = 8
-
 //go:generate msgp -file $GOFILE -unexported
 
 // dataUsageHash is the hash type used.
-type dataUsageHash uint64
+type dataUsageHash string
 
 // sizeHistogram is a size histogram.
 type sizeHistogram [dataUsageBucketLen]uint64
@@ -54,6 +53,12 @@ type dataUsageEntry struct {
 	Children dataUsageHashMap
 }
 
+// dataUsageCache contains a cache of data usage entries.
+type dataUsageCache struct {
+	Info  dataUsageCacheInfo
+	Cache map[string]dataUsageEntry
+}
+
 //msgp:ignore dataUsageEntryInfo
 type dataUsageEntryInfo struct {
 	Name   string
@@ -66,7 +71,8 @@ type dataUsageCacheInfo struct {
 	Name        string
 	LastUpdate  time.Time
 	NextCycle   uint32
-	BloomFilter []byte `msg:"BloomFilter,omitempty"`
+	BloomFilter []byte               `msg:"BloomFilter,omitempty"`
+	lifeCycle   *lifecycle.Lifecycle `msg:"-"`
 }
 
 // merge other data usage entry into this, excluding children.
@@ -80,7 +86,7 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 
 // mod returns true if the hash mod cycles == cycle.
 func (h dataUsageHash) mod(cycle uint32, cycles uint32) bool {
-	return uint32(h)%cycles == cycle%cycles
+	return uint32(xxhash.Sum64String(string(h)))%cycles == cycle%cycles
 }
 
 // addChildString will add a child based on its name.
@@ -92,40 +98,110 @@ func (e *dataUsageEntry) addChildString(name string) {
 // addChild will add a child based on its hash.
 // If it already exists it will not be added again.
 func (e *dataUsageEntry) addChild(hash dataUsageHash) {
-	if _, ok := e.Children[hash]; ok {
+	if _, ok := e.Children[hash.Key()]; ok {
 		return
 	}
 	if e.Children == nil {
 		e.Children = make(dataUsageHashMap, 1)
 	}
-	e.Children[hash] = struct{}{}
+	e.Children[hash.Key()] = struct{}{}
 }
 
 // find a path in the cache.
 // Returns nil if not found.
 func (d *dataUsageCache) find(path string) *dataUsageEntry {
-	due, ok := d.Cache[hashPath(path)]
+	due, ok := d.Cache[hashPath(path).Key()]
 	if !ok {
 		return nil
 	}
 	return &due
 }
 
+// Returns nil if not found.
+func (d *dataUsageCache) subCache(path string) dataUsageCache {
+	dst := dataUsageCache{Info: dataUsageCacheInfo{
+		Name:        path,
+		LastUpdate:  d.Info.LastUpdate,
+		BloomFilter: d.Info.BloomFilter,
+	}}
+	dst.copyWithChildren(d, dataUsageHash(hashPath(path).Key()), nil)
+	return dst
+}
+
+func (d *dataUsageCache) deleteRecursive(h dataUsageHash) {
+	if existing, ok := d.Cache[h.String()]; ok {
+		// Delete first if there should be a loop.
+		delete(d.Cache, h.Key())
+		for child := range existing.Children {
+			d.deleteRecursive(dataUsageHash(child))
+		}
+	}
+}
+
+// replaceRootChild will replace the child of root in d with the root of 'other'.
+func (d *dataUsageCache) replaceRootChild(other dataUsageCache) {
+	otherRoot := other.root()
+	if otherRoot == nil {
+		logger.LogIf(GlobalContext, errors.New("replaceRootChild: Source has no root"))
+		return
+	}
+	thisRoot := d.root()
+	if thisRoot == nil {
+		logger.LogIf(GlobalContext, errors.New("replaceRootChild: Root of current not found"))
+		return
+	}
+	thisRootHash := d.rootHash()
+	otherRootHash := other.rootHash()
+	if thisRootHash == otherRootHash {
+		logger.LogIf(GlobalContext, errors.New("replaceRootChild: Root of child matches root of destination"))
+		return
+	}
+	d.deleteRecursive(other.rootHash())
+	d.copyWithChildren(&other, other.rootHash(), &thisRootHash)
+}
+
+// keepBuckets will keep only the buckets specified specified by delete all others.
+func (d *dataUsageCache) keepBuckets(b []BucketInfo) {
+	lu := make(map[dataUsageHash]struct{})
+	for _, v := range b {
+		lu[hashPath(v.Name)] = struct{}{}
+	}
+	d.keepRootChildren(lu)
+}
+
+// keepRootChildren will keep the root children specified by delete all others.
+func (d *dataUsageCache) keepRootChildren(list map[dataUsageHash]struct{}) {
+	if d.root() == nil {
+		return
+	}
+	rh := d.rootHash()
+	for k := range d.Cache {
+		h := dataUsageHash(k)
+		if h == rh {
+			continue
+		}
+		if _, ok := list[h]; !ok {
+			delete(d.Cache, k)
+			d.deleteRecursive(h)
+		}
+	}
+}
+
 // dui converts the flattened version of the path to DataUsageInfo.
 // As a side effect d will be flattened, use a clone if this is not ok.
 func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
 	e := d.find(path)
 	if e == nil {
-		return DataUsageInfo{LastUpdate: UTCNow()}
+		// No entry found, return empty.
+		return DataUsageInfo{}
 	}
 	flat := d.flatten(*e)
 	return DataUsageInfo{
-		LastUpdate:            d.Info.LastUpdate,
-		ObjectsCount:          flat.Objects,
-		ObjectsTotalSize:      uint64(flat.Size),
-		ObjectsSizesHistogram: flat.ObjSizes.asMap(),
-		BucketsCount:          uint64(len(e.Children)),
-		BucketsSizes:          d.pathSizes(buckets),
+		LastUpdate:        d.Info.LastUpdate,
+		ObjectsTotalCount: flat.Objects,
+		ObjectsTotalSize:  uint64(flat.Size),
+		BucketsCount:      uint64(len(e.Children)),
+		BucketsUsage:      d.bucketsUsageInfo(buckets),
 	}
 }
 
@@ -135,14 +211,14 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
 func (d *dataUsageCache) replace(path, parent string, e dataUsageEntry) {
 	hash := hashPath(path)
 	if d.Cache == nil {
-		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+		d.Cache = make(map[string]dataUsageEntry, 100)
 	}
-	d.Cache[hash] = e
+	d.Cache[hash.Key()] = e
 	if parent != "" {
 		phash := hashPath(parent)
-		p := d.Cache[phash]
+		p := d.Cache[phash.Key()]
 		p.addChild(hash)
-		d.Cache[phash] = p
+		d.Cache[phash.Key()] = p
 	}
 }
 
@@ -151,13 +227,13 @@ func (d *dataUsageCache) replace(path, parent string, e dataUsageEntry) {
 // If the parent does not exist, it will be added.
 func (d *dataUsageCache) replaceHashed(hash dataUsageHash, parent *dataUsageHash, e dataUsageEntry) {
 	if d.Cache == nil {
-		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+		d.Cache = make(map[string]dataUsageEntry, 100)
 	}
-	d.Cache[hash] = e
+	d.Cache[hash.Key()] = e
 	if parent != nil {
-		p := d.Cache[*parent]
+		p := d.Cache[parent.Key()]
 		p.addChild(hash)
-		d.Cache[*parent] = p
+		d.Cache[parent.Key()] = p
 	}
 }
 
@@ -166,24 +242,24 @@ func (d *dataUsageCache) replaceHashed(hash dataUsageHash, parent *dataUsageHash
 // If the parent does not exist, it will be added.
 func (d *dataUsageCache) copyWithChildren(src *dataUsageCache, hash dataUsageHash, parent *dataUsageHash) {
 	if d.Cache == nil {
-		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+		d.Cache = make(map[string]dataUsageEntry, 100)
 	}
-	e, ok := src.Cache[hash]
+	e, ok := src.Cache[hash.String()]
 	if !ok {
 		return
 	}
-	d.Cache[hash] = e
+	d.Cache[hash.Key()] = e
 	for ch := range e.Children {
-		if ch == hash {
+		if ch == hash.Key() {
 			logger.LogIf(GlobalContext, errors.New("dataUsageCache.copyWithChildren: Circular reference"))
 			return
 		}
-		d.copyWithChildren(src, ch, &hash)
+		d.copyWithChildren(src, dataUsageHash(ch), &hash)
 	}
 	if parent != nil {
-		p := d.Cache[*parent]
+		p := d.Cache[parent.Key()]
 		p.addChild(hash)
-		d.Cache[*parent] = p
+		d.Cache[parent.Key()] = p
 	}
 }
 
@@ -196,15 +272,14 @@ func (d *dataUsageCache) StringAll() string {
 	return strings.TrimSpace(s)
 }
 
-// insert the hash into dst.
-// dst must be at least dataUsageHashLen bytes long.
-func (h dataUsageHash) bytes(dst []byte) {
-	binary.LittleEndian.PutUint64(dst, uint64(h))
+// String returns a human readable representation of the string.
+func (h dataUsageHash) String() string {
+	return string(h)
 }
 
 // String returns a human readable representation of the string.
-func (h dataUsageHash) String() string {
-	return fmt.Sprintf("%x", uint64(h))
+func (h dataUsageHash) Key() string {
+	return string(h)
 }
 
 // flatten all children of the root into the root element and return it.
@@ -232,25 +307,30 @@ func (h *sizeHistogram) add(size int64) {
 	}
 }
 
-// asMap returns the map as a map[string]uint64.
-func (h *sizeHistogram) asMap() map[string]uint64 {
-	res := make(map[string]uint64, 7)
+// toMap returns the map to a map[string]uint64.
+func (h *sizeHistogram) toMap() map[string]uint64 {
+	res := make(map[string]uint64, dataUsageBucketLen)
 	for i, count := range h {
 		res[ObjectsHistogramIntervals[i].name] = count
 	}
 	return res
 }
 
-// pathSizes returns the path sizes as a map.
-func (d *dataUsageCache) pathSizes(buckets []BucketInfo) map[string]uint64 {
-	var dst = make(map[string]uint64, len(buckets))
+// bucketsUsageInfo returns the buckets usage info as a map, with
+// key as bucket name
+func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]BucketUsageInfo {
+	var dst = make(map[string]BucketUsageInfo, len(buckets))
 	for _, bucket := range buckets {
 		e := d.find(bucket.Name)
 		if e == nil {
 			continue
 		}
 		flat := d.flatten(*e)
-		dst[bucket.Name] = uint64(flat.Size)
+		dst[bucket.Name] = BucketUsageInfo{
+			Size:                 uint64(flat.Size),
+			ObjectsCount:         uint64(flat.Objects),
+			ObjectSizesHistogram: flat.ObjSizes.toMap(),
+		}
 	}
 	return dst
 }
@@ -265,13 +345,6 @@ func (d *dataUsageCache) sizeRecursive(path string) *dataUsageEntry {
 	return &flat
 }
 
-// dataUsageCache contains a cache of data usage entries.
-//msgp:ignore dataUsageCache
-type dataUsageCache struct {
-	Info  dataUsageCacheInfo
-	Cache map[dataUsageHash]dataUsageEntry
-}
-
 // root returns the root of the cache.
 func (d *dataUsageCache) root() *dataUsageEntry {
 	return d.find(d.Info.Name)
@@ -286,7 +359,7 @@ func (d *dataUsageCache) rootHash() dataUsageHash {
 func (d *dataUsageCache) clone() dataUsageCache {
 	clone := dataUsageCache{
 		Info:  d.Info,
-		Cache: make(map[dataUsageHash]dataUsageEntry, len(d.Cache)),
+		Cache: make(map[string]dataUsageEntry, len(d.Cache)),
 	}
 	for k, v := range d.Cache {
 		clone.Cache[k] = v
@@ -321,7 +394,7 @@ func (d *dataUsageCache) merge(other dataUsageCache) {
 		existing := d.Cache[key]
 		// If not found, merging simply adds.
 		existing.merge(flat)
-		d.replaceHashed(key, &eHash, existing)
+		d.replaceHashed(dataUsageHash(key), &eHash, existing)
 	}
 }
 
@@ -332,13 +405,13 @@ func (d *dataUsageCache) load(ctx context.Context, store ObjectLayer, name strin
 	var buf bytes.Buffer
 	err := store.GetObject(ctx, dataUsageBucket, name, 0, -1, &buf, "", ObjectOptions{})
 	if err != nil {
-		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) {
+		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) && !errors.Is(err, InsufficientReadQuorum{}) {
 			return toObjectErr(err, dataUsageBucket, name)
 		}
 		*d = dataUsageCache{}
 		return nil
 	}
-	err = d.deserialize(buf.Bytes())
+	err = d.deserialize(&buf)
 	if err != nil {
 		*d = dataUsageCache{}
 		logger.LogIf(ctx, err)
@@ -369,89 +442,60 @@ func (d *dataUsageCache) save(ctx context.Context, store ObjectLayer, name strin
 // dataUsageCacheVer indicates the cache version.
 // Bumping the cache version will drop data from previous versions
 // and write new data with the new version.
-const dataUsageCacheVer = 1
+const dataUsageCacheVer = 2
 
 // serialize the contents of the cache.
 func (d *dataUsageCache) serialize() []byte {
-	// Alloc pessimistically
-	// dataUsageCacheVer
-	due := dataUsageEntry{}
-	msgLen := 1
-	msgLen += d.Info.Msgsize()
-	// len(d.Cache)
-	msgLen += binary.MaxVarintLen64
-	// Hashes (one for key, assume 1 child/node)
-	msgLen += len(d.Cache) * dataUsageHashLen * 2
-	msgLen += len(d.Cache) * due.Msgsize()
-
-	// Create destination buffer...
-	dst := make([]byte, 0, msgLen)
-
-	var n int
-	tmp := make([]byte, 1024)
-	// byte: version.
+	// Prepend version and compress.
+	dst := make([]byte, 0, d.Msgsize()+1)
 	dst = append(dst, dataUsageCacheVer)
-	// Info...
-	dst, err := d.Info.MarshalMsg(dst)
+	buf := bytes.NewBuffer(dst)
+	enc, err := zstd.NewWriter(buf,
+		zstd.WithEncoderLevel(zstd.SpeedFastest),
+		zstd.WithWindowSize(1<<20),
+		zstd.WithEncoderConcurrency(2))
 	if err != nil {
-		panic(err)
+		logger.LogIf(GlobalContext, err)
+		return nil
 	}
-	n = binary.PutUvarint(tmp, uint64(len(d.Cache)))
-	dst = append(dst, tmp[:n]...)
-
-	for k, v := range d.Cache {
-		// Put key
-		binary.LittleEndian.PutUint64(tmp[:dataUsageHashLen], uint64(k))
-		dst = append(dst, tmp[:8]...)
-		tmp, err = v.MarshalMsg(tmp[:0])
-		if err != nil {
-			panic(err)
-		}
-		// key, value pairs.
-		dst = append(dst, tmp...)
-
+	mEnc := msgp.NewWriter(enc)
+	err = d.EncodeMsg(mEnc)
+	if err != nil {
+		logger.LogIf(GlobalContext, err)
+		return nil
 	}
-	return dst
+	mEnc.Flush()
+	err = enc.Close()
+	if err != nil {
+		logger.LogIf(GlobalContext, err)
+		return nil
+	}
+	return buf.Bytes()
 }
 
 // deserialize the supplied byte slice into the cache.
-func (d *dataUsageCache) deserialize(b []byte) error {
-	if len(b) < 1 {
+func (d *dataUsageCache) deserialize(r io.Reader) error {
+	var b [1]byte
+	n, _ := r.Read(b[:])
+	if n != 1 {
 		return io.ErrUnexpectedEOF
 	}
 	switch b[0] {
 	case 1:
+		return errors.New("cache version deprecated (will autoupdate)")
+	case dataUsageCacheVer:
 	default:
 		return fmt.Errorf("dataUsageCache: unknown version: %d", int(b[0]))
 	}
-	b = b[1:]
 
-	// Info...
-	b, err := d.Info.UnmarshalMsg(b)
+	// Zstd compressed.
+	dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
 	if err != nil {
 		return err
 	}
-	cacheLen, n := binary.Uvarint(b)
-	if n <= 0 {
-		return fmt.Errorf("dataUsageCache: reading cachelen, n <= 0 ")
-	}
-	b = b[n:]
-	d.Cache = make(map[dataUsageHash]dataUsageEntry, cacheLen)
+	defer dec.Close()
 
-	for i := 0; i < int(cacheLen); i++ {
-		if len(b) <= dataUsageHashLen {
-			return io.ErrUnexpectedEOF
-		}
-		k := binary.LittleEndian.Uint64(b[:dataUsageHashLen])
-		b = b[dataUsageHashLen:]
-		var v dataUsageEntry
-		b, err = v.UnmarshalMsg(b)
-		if err != nil {
-			return err
-		}
-		d.Cache[dataUsageHash(k)] = v
-	}
-	return nil
+	return d.DecodeMsg(msgp.NewReader(dec))
 }
 
 // Trim this from start+end of hashes.
@@ -468,88 +512,91 @@ func hashPath(data string) dataUsageHash {
 	if data != dataUsageRoot {
 		data = strings.Trim(data, hashPathCutSet)
 	}
-	data = path.Clean(data)
-	return dataUsageHash(xxhash.Sum64String(data))
+	return dataUsageHash(path.Clean(data))
 }
 
-//msgp:ignore dataUsageEntryInfo
-type dataUsageHashMap map[dataUsageHash]struct{}
+//msgp:ignore dataUsageHashMap
+type dataUsageHashMap map[string]struct{}
 
-// MarshalMsg implements msgp.Marshaler
-func (d dataUsageHashMap) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, d.Msgsize())
-
-	// Write bin header manually
-	const mbin32 uint8 = 0xc6
-	sz := uint32(len(d)) * dataUsageHashLen
-	o = append(o, mbin32, byte(sz>>24), byte(sz>>16), byte(sz>>8), byte(sz))
-
-	var tmp [dataUsageHashLen]byte
-	for k := range d {
-		binary.LittleEndian.PutUint64(tmp[:], uint64(k))
-		o = append(o, tmp[:]...)
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageHashMap) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0002 uint32
+	zb0002, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	*z = make(dataUsageHashMap, zb0002)
+	for i := uint32(0); i < zb0002; i++ {
+		{
+			var zb0003 string
+			zb0003, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+			(*z)[zb0003] = struct{}{}
+		}
 	}
 	return
 }
 
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (d dataUsageHashMap) Msgsize() (s int) {
-	s = 5 + len(d)*dataUsageHashLen
+// EncodeMsg implements msgp.Encodable
+func (z dataUsageHashMap) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteArrayHeader(uint32(len(z)))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0004 := range z {
+		err = en.WriteString(zb0004)
+		if err != nil {
+			err = msgp.WrapError(err, zb0004)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z dataUsageHashMap) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendArrayHeader(o, uint32(len(z)))
+	for zb0004 := range z {
+		o = msgp.AppendString(o, zb0004)
+	}
 	return
 }
 
 // UnmarshalMsg implements msgp.Unmarshaler
-func (d *dataUsageHashMap) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var hashes []byte
-	hashes, bts, err = msgp.ReadBytesZC(bts)
+func (z *dataUsageHashMap) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0002 uint32
+	zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
 	if err != nil {
-		err = msgp.WrapError(err, "dataUsageHashMap")
+		err = msgp.WrapError(err)
 		return
 	}
-
-	var dst = make(dataUsageHashMap, len(hashes)/dataUsageHashLen)
-	for len(hashes) >= dataUsageHashLen {
-		dst[dataUsageHash(binary.LittleEndian.Uint64(hashes[:dataUsageHashLen]))] = struct{}{}
-		hashes = hashes[dataUsageHashLen:]
+	*z = make(dataUsageHashMap, zb0002)
+	for i := uint32(0); i < zb0002; i++ {
+		{
+			var zb0003 string
+			zb0003, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+			(*z)[zb0003] = struct{}{}
+		}
 	}
-	*d = dst
 	o = bts
 	return
 }
 
-func (d *dataUsageHashMap) DecodeMsg(dc *msgp.Reader) (err error) {
-	var zb0001 uint32
-	zb0001, err = dc.ReadBytesHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z dataUsageHashMap) Msgsize() (s int) {
+	s = msgp.ArrayHeaderSize
+	for zb0004 := range z {
+		s += msgp.StringPrefixSize + len(zb0004)
 	}
-	var dst = make(dataUsageHashMap, zb0001)
-	var tmp [8]byte
-	for i := uint32(0); i < zb0001; i++ {
-		_, err = io.ReadFull(dc, tmp[:])
-		if err != nil {
-			err = msgp.WrapError(err, "dataUsageHashMap")
-			return
-		}
-		dst[dataUsageHash(binary.LittleEndian.Uint64(tmp[:]))] = struct{}{}
-	}
-	return nil
-}
-func (d dataUsageHashMap) EncodeMsg(en *msgp.Writer) (err error) {
-	err = en.WriteBytesHeader(uint32(len(d)) * dataUsageHashLen)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	var tmp [dataUsageHashLen]byte
-	for k := range d {
-		binary.LittleEndian.PutUint64(tmp[:], uint64(k))
-		_, err = en.Write(tmp[:])
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-	}
-	return nil
+	return
 }

cmd/data-usage-cache_gen.go

@@ -6,6 +6,212 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageCache) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Info":
+			err = z.Info.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "Info")
+				return
+			}
+		case "Cache":
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Cache")
+				return
+			}
+			if z.Cache == nil {
+				z.Cache = make(map[string]dataUsageEntry, zb0002)
+			} else if len(z.Cache) > 0 {
+				for key := range z.Cache {
+					delete(z.Cache, key)
+				}
+			}
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 dataUsageEntry
+				za0001, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Cache")
+					return
+				}
+				err = za0002.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache", za0001)
+					return
+				}
+				z.Cache[za0001] = za0002
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *dataUsageCache) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "Info"
+	err = en.Append(0x82, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
+	if err != nil {
+		return
+	}
+	err = z.Info.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Info")
+		return
+	}
+	// write "Cache"
+	err = en.Append(0xa5, 0x43, 0x61, 0x63, 0x68, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteMapHeader(uint32(len(z.Cache)))
+	if err != nil {
+		err = msgp.WrapError(err, "Cache")
+		return
+	}
+	for za0001, za0002 := range z.Cache {
+		err = en.WriteString(za0001)
+		if err != nil {
+			err = msgp.WrapError(err, "Cache")
+			return
+		}
+		err = za0002.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Cache", za0001)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *dataUsageCache) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "Info"
+	o = append(o, 0x82, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
+	o, err = z.Info.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Info")
+		return
+	}
+	// string "Cache"
+	o = append(o, 0xa5, 0x43, 0x61, 0x63, 0x68, 0x65)
+	o = msgp.AppendMapHeader(o, uint32(len(z.Cache)))
+	for za0001, za0002 := range z.Cache {
+		o = msgp.AppendString(o, za0001)
+		o, err = za0002.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Cache", za0001)
+			return
+		}
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageCache) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Info":
+			bts, err = z.Info.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Info")
+				return
+			}
+		case "Cache":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Cache")
+				return
+			}
+			if z.Cache == nil {
+				z.Cache = make(map[string]dataUsageEntry, zb0002)
+			} else if len(z.Cache) > 0 {
+				for key := range z.Cache {
+					delete(z.Cache, key)
+				}
+			}
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 dataUsageEntry
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache")
+					return
+				}
+				bts, err = za0002.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache", za0001)
+					return
+				}
+				z.Cache[za0001] = za0002
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *dataUsageCache) Msgsize() (s int) {
+	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize
+	if z.Cache != nil {
+		for za0001, za0002 := range z.Cache {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
+		}
+	}
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
@@ -373,8 +579,8 @@ func (z *dataUsageEntry) Msgsize() (s int) {
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
 	{
-		var zb0001 uint64
-		zb0001, err = dc.ReadUint64()
+		var zb0001 string
+		zb0001, err = dc.ReadString()
 		if err != nil {
 			err = msgp.WrapError(err)
 			return
@@ -386,7 +592,7 @@ func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z dataUsageHash) EncodeMsg(en *msgp.Writer) (err error) {
-	err = en.WriteUint64(uint64(z))
+	err = en.WriteString(string(z))
 	if err != nil {
 		err = msgp.WrapError(err)
 		return
@@ -397,15 +603,15 @@ func (z dataUsageHash) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z dataUsageHash) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	o = msgp.AppendUint64(o, uint64(z))
+	o = msgp.AppendString(o, string(z))
 	return
 }
 
 // UnmarshalMsg implements msgp.Unmarshaler
 func (z *dataUsageHash) UnmarshalMsg(bts []byte) (o []byte, err error) {
 	{
-		var zb0001 uint64
-		zb0001, bts, err = msgp.ReadUint64Bytes(bts)
+		var zb0001 string
+		zb0001, bts, err = msgp.ReadStringBytes(bts)
 		if err != nil {
 			err = msgp.WrapError(err)
 			return
@@ -418,7 +624,7 @@ func (z *dataUsageHash) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z dataUsageHash) Msgsize() (s int) {
-	s = msgp.Uint64Size
+	s = msgp.StringPrefixSize + len(string(z))
 	return
 }
 

cmd/data-usage-cache_gen_test.go

@@ -9,6 +9,119 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
+func TestMarshalUnmarshaldataUsageCache(t *testing.T) {
+	v := dataUsageCache{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgdataUsageCache(b *testing.B) {
+	v := dataUsageCache{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgdataUsageCache(b *testing.B) {
+	v := dataUsageCache{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshaldataUsageCache(b *testing.B) {
+	v := dataUsageCache{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodedataUsageCache(t *testing.T) {
+	v := dataUsageCache{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodedataUsageCache Msgsize() is inaccurate")
+	}
+
+	vn := dataUsageCache{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodedataUsageCache(b *testing.B) {
+	v := dataUsageCache{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodedataUsageCache(b *testing.B) {
+	v := dataUsageCache{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshaldataUsageCacheInfo(t *testing.T) {
 	v := dataUsageCacheInfo{}
 	bts, err := v.MarshalMsg(nil)

cmd/data-usage.go

@@ -19,21 +19,11 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"encoding/binary"
 	"encoding/json"
-	"errors"
-	"os"
-	"path"
-	"strconv"
-	"time"
 
 	jsoniter "github.com/json-iterator/go"
-	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/color"
-	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/hash"
-	"github.com/willf/bloom"
 )
 
 const (
@@ -44,76 +34,11 @@ const (
 	dataUsageRoot   = SlashSeparator
 	dataUsageBucket = minioMetaBucket + SlashSeparator + bucketMetaPrefix
 
-	dataUsageObjName         = ".usage.json"
-	dataUsageCacheName       = ".usage-cache.bin"
-	dataUsageBloomName       = ".bloomcycle.bin"
-	dataUsageSleepPerFolder  = 1 * time.Millisecond
-	dataUsageSleepDefMult    = 10.0
-	dataUsageUpdateDirCycles = 16
-	dataUsageStartDelay      = 5 * time.Minute // Time to wait on startup and between cycles.
+	dataUsageObjName   = ".usage.json"
+	dataUsageCacheName = ".usage-cache.bin"
+	dataUsageBloomName = ".bloomcycle.bin"
 )
 
-// initDataUsageStats will start the crawler unless disabled.
-func initDataUsageStats(ctx context.Context, objAPI ObjectLayer) {
-	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
-		go runDataUsageInfo(ctx, objAPI)
-	}
-}
-
-func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer) {
-	// Load current bloom cycle
-	nextBloomCycle := intDataUpdateTracker.current() + 1
-	var buf bytes.Buffer
-	err := objAPI.GetObject(ctx, dataUsageBucket, dataUsageBloomName, 0, -1, &buf, "", ObjectOptions{})
-	if err != nil {
-		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) {
-			logger.LogIf(ctx, err)
-		}
-	} else {
-		if buf.Len() == 8 {
-			nextBloomCycle = binary.LittleEndian.Uint64(buf.Bytes())
-		}
-	}
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.NewTimer(dataUsageStartDelay).C:
-			// Wait before starting next cycle and wait on startup.
-			results := make(chan DataUsageInfo, 1)
-			go storeDataUsageInBackend(ctx, objAPI, results)
-			bf, err := globalNotificationSys.updateBloomFilter(ctx, nextBloomCycle)
-			logger.LogIf(ctx, err)
-			err = objAPI.CrawlAndGetDataUsage(ctx, bf, results)
-			close(results)
-			logger.LogIf(ctx, err)
-			if err == nil {
-				// Store new cycle...
-				nextBloomCycle++
-				if nextBloomCycle%dataUpdateTrackerResetEvery == 0 {
-					if intDataUpdateTracker.debug {
-						logger.Info(color.Green("runDataUsageInfo:") + " Resetting bloom filter for next runs.")
-					}
-					nextBloomCycle++
-				}
-				var tmp [8]byte
-				binary.LittleEndian.PutUint64(tmp[:], nextBloomCycle)
-				r, err := hash.NewReader(bytes.NewReader(tmp[:]), int64(len(tmp)), "", "", int64(len(tmp)), false)
-				if err != nil {
-					logger.LogIf(ctx, err)
-					continue
-				}
-
-				_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageBloomName, NewPutObjReader(r, nil, nil), ObjectOptions{})
-				if !isErrBucketNotFound(err) {
-					logger.LogIf(ctx, err)
-				}
-			}
-		}
-	}
-}
-
 // storeDataUsageInBackend will store all objects sent on the gui channel until closed.
 func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, gui <-chan DataUsageInfo) {
 	for dataUsageInfo := range gui {
@@ -154,356 +79,21 @@ func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsag
 		return DataUsageInfo{}, err
 	}
 
+	// For forward compatibility reasons, we need to add this code.
+	if len(dataUsageInfo.BucketsUsage) == 0 {
+		dataUsageInfo.BucketsUsage = make(map[string]BucketUsageInfo, len(dataUsageInfo.BucketSizes))
+		for bucket, size := range dataUsageInfo.BucketSizes {
+			dataUsageInfo.BucketsUsage[bucket] = BucketUsageInfo{Size: size}
+		}
+	}
+
+	// For backward compatibility reasons, we need to add this code.
+	if len(dataUsageInfo.BucketSizes) == 0 {
+		dataUsageInfo.BucketSizes = make(map[string]uint64, len(dataUsageInfo.BucketsUsage))
+		for bucket, bui := range dataUsageInfo.BucketsUsage {
+			dataUsageInfo.BucketSizes[bucket] = bui.Size
+		}
+	}
+
 	return dataUsageInfo, nil
 }
-
-// Item represents each file while walking.
-type Item struct {
-	Path string
-	Typ  os.FileMode
-}
-
-type getSizeFn func(item Item) (int64, error)
-
-type cachedFolder struct {
-	name   string
-	parent *dataUsageHash
-}
-
-type folderScanner struct {
-	root               string
-	getSize            getSizeFn
-	oldCache           dataUsageCache
-	newCache           dataUsageCache
-	withFilter         *bloomFilter
-	waitForLowActiveIO func()
-
-	dataUsageCrawlMult  float64
-	dataUsageCrawlDebug bool
-
-	newFolders      []cachedFolder
-	existingFolders []cachedFolder
-}
-
-// sleepDuration multiplies the duration d by x and sleeps if is more than 100 micro seconds.
-// sleep is limited to max 1 second.
-func sleepDuration(d time.Duration, x float64) {
-	// Don't sleep for really small amount of time
-	if d := time.Duration(float64(d) * x); d > time.Microsecond*100 {
-		if d > time.Second {
-			d = time.Second
-		}
-		time.Sleep(d)
-	}
-}
-
-// scanQueuedLevels will scan the provided folders.
-// Files found in the folders will be added to f.newCache.
-// If final is provided folders will be put into f.newFolders or f.existingFolders.
-// If final is not provided the folders found are returned from the function.
-func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool) ([]cachedFolder, error) {
-	var nextFolders []cachedFolder
-	done := ctx.Done()
-	for _, folder := range folders {
-		select {
-		case <-done:
-			return nil, ctx.Err()
-		default:
-		}
-		thisHash := hashPath(folder.name)
-
-		if _, ok := f.oldCache.Cache[thisHash]; f.withFilter != nil && ok {
-			// If folder isn't in filter and we have data, skip it completely.
-			if folder.name != dataUsageRoot && !f.withFilter.containsDir(folder.name) {
-				f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
-				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("data-usage:")+" Skipping non-updated folder: %v", folder.name)
-				}
-				continue
-			}
-		}
-		f.waitForLowActiveIO()
-		sleepDuration(dataUsageSleepPerFolder, f.dataUsageCrawlMult)
-
-		cache := dataUsageEntry{}
-
-		err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error {
-			// Parse
-			entName = path.Clean(path.Join(folder.name, entName))
-			bucket, _ := path2BucketObjectWithBasePath(f.root, entName)
-			if bucket == "" {
-				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("data-usage:")+" no bucket (%s,%s)", f.root, entName)
-				}
-				return nil
-			}
-
-			if isReservedOrInvalidBucket(bucket, false) {
-				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("data-usage:")+" invalid bucket: %v, entry: %v", bucket, entName)
-				}
-				return nil
-			}
-
-			select {
-			case <-done:
-				return ctx.Err()
-			default:
-			}
-
-			if typ&os.ModeDir != 0 {
-				h := hashPath(entName)
-				_, exists := f.oldCache.Cache[h]
-				cache.addChildString(entName)
-
-				this := cachedFolder{name: entName, parent: &thisHash}
-				cache.addChild(h)
-				if final {
-					if exists {
-						f.existingFolders = append(f.existingFolders, this)
-					} else {
-						f.newFolders = append(f.newFolders, this)
-					}
-				} else {
-					nextFolders = append(nextFolders, this)
-				}
-				return nil
-			}
-			f.waitForLowActiveIO()
-			// Dynamic time delay.
-			t := UTCNow()
-
-			// Get file size, ignore errors.
-			size, err := f.getSize(Item{Path: path.Join(f.root, entName), Typ: typ})
-
-			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
-			if err == errSkipFile {
-				return nil
-			}
-			logger.LogIf(ctx, err)
-			cache.Size += size
-			cache.Objects++
-			cache.ObjSizes.add(size)
-
-			return nil
-		})
-		if err != nil {
-			return nil, err
-		}
-		f.newCache.replaceHashed(thisHash, folder.parent, cache)
-	}
-	return nextFolders, nil
-}
-
-// deepScanFolder will deep scan a folder and return the size if no error occurs.
-func (f *folderScanner) deepScanFolder(ctx context.Context, folder string) (*dataUsageEntry, error) {
-	var cache dataUsageEntry
-
-	done := ctx.Done()
-
-	var addDir func(entName string, typ os.FileMode) error
-	var dirStack = []string{f.root, folder}
-
-	addDir = func(entName string, typ os.FileMode) error {
-		select {
-		case <-done:
-			return ctx.Err()
-		default:
-		}
-
-		f.waitForLowActiveIO()
-		if typ&os.ModeDir != 0 {
-			dirStack = append(dirStack, entName)
-			err := readDirFn(path.Join(dirStack...), addDir)
-			dirStack = dirStack[:len(dirStack)-1]
-			sleepDuration(dataUsageSleepPerFolder, f.dataUsageCrawlMult)
-			return err
-		}
-
-		// Dynamic time delay.
-		t := UTCNow()
-
-		// Get file size, ignore errors.
-		dirStack = append(dirStack, entName)
-		fileName := path.Join(dirStack...)
-		dirStack = dirStack[:len(dirStack)-1]
-
-		size, err := f.getSize(Item{Path: fileName, Typ: typ})
-
-		// Don't sleep for really small amount of time
-		sleepDuration(time.Since(t), f.dataUsageCrawlMult)
-
-		if err == errSkipFile {
-			return nil
-		}
-		logger.LogIf(ctx, err)
-		cache.Size += size
-		cache.Objects++
-		cache.ObjSizes.add(size)
-		return nil
-	}
-	err := readDirFn(path.Join(dirStack...), addDir)
-	if err != nil {
-		return nil, err
-	}
-	return &cache, nil
-}
-
-// updateUsage will crawl the basepath+cache.Info.Name and return an updated cache.
-// The returned cache will always be valid, but may not be updated from the existing.
-// Before each operation waitForLowActiveIO is called which can be used to temporarily halt the crawler.
-// If the supplied context is canceled the function will return at the first chance.
-func updateUsage(ctx context.Context, basePath string, cache dataUsageCache, waitForLowActiveIO func(), getSize getSizeFn) (dataUsageCache, error) {
-	t := UTCNow()
-
-	dataUsageDebug := env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn
-	logPrefix := color.Green("data-usage: ")
-	logSuffix := color.Blue(" - %v + %v", basePath, cache.Info.Name)
-	if dataUsageDebug {
-		defer func() {
-			logger.Info(logPrefix+" Crawl time: %v"+logSuffix, time.Since(t))
-		}()
-
-	}
-
-	if cache.Info.Name == "" {
-		cache.Info.Name = dataUsageRoot
-	}
-
-	delayMult, err := strconv.ParseFloat(env.Get(envDataUsageCrawlDelay, "10.0"), 64)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		delayMult = dataUsageSleepDefMult
-	}
-
-	s := folderScanner{
-		root:                basePath,
-		getSize:             getSize,
-		oldCache:            cache,
-		newCache:            dataUsageCache{Info: cache.Info},
-		waitForLowActiveIO:  waitForLowActiveIO,
-		newFolders:          nil,
-		existingFolders:     nil,
-		dataUsageCrawlMult:  delayMult,
-		dataUsageCrawlDebug: dataUsageDebug,
-	}
-
-	if len(cache.Info.BloomFilter) > 0 {
-		s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
-		_, err := s.withFilter.ReadFrom(bytes.NewBuffer(cache.Info.BloomFilter))
-		if err != nil {
-			logger.LogIf(ctx, err, logPrefix+"Error reading bloom filter")
-			s.withFilter = nil
-		}
-	}
-	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Start crawling. Bloom filter: %v"+logSuffix, s.withFilter != nil)
-	}
-
-	done := ctx.Done()
-	var flattenLevels = 3
-
-	// If we are scanning inside a bucket reduce depth by 1.
-	if cache.Info.Name != dataUsageRoot {
-		flattenLevels--
-	}
-
-	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Cycle: %v, Entries: %v"+logSuffix, cache.Info.NextCycle, len(cache.Cache))
-	}
-
-	// Always scan flattenLevels deep. Cache root is level 0.
-	todo := []cachedFolder{{name: cache.Info.Name}}
-	for i := 0; i < flattenLevels; i++ {
-		if s.dataUsageCrawlDebug {
-			logger.Info(logPrefix+"Level %v, scanning %v directories."+logSuffix, i, len(todo))
-		}
-		select {
-		case <-done:
-			return cache, ctx.Err()
-		default:
-		}
-		var err error
-		todo, err = s.scanQueuedLevels(ctx, todo, i == flattenLevels-1)
-		if err != nil {
-			// No useful information...
-			return cache, err
-		}
-	}
-
-	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"New folders: %v"+logSuffix, s.newFolders)
-	}
-
-	// Add new folders first
-	for _, folder := range s.newFolders {
-		select {
-		case <-done:
-			return s.newCache, ctx.Err()
-		default:
-		}
-		du, err := s.deepScanFolder(ctx, folder.name)
-		if err != nil {
-			logger.LogIf(ctx, err)
-			continue
-		}
-		if du == nil {
-			logger.Info(logPrefix + "no disk usage provided" + logSuffix)
-			continue
-		}
-
-		s.newCache.replace(folder.name, "", *du)
-		// Add to parent manually
-		if folder.parent != nil {
-			parent := s.newCache.Cache[*folder.parent]
-			parent.addChildString(folder.name)
-		}
-	}
-
-	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Existing folders: %v"+logSuffix, len(s.existingFolders))
-	}
-
-	// Do selective scanning of existing folders.
-	for _, folder := range s.existingFolders {
-		select {
-		case <-done:
-			return s.newCache, ctx.Err()
-		default:
-		}
-		h := hashPath(folder.name)
-		if !h.mod(s.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
-			s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h])
-			continue
-		}
-
-		if s.withFilter != nil {
-			// If folder isn't in filter, skip it completely.
-			if !s.withFilter.containsDir(folder.name) {
-				if s.dataUsageCrawlDebug {
-					logger.Info(logPrefix+"Skipping non-updated folder: %v"+logSuffix, folder)
-				}
-				s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h])
-				continue
-			}
-		}
-
-		// Update on this cycle...
-		du, err := s.deepScanFolder(ctx, folder.name)
-		if err != nil {
-			logger.LogIf(ctx, err)
-			continue
-		}
-		if du == nil {
-			logger.LogIf(ctx, errors.New("data-usage: no disk usage provided"))
-			continue
-		}
-		s.newCache.replaceHashed(h, folder.parent, *du)
-	}
-	if s.dataUsageCrawlDebug {
-		logger.Info(logPrefix+"Finished crawl, %v entries"+logSuffix, len(s.newCache.Cache))
-	}
-	s.newCache.Info.LastUpdate = UTCNow()
-	s.newCache.Info.NextCycle++
-	return s.newCache, nil
-}

cmd/data-usage_test.go

@@ -17,9 +17,12 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"testing"
 )
@@ -34,6 +37,7 @@ func TestDataUsageUpdate(t *testing.T) {
 	if err != nil {
 		t.Skip(err)
 	}
+	const bucket = "bucket"
 	defer os.RemoveAll(base)
 	var files = []usageTestFile{
 		{name: "rootfile", size: 10000},
@@ -45,9 +49,9 @@ func TestDataUsageUpdate(t *testing.T) {
 		{name: "dir1/dira/dirasub/dcfile", size: 1000000},
 		{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
 	}
-	createUsageTestFiles(t, base, files)
+	createUsageTestFiles(t, base, bucket, files)
 
-	getSize := func(item Item) (i int64, err error) {
+	getSize := func(item crawlItem) (i int64, err error) {
 		if item.Typ&os.ModeDir == 0 {
 			s, err := os.Stat(item.Path)
 			if err != nil {
@@ -58,7 +62,7 @@ func TestDataUsageUpdate(t *testing.T) {
 		return 0, nil
 	}
 
-	got, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize)
+	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, func() {}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -105,16 +109,17 @@ func TestDataUsageUpdate(t *testing.T) {
 			oSizes:  sizeHistogram{0: 1, 1: 3},
 		},
 		{
-			path:   "/dir1/dira",
-			size:   300000,
-			objs:   2,
-			oSizes: sizeHistogram{0: 0, 1: 2},
+			path:   "/dir1",
+			size:   2000,
+			objs:   1,
+			oSizes: sizeHistogram{0: 0, 1: 1},
 		},
 		{
+			// Children are flattened
 			path:   "/dir1/dira/",
-			size:   300000,
-			objs:   2,
-			oSizes: sizeHistogram{0: 0, 1: 2},
+			size:   1300010,
+			objs:   4,
+			oSizes: sizeHistogram{0: 1, 1: 3},
 		},
 		{
 			path:  "/nonexistying",
@@ -123,8 +128,9 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 
 	for _, w := range want {
-		t.Run(w.path, func(t *testing.T) {
-			e := got.find(w.path)
+		p := path.Join(bucket, w.path)
+		t.Run(p, func(t *testing.T) {
+			e := got.find(p)
 			if w.isNil {
 				if e != nil {
 					t.Error("want nil, got", e)
@@ -134,6 +140,7 @@ func TestDataUsageUpdate(t *testing.T) {
 			if e == nil {
 				t.Fatal("got nil result")
 			}
+			t.Log(e.Children)
 			if w.flatten {
 				*e = got.flatten(*e)
 			}
@@ -175,8 +182,8 @@ func TestDataUsageUpdate(t *testing.T) {
 			size: 1000,
 		},
 	}
-	createUsageTestFiles(t, base, files)
-	got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+	createUsageTestFiles(t, base, bucket, files)
+	got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -222,7 +229,7 @@ func TestDataUsageUpdate(t *testing.T) {
 
 	for _, w := range want {
 		t.Run(w.path, func(t *testing.T) {
-			e := got.find(w.path)
+			e := got.find(path.Join(bucket, w.path))
 			if w.isNil {
 				if e != nil {
 					t.Error("want nil, got", e)
@@ -233,6 +240,7 @@ func TestDataUsageUpdate(t *testing.T) {
 				t.Fatal("got nil result")
 			}
 			if w.flatten {
+				t.Log(e.Children)
 				*e = got.flatten(*e)
 			}
 			if e.Size != int64(w.size) {
@@ -254,14 +262,14 @@ func TestDataUsageUpdate(t *testing.T) {
 		},
 	}
 
-	createUsageTestFiles(t, base, files)
-	err = os.RemoveAll(filepath.Join(base, "dir1/dira/dirasub/dcfile"))
+	createUsageTestFiles(t, base, bucket, files)
+	err = os.RemoveAll(filepath.Join(base, bucket, "dir1/dira/dirasub/dcfile"))
 	if err != nil {
 		t.Fatal(err)
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+		got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -291,8 +299,9 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 
 	for _, w := range want {
-		t.Run(w.path, func(t *testing.T) {
-			e := got.find(w.path)
+		p := path.Join(bucket, w.path)
+		t.Run(p, func(t *testing.T) {
+			e := got.find(p)
 			if w.isNil {
 				if e != nil {
 					t.Error("want nil, got", e)
@@ -335,9 +344,9 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		{name: "bucket/dir1/dira/dirasub/dcfile", size: 1000000},
 		{name: "bucket/dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
 	}
-	createUsageTestFiles(t, base, files)
+	createUsageTestFiles(t, base, "", files)
 
-	getSize := func(item Item) (i int64, err error) {
+	getSize := func(item crawlItem) (i int64, err error) {
 		if item.Typ&os.ModeDir == 0 {
 			s, err := os.Stat(item.Path)
 			if err != nil {
@@ -347,7 +356,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		}
 		return 0, nil
 	}
-	got, err := updateUsage(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, func() {}, getSize)
+	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, func() {}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -449,8 +458,8 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 			size: 1000,
 		},
 	}
-	createUsageTestFiles(t, base, files)
-	got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+	createUsageTestFiles(t, base, "", files)
+	got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -526,14 +535,14 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		},
 	}
 
-	createUsageTestFiles(t, base, files)
+	createUsageTestFiles(t, base, "", files)
 	err = os.RemoveAll(filepath.Join(base, "bucket/dir1/dira/dirasub/dcfile"))
 	if err != nil {
 		t.Fatal(err)
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = updateUsage(context.Background(), base, got, func() {}, getSize)
+		got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -588,13 +597,13 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 }
 
-func createUsageTestFiles(t *testing.T, base string, files []usageTestFile) {
+func createUsageTestFiles(t *testing.T, base, bucket string, files []usageTestFile) {
 	for _, f := range files {
-		err := os.MkdirAll(filepath.Dir(filepath.Join(base, f.name)), os.ModePerm)
+		err := os.MkdirAll(filepath.Dir(filepath.Join(base, bucket, f.name)), os.ModePerm)
 		if err != nil {
 			t.Fatal(err)
 		}
-		err = ioutil.WriteFile(filepath.Join(base, f.name), make([]byte, f.size), os.ModePerm)
+		err = ioutil.WriteFile(filepath.Join(base, bucket, f.name), make([]byte, f.size), os.ModePerm)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -606,20 +615,28 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 	if err != nil {
 		t.Skip(err)
 	}
+	const bucket = "abucket"
 	defer os.RemoveAll(base)
 	var files = []usageTestFile{
 		{name: "rootfile", size: 10000},
 		{name: "rootfile2", size: 10000},
 		{name: "dir1/d1file", size: 2000},
 		{name: "dir2/d2file", size: 300},
+		{name: "dir2/d2file2", size: 300},
+		{name: "dir2/d2file3/", size: 300},
+		{name: "dir2/d2file4/", size: 300},
+		{name: "dir2/d2file5", size: 300},
 		{name: "dir1/dira/dafile", size: 100000},
 		{name: "dir1/dira/dbfile", size: 200000},
 		{name: "dir1/dira/dirasub/dcfile", size: 1000000},
 		{name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10},
+		{name: "dir1/dira/dirasub/sublevel3/dccccfile20", size: 20},
+		{name: "dir1/dira/dirasub/sublevel3/dccccfile30", size: 30},
+		{name: "dir1/dira/dirasub/sublevel3/dccccfile40", size: 40},
 	}
-	createUsageTestFiles(t, base, files)
+	createUsageTestFiles(t, base, bucket, files)
 
-	getSize := func(item Item) (i int64, err error) {
+	getSize := func(item crawlItem) (i int64, err error) {
 		if item.Typ&os.ModeDir == 0 {
 			s, err := os.Stat(item.Path)
 			if err != nil {
@@ -629,23 +646,33 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 		}
 		return 0, nil
 	}
-	want, err := updateUsage(context.Background(), base, dataUsageCache{}, func() {}, getSize)
+	want, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, func() {}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	b := want.serialize()
 	var got dataUsageCache
-	err = got.deserialize(b)
+	err = got.deserialize(bytes.NewBuffer(b))
 	if err != nil {
 		t.Fatal(err)
 	}
-
+	t.Log("serialized size:", len(b), "bytes")
 	if got.Info.LastUpdate.IsZero() {
 		t.Error("lastupdate not set")
 	}
 
 	if !want.Info.LastUpdate.Equal(got.Info.LastUpdate) {
-		t.Fatalf("deserialize mismatch\nwant: %+v\ngot:  %+v", want, got)
+		t.Fatalf("deserialize LastUpdate mismatch\nwant: %+v\ngot:  %+v", want, got)
 	}
+	if len(want.Cache) != len(got.Cache) {
+		t.Errorf("deserialize mismatch length\nwant: %+v\ngot:  %+v", len(want.Cache), len(got.Cache))
+	}
+	for wkey, wval := range want.Cache {
+		gotv := got.Cache[wkey]
+		if fmt.Sprint(gotv) != fmt.Sprint(wval) {
+			t.Errorf("deserialize mismatch, key %v\nwant: %+v\ngot:  %+v", wkey, wval, gotv)
+		}
+	}
+
 }

cmd/disk-cache-backend.go

@@ -125,12 +125,14 @@ func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
 
 // represents disk cache struct
 type diskCache struct {
-	gcCounter uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
 	// is set to 0 if drive is offline
-	online uint32
+	online       uint32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
+	purgeRunning int32
 
-	dir           string // caching directory
-	quotaPct      int    // max usage in %
+	triggerGC     chan struct{}
+	dir           string         // caching directory
+	stats         CacheDiskStats // disk cache stats for prometheus
+	quotaPct      int            // max usage in %
 	pool          sync.Pool
 	after         int // minimum accesses before an object is cached.
 	lowWatermark  int
@@ -142,12 +144,14 @@ type diskCache struct {
 }
 
 // Inits the disk cache dir if it is not initialized already.
-func newDiskCache(dir string, quotaPct, after, lowWatermark, highWatermark int) (*diskCache, error) {
+func newDiskCache(ctx context.Context, dir string, quotaPct, after, lowWatermark, highWatermark int) (*diskCache, error) {
 	if err := os.MkdirAll(dir, 0777); err != nil {
 		return nil, fmt.Errorf("Unable to initialize '%s' dir, %w", dir, err)
 	}
 	cache := diskCache{
 		dir:           dir,
+		triggerGC:     make(chan struct{}),
+		stats:         CacheDiskStats{Dir: dir},
 		quotaPct:      quotaPct,
 		after:         after,
 		lowWatermark:  lowWatermark,
@@ -161,6 +165,8 @@ func newDiskCache(dir string, quotaPct, after, lowWatermark, highWatermark int)
 		},
 		nsMutex: newNSLock(false),
 	}
+	go cache.purgeWait(ctx)
+	cache.diskUsageHigh() // update if cache usage is already high.
 	cache.NewNSLockFn = func(ctx context.Context, cachePath string) RWLocker {
 		return cache.nsMutex.NewNSLock(ctx, nil, cachePath, "")
 	}
@@ -181,7 +187,12 @@ func (c *diskCache) diskUsageLow() bool {
 		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) < gcStopPct
+	low := int(usedPercent) < gcStopPct
+	atomic.StoreUint64(&c.stats.UsagePercent, usedPercent)
+	if low {
+		atomic.StoreInt32(&c.stats.UsageState, 0)
+	}
+	return low
 }
 
 // Returns if the disk usage reaches high water mark w.r.t the configured cache quota.
@@ -196,7 +207,12 @@ func (c *diskCache) diskUsageHigh() bool {
 		return false
 	}
 	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) >= gcTriggerPct
+	high := int(usedPercent) >= gcTriggerPct
+	atomic.StoreUint64(&c.stats.UsagePercent, usedPercent)
+	if high {
+		atomic.StoreInt32(&c.stats.UsageState, 1)
+	}
+	return high
 }
 
 // Returns if size space can be allocated without exceeding
@@ -230,24 +246,36 @@ var (
 	errDoneForNow = errors.New("done for now")
 )
 
+func (c *diskCache) purgeWait(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+		case <-c.triggerGC: // wait here until someone triggers.
+			c.purge(ctx)
+		}
+	}
+}
+
 // Purge cache entries that were not accessed.
 func (c *diskCache) purge(ctx context.Context) {
-	if c.diskUsageLow() {
+	if atomic.LoadInt32(&c.purgeRunning) == 1 || c.diskUsageLow() {
 		return
 	}
+
 	toFree := c.toClear()
 	if toFree == 0 {
 		return
 	}
+
+	atomic.StoreInt32(&c.purgeRunning, 1) // do not run concurrent purge()
+	defer atomic.StoreInt32(&c.purgeRunning, 0)
+
 	// expiry for cleaning up old cache.json files that
 	// need to be cleaned up.
 	expiry := UTCNow().Add(-cacheExpiryDays)
 	// defaulting max hits count to 100
-	scorer, err := newFileScorer(toFree, time.Now().Unix(), 100)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return
-	}
+	// ignore error we know what value we are passing.
+	scorer, _ := newFileScorer(toFree, time.Now().Unix(), 100)
 
 	// this function returns FileInfo for cached range files and cache data file.
 	fiStatFn := func(ranges map[string]string, dataFile, pathPrefix string) map[string]os.FileInfo {
@@ -326,27 +354,20 @@ func (c *diskCache) purge(ctx context.Context) {
 		return
 	}
 
-	for _, path := range scorer.fileNames() {
-		removeAll(path)
-		slashIdx := strings.LastIndex(path, SlashSeparator)
-		pathPrefix := path[0:slashIdx]
-		fname := path[slashIdx+1:]
-		if fname == cacheDataFile {
-			removeAll(pathPrefix)
+	scorer.purgeFunc(func(qfile queuedFile) {
+		fileName := qfile.name
+		removeAll(fileName)
+		slashIdx := strings.LastIndex(fileName, SlashSeparator)
+		if slashIdx >= 0 {
+			fileNamePrefix := fileName[0:slashIdx]
+			fname := fileName[slashIdx+1:]
+			if fname == cacheDataFile {
+				removeAll(fileNamePrefix)
+			}
 		}
-	}
-}
+	})
 
-func (c *diskCache) incGCCounter() {
-	atomic.AddUint64(&c.gcCounter, 1)
-}
-
-func (c *diskCache) resetGCCounter() {
-	atomic.StoreUint64(&c.gcCounter, 0)
-}
-
-func (c *diskCache) gcCount() uint64 {
-	return atomic.LoadUint64(&c.gcCounter)
+	scorer.reset()
 }
 
 // sets cache drive status
@@ -498,10 +519,14 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 	}
 	// increment hits
 	if rs != nil {
-		if m.Ranges == nil {
-			m.Ranges = make(map[string]string)
+		// rsFileName gets set by putRange. Check for blank values here
+		// coming from other code paths that set rs only (eg initial creation or hit increment).
+		if rsFileName != "" {
+			if m.Ranges == nil {
+				m.Ranges = make(map[string]string)
+			}
+			m.Ranges[rs.String(actualSize)] = rsFileName
 		}
-		m.Ranges[rs.String(actualSize)] = rsFileName
 	} else {
 		// this is necessary cleanup of range files if entire object is cached.
 		for _, f := range m.Ranges {
@@ -626,7 +651,7 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
 // Caches the object to disk
 func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) error {
 	if c.diskUsageHigh() {
-		c.incGCCounter()
+		c.triggerGC <- struct{}{}
 		io.Copy(ioutil.Discard, data)
 		return errDiskFull
 	}

cmd/disk-cache-check-support_other.go

@@ -0,0 +1,56 @@
+// +build !windows
+
+/*
+ * MinIO Cloud Storage, (C) 2019-2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"io"
+	"io/ioutil"
+	"os"
+	"time"
+
+	"github.com/djherbis/atime"
+)
+
+// Return error if Atime is disabled on the O/S
+func checkAtimeSupport(dir string) (err error) {
+	file, err := ioutil.TempFile(dir, "prefix")
+	if err != nil {
+		return
+	}
+	defer os.Remove(file.Name())
+	defer file.Close()
+	finfo1, err := os.Stat(file.Name())
+	if err != nil {
+		return
+	}
+	// add a sleep to ensure atime change is detected
+	time.Sleep(10 * time.Millisecond)
+
+	if _, err = io.Copy(ioutil.Discard, file); err != nil {
+		return
+	}
+
+	finfo2, err := os.Stat(file.Name())
+
+	if atime.Get(finfo2).Equal(atime.Get(finfo1)) {
+		return errors.New("Atime not supported")
+	}
+	return
+}

cmd/disk-cache-check-support_windows.go

@@ -0,0 +1,60 @@
+// +build windows
+
+/*
+ * MinIO Cloud Storage, (C) 2019-2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"io/ioutil"
+	"os"
+
+	"github.com/djherbis/atime"
+	"golang.org/x/sys/windows/registry"
+)
+
+// Return error if Atime is disabled on the O/S
+func checkAtimeSupport(dir string) (err error) {
+	file, err := ioutil.TempFile(dir, "prefix")
+	if err != nil {
+		return
+	}
+	defer os.Remove(file.Name())
+	defer file.Close()
+	finfo1, err := os.Stat(file.Name())
+	if err != nil {
+		return
+	}
+	atime.Get(finfo1)
+
+	k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SYSTEM\CurrentControlSet\Control\FileSystem`, registry.QUERY_VALUE)
+	if err != nil {
+		return
+	}
+	defer k.Close()
+
+	setting, _, err := k.GetIntegerValue("NtfsDisableLastAccessUpdate")
+	if err != nil {
+		return
+	}
+
+	lowSetting := setting & 0xFFFF
+	if lowSetting != uint64(0x0000) && lowSetting != uint64(0x0002) {
+		return errors.New("Atime not supported")
+	}
+	return
+}

cmd/disk-cache-stats.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019, 2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,14 +16,27 @@
 
 package cmd
 
-import "sync/atomic"
+import (
+	"sync/atomic"
+)
+
+// CacheDiskStats represents cache disk statistics
+// such as current disk usage and available.
+type CacheDiskStats struct {
+	// indicates if usage is high or low, if high value is '1', if low its '0'
+	UsageState int32
+	// indicates the current usage percentage of this cache disk
+	UsagePercent uint64
+	Dir          string
+}
 
 // CacheStats - represents bytes served from cache,
 // cache hits and cache misses.
 type CacheStats struct {
-	BytesServed uint64
-	Hits        uint64
-	Misses      uint64
+	BytesServed  uint64
+	Hits         uint64
+	Misses       uint64
+	GetDiskStats func() []CacheDiskStats
 }
 
 // Increase total bytes served from cache