fix: CopyObject behavior on expanded zones (#9729)

CopyObject was not correctly figuring out the correct
destination object location and would end up creating
duplicate objects on two different zones, reproduced
by doing encryption based key rotation.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: MinIO Community Support
+    url: https://slack.min.io
+    about: Please ask and answer questions here.
+  - name: MinIO SUBNET Support
+    url: https://min.io/pricing
+    about: Join this for Enterprise Support.

.github/lock.yml

@@ -0,0 +1,39 @@
+# Configuration for Lock Threads - https://github.com/dessant/lock-threads-app
+
+# Number of days of inactivity before a closed issue or pull request is locked
+daysUntilLock: 365
+
+# Skip issues and pull requests created before a given timestamp. Timestamp must
+# follow ISO 8601 (`YYYY-MM-DD`). Set to `false` to disable
+skipCreatedBefore: false
+
+# Issues and pull requests with these labels will be ignored. Set to `[]` to disable
+exemptLabels: []
+
+# Label to add before locking, such as `outdated`. Set to `false` to disable
+lockLabel: false
+
+# Comment to post before locking. Set to `false` to disable
+lockComment: >-
+
+  This thread has been automatically locked since there has not been
+  any recent activity after it was closed. Please open a new issue for
+  related bugs.
+
+# Assign `resolved` as the reason for locking. Set to `false` to disable
+setLockReason: true
+
+# Limit to only `issues` or `pulls`
+only: issues
+
+# Optionally, specify configuration settings just for `issues` or `pulls`
+# issues:
+#   exemptLabels:
+#     - help-wanted
+#   lockLabel: outdated
+
+# pulls:
+#   daysUntilLock: 30
+
+# Repository to extend settings from
+# _extends: repo

.github/stale.yml

@@ -0,0 +1,59 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 90
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+daysUntilClose: 30
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
+exemptLabels:
+  - "security"
+  - "pending discussion"
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking as stale
+staleLabel: stale
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >-
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed after 21 days if no further activity
+  occurs. Thank you for your contributions.
+# Comment to post when removing the stale label.
+# unmarkComment: >
+#   Your comment here.
+
+# Comment to post when closing a stale Issue or Pull Request.
+# closeComment: >
+#   Your comment here.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 1
+
+# Limit to only `issues` or `pulls`
+# only: issues
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed

.github/workflows/go.yml

@@ -0,0 +1,52 @@
+name: Go
+
+on:
+  pull_request:
+    branches:
+    - master
+  push:
+    branches:
+    - master
+
+jobs:
+  build:
+    name: Test on Go ${{ matrix.go-version }} and ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        go-version: [1.13.x]
+        os: [ubuntu-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '12'
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Build on ${{ matrix.os }}
+        if: matrix.os == 'windows-latest'
+        env:
+          CGO_ENABLED: 0
+          GO111MODULE: on
+          MINIO_CI_CD: 1
+        run: |
+          go build --ldflags="-s -w" -o %GOPATH%\bin\minio.exe
+          go test -v --timeout 50m ./...
+      - name: Build on ${{ matrix.os }}
+        if: matrix.os == 'ubuntu-latest'
+        env:
+          CGO_ENABLED: 0
+          GO111MODULE: on
+          MINIO_CI_CD: 1
+        run: |
+          sudo apt-get install devscripts shellcheck
+          make
+          diff -au <(gofmt -s -d cmd) <(printf "")
+          diff -au <(gofmt -s -d pkg) <(printf "")
+          make test-race
+          make crosscompile
+          make verify
+          make verify-healing
+          cd browser && npm install && npm run test && cd ..
+          bash -c 'shopt -s globstar; shellcheck mint/**/*.sh'

.golangci.yml

@@ -0,0 +1,27 @@
+linters-settings:
+  golint:
+    min-confidence: 0
+
+  misspell:
+    locale: US
+
+linters:
+  disable-all: true
+  enable:
+    - typecheck
+    - goimports
+    - misspell
+    - govet
+    - golint
+    - ineffassign
+    - gosimple
+    - deadcode
+    - structcheck
+
+issues:
+  exclude-use-default: false
+  exclude:
+      - should have a package comment
+      - error strings should not be capitalized or end with punctuation or a newline
+service:
+  golangci-lint-version: 1.20.0 # use the fixed version to not introduce new linters unexpectedly

.travis.yml

@@ -1,58 +0,0 @@
-go_import_path: github.com/minio/minio
-
-language: go
-
-addons:
-  apt:
-    packages:
-    - shellcheck
-
-services:
-  - docker
-
-# this ensures PRs based on a local branch are not built twice
-# the downside is that a PR targeting a different branch is not built
-# but as a workaround you can add the branch to this list
-branches:
-  only:
-    - master
-
-matrix:
-  include:
-    - os: linux
-      dist: trusty
-      sudo: required
-      env:
-        - ARCH=x86_64
-        - CGO_ENABLED=0
-        - GO111MODULE=on
-        - SIMPLE_CI=1
-      go: 1.13.x
-      script:
-        - make
-        - diff -au <(gofmt -s -d cmd) <(printf "")
-        - diff -au <(gofmt -s -d pkg) <(printf "")
-        - make test-race
-        - make crosscompile
-        - make verify
-        - cd browser && npm install && npm run test && cd ..
-        - bash -c 'shopt -s globstar; shellcheck mint/**/*.sh'
-
-    - os: windows
-      env:
-        - ARCH=x86_64
-        - CGO_ENABLED=0
-        - GO111MODULE=on
-        - SIMPLE_CI=1
-      go: 1.13.x
-      script:
-        - go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
-        - for d in $(go list ./... | grep -v browser); do go test -v --timeout 20m "$d"; done
-
-before_script:
-  # Add an IPv6 config - see the corresponding Travis issue
-  # https://github.com/travis-ci/travis-ci/issues/8361
-  - if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'; fi
-
-before_install:
- - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi

CREDITS

@@ -1306,33 +1306,6 @@ https://github.com/aws/aws-sdk-go
 
 ================================================================
 
-github.com/baiyubin/aliyun-sts-go-sdk
-https://github.com/baiyubin/aliyun-sts-go-sdk
-----------------------------------------------------------------
-The MIT License (MIT)
-
-Copyright (c) Aliyun.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-================================================================
-
 github.com/bcicen/jstream
 https://github.com/bcicen/jstream
 ----------------------------------------------------------------
@@ -4514,214 +4487,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ================================================================
 
-github.com/google/gofuzz
-https://github.com/google/gofuzz
-----------------------------------------------------------------
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-================================================================
-
 github.com/google/martian
 https://github.com/google/martian
 ----------------------------------------------------------------
@@ -10963,25 +10728,6 @@ THE SOFTWARE.
 
 ================================================================
 
-github.com/kurin/blazer
-https://github.com/kurin/blazer
-----------------------------------------------------------------
-Copyright 2016, the Blazer authors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-================================================================
-
 github.com/lib/pq
 https://github.com/lib/pq
 ----------------------------------------------------------------
@@ -16597,32 +16343,6 @@ SOFTWARE.
 
 ================================================================
 
-github.com/satori/go.uuid
-https://github.com/satori/go.uuid
-----------------------------------------------------------------
-Copyright (C) 2013-2018 by Maxim Bublis <b@codemonkey.ru>
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-================================================================
-
 github.com/secure-io/sio-go
 https://github.com/secure-io/sio-go
 ----------------------------------------------------------------
@@ -17009,6 +16729,36 @@ https://github.com/soheilhy/cmux
 
 ================================================================
 
+github.com/spaolacci/murmur3
+https://github.com/spaolacci/murmur3
+----------------------------------------------------------------
+Copyright 2013, Sébastien Paolacci.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of the library nor the
+      names of its contributors may be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
+DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
 github.com/streadway/amqp
 https://github.com/streadway/amqp
 ----------------------------------------------------------------
@@ -17279,6 +17029,69 @@ SOFTWARE.
 
 ================================================================
 
+github.com/willf/bitset
+https://github.com/willf/bitset
+----------------------------------------------------------------
+Copyright (c) 2014 Will Fitzgerald. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
+github.com/willf/bloom
+https://github.com/willf/bloom
+----------------------------------------------------------------
+Copyright (c) 2014 Will Fitzgerald. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
 github.com/xdg/scram
 https://github.com/xdg/scram
 ----------------------------------------------------------------
@@ -17952,6 +17765,31 @@ THE SOFTWARE.
 
 ================================================================
 
+go.uber.org/tools
+https://go.uber.org/tools
+----------------------------------------------------------------
+Copyright (c) 2017 Uber Technologies, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+================================================================
+
 go.uber.org/zap
 https://go.uber.org/zap
 ----------------------------------------------------------------
@@ -18010,6 +17848,72 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ================================================================
 
+golang.org/x/lint
+https://golang.org/x/lint
+----------------------------------------------------------------
+Copyright (c) 2013 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
+golang.org/x/mod
+https://golang.org/x/mod
+----------------------------------------------------------------
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
 golang.org/x/net
 https://golang.org/x/net
 ----------------------------------------------------------------
@@ -18241,6 +18145,39 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ================================================================
 
+golang.org/x/xerrors
+https://golang.org/x/xerrors
+----------------------------------------------------------------
+Copyright (c) 2019 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
 google.golang.org/api
 https://google.golang.org/api
 ----------------------------------------------------------------
@@ -20675,3 +20612,29 @@ https://gopkg.in/yaml.v2
 
 ================================================================
 
+honnef.co/go/tools
+https://honnef.co/go/tools
+----------------------------------------------------------------
+Copyright (c) 2016 Dominik Honnef
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+================================================================
+

Dockerfile.dev.browser

@@ -0,0 +1,13 @@
+FROM ubuntu
+
+LABEL maintainer="MinIO Inc <dev@min.io>"
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends --no-install-suggests \
+      git golang make npm && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+ENV PATH=$PATH:/root/go/bin
+
+RUN go get github.com/go-bindata/go-bindata/go-bindata && \
+    go get github.com/elazarl/go-bindata-assetfs/go-bindata-assetfs
+

Dockerfile.mint

@@ -1,17 +1,11 @@
-FROM ubuntu:16.04
+FROM ubuntu:18.04
 
 ENV DEBIAN_FRONTEND noninteractive
-
 ENV LANG C.UTF-8
-
 ENV GOROOT /usr/local/go
-
-ENV GOPATH /usr/local
-
+ENV GOPATH /usr/local/gopath
 ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
-
 ENV MINT_ROOT_DIR /mint
-
 COPY mint /mint
 
 RUN apt-get --yes update && apt-get --yes upgrade && \

Dockerfile.simpleci

@@ -1,80 +0,0 @@
-#-------------------------------------------------------------
-# Stage 1: Build and Unit tests
-#-------------------------------------------------------------
-FROM golang:1.13
-
-COPY . /go/src/github.com/minio/minio
-WORKDIR /go/src/github.com/minio/minio
-
-RUN apt-get update && apt-get install -y jq
-ENV GO111MODULE=on
-ENV SIMPLE_CI 1
-
-RUN git config --global http.cookiefile /gitcookie/.gitcookie
-
-RUN apt-get update && \
-      apt-get -y install sudo
-RUN touch /etc/sudoers
-
-RUN echo "root ALL=(ALL) ALL" >> /etc/sudoers
-RUN echo "ci ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
-RUN echo "Defaults    env_reset" >> /etc/sudoers
-RUN echo 'Defaults    secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go:/usr/local/go/bin"' >> /etc/sudoers
-
-RUN mkdir -p /home/ci/.cache
-
-RUN groupadd -g 999 ci && \
-    useradd -r -u 999 -g ci ci && \
-    chown -R ci:ci /go /home/ci && \
-    chmod -R a+rw /go
-
-USER ci
-
-# -- tests --
-RUN make
-RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")'
-RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")'
-RUN make test-race
-RUN make crosscompile
-RUN make verify
-
-## -- add healing tests
-RUN make verify-healing
-
-#-------------------------------------------------------------
-# Stage 2: Test Frontend
-#-------------------------------------------------------------
-FROM node:10.15-stretch-slim
-
-ENV SIMPLE_CI 1
-
-COPY browser /minio/browser
-WORKDIR /minio/browser
-
-RUN yarn
-RUN yarn test
-
-#-------------------------------------------------------------
-# Stage 3: Run Gateway Tests
-#-------------------------------------------------------------
-FROM ubuntu:16.04
-
-COPY --from=0 /go/src/github.com/minio/minio/minio /usr/bin/minio
-COPY buildscripts/gateway-tests.sh /usr/bin/gateway-tests.sh
-COPY mint /mint
-
-ENV DEBIAN_FRONTEND noninteractive
-ENV LANG C.UTF-8
-ENV GOROOT /usr/local/go
-ENV GOPATH /usr/local
-ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
-ENV SIMPLE_CI 1
-ENV MINT_ROOT_DIR /mint
-
-RUN apt-get --yes update && apt-get --yes upgrade && \
-    apt-get --yes --quiet install wget jq curl git dnsmasq && \
-    cd /mint && /mint/release.sh
-
-WORKDIR /mint
-
-RUN /usr/bin/gateway-tests.sh

Makefile

@@ -18,22 +18,12 @@ checks:
 
 getdeps:
 	@mkdir -p ${GOPATH}/bin
-	@which golint 1>/dev/null || (echo "Installing golint" && GO111MODULE=off go get -u golang.org/x/lint/golint)
-ifeq ($(GOARCH),s390x)
-	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && GO111MODULE=off go get honnef.co/go/tools/cmd/staticcheck)
-else
-	@which staticcheck 1>/dev/null || (echo "Installing staticcheck" && wget --quiet https://github.com/dominikh/go-tools/releases/download/2020.1.3/staticcheck_${GOOS}_${GOARCH}.tar.gz && tar xf staticcheck_${GOOS}_${GOARCH}.tar.gz && mv staticcheck/staticcheck ${GOPATH}/bin/staticcheck && chmod +x ${GOPATH}/bin/staticcheck && rm -f staticcheck_${GOOS}_${GOARCH}.tar.gz && rm -rf staticcheck)
-endif
-	@which misspell 1>/dev/null || (echo "Installing misspell" && GO111MODULE=off go get -u github.com/client9/misspell/cmd/misspell)
+	@which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0)
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
 
-verifiers: getdeps vet fmt lint staticcheck spelling
-
-vet:
-	@echo "Running $@ check"
-	@GO111MODULE=on go vet github.com/minio/minio/...
+verifiers: getdeps fmt lint
 
 fmt:
 	@echo "Running $@ check"
@@ -42,21 +32,8 @@ fmt:
 
 lint:
 	@echo "Running $@ check"
-	@GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd/...
-	@GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg/...
-
-staticcheck:
-	@echo "Running $@ check"
-	@GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/cmd/...
-	@GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/pkg/...
-
-spelling:
-	@echo "Running $@ check"
-	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find cmd/`
-	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find pkg/`
-	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find docs/`
-	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
-	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
+	@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
+	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ./.golangci.yml
 
 # Builds minio, runs the verifiers then runs the tests.
 check: test

README.md

@@ -3,7 +3,7 @@
 
 [![MinIO](https://raw.githubusercontent.com/minio/minio/master/.github/logo.svg?sanitize=true)](https://min.io)
 
-MinIO is High Performance Object Storage released under Apache License v2.0. It is API compatible with Amazon S3 cloud storage service. Using MinIO build high performance infrastructure for machine learning, analytics and application data workloads.
+MinIO is a High Performance Object Storage released under Apache License v2.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads.
 
 ## Docker Container
 ### Stable
@@ -22,7 +22,7 @@ docker run -p 9000:9000 minio/minio:edge server /data
 
 ## macOS
 ### Homebrew (recommended)
-Install minio packages using [Homebrew](http://brew.sh/)
+Install minio packages using [Homebrew](https://brew.sh/)
 ```sh
 brew install minio/stable/minio
 minio server /data

browser/README.md

@@ -2,6 +2,7 @@
 
 ``MinIO Browser`` provides minimal set of UI to manage buckets and objects on ``minio`` server. ``MinIO Browser`` is written in javascript and released under [Apache 2.0 License](./LICENSE).
 
+
 ## Installation
 
 ### Install node
@@ -11,6 +12,11 @@ exec -l $SHELL
 nvm install stable
 ```
 
+### Install node dependencies
+```sh
+npm install
+```
+
 ### Install `go-bindata` and `go-bindata-assetfs`
 
 If you do not have a working Golang environment, please follow [Install Golang](https://golang.org/doc/install)
@@ -30,13 +36,16 @@ npm run release
 
 This generates ui-assets.go in the current directory. Now do `make` in the parent directory to build the minio binary with the newly generated ``ui-assets.go``
 
+
+## Run MinIO Browser with live reload
+
 ### Run MinIO Browser with live reload
 
 ```sh
 npm run dev
 ```
 
-Open [http://localhost:8080/minio/](http://localhost:8080/minio/) in your browser to play with the application
+Open [http://localhost:8080/minio/](http://localhost:8080/minio/) in your browser to play with the application.
 
 ### Run MinIO Browser with live reload on custom port
 
@@ -54,7 +63,7 @@ index 3ccdaba..9496c56 100644
 +    port: 8888,
      proxy: {
        '/minio/webrpc': {
-        target: 'http://localhost:9000',
+         target: 'http://localhost:9000',
 @@ -97,7 +98,7 @@ var exports = {
  if (process.env.NODE_ENV === 'dev') {
    exports.entry = [
@@ -70,4 +79,102 @@ index 3ccdaba..9496c56 100644
 npm run dev
 ```
 
-Open [http://localhost:8888/minio/](http://localhost:8888/minio/) in your browser to play with the application
+Open [http://localhost:8888/minio/](http://localhost:8888/minio/) in your browser to play with the application.
+
+### Run MinIO Browser with live reload on any IP
+
+Edit `browser/webpack.config.js`
+
+```diff
+diff --git a/browser/webpack.config.js b/browser/webpack.config.js
+index 8bdbba53..139f6049 100644
+--- a/browser/webpack.config.js
++++ b/browser/webpack.config.js
+@@ -71,6 +71,7 @@ var exports = {
+     historyApiFallback: {
+       index: '/minio/'
+     },
++    host: '0.0.0.0',
+     proxy: {
+       '/minio/webrpc': {
+         target: 'http://localhost:9000',
+```
+
+```sh
+npm run dev
+```
+
+Open [http://IP:8080/minio/](http://IP:8080/minio/) in your browser to play with the application.
+
+
+## Run tests
+
+    npm run test
+
+
+## Docker development environment
+
+This approach will download the sources on your machine such that you are able to use your IDE or editor of choice.
+A Docker container will be used in order to provide a controlled build environment without messing with your host system.
+
+### Prepare host system
+
+Install [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) and [Docker](https://docs.docker.com/get-docker/).
+
+### Development within container
+
+Prepare and build container
+```
+git clone git@github.com:minio/minio.git
+cd minio
+docker build -t minio-dev -f Dockerfile.dev.browser .
+```
+
+Run container, build and run core
+```sh
+docker run -it --rm --name minio-dev -v "$PWD":/minio minio-dev
+
+cd /minio/browser
+npm install
+npm run release
+cd /minio
+make
+./minio server /data
+```
+Note `Endpoint` IP (the one which is _not_ `127.0.0.1`), `AccessKey` and `SecretKey` (both default to `minioadmin`) in order to enter them in the browser later.
+
+
+Open another terminal.
+Connect to container
+```sh
+docker exec -it minio-dev bash
+```
+
+Apply patch to allow access from outside container
+```sh
+cd /minio
+git apply --ignore-whitespace <<EOF
+diff --git a/browser/webpack.config.js b/browser/webpack.config.js
+index 8bdbba53..139f6049 100644
+--- a/browser/webpack.config.js
++++ b/browser/webpack.config.js
+@@ -71,6 +71,7 @@ var exports = {
+     historyApiFallback: {
+       index: '/minio/'
+     },
++    host: '0.0.0.0',
+     proxy: {
+       '/minio/webrpc': {
+         target: 'http://localhost:9000',
+EOF
+```
+
+Build and run frontend with auto-reload
+```sh
+cd /minio/browser
+npm install
+npm run dev
+```
+
+Open [http://IP:8080/minio/](http://IP:8080/minio/) in your browser to play with the application.
+

browser/app/js/browser/ChangePasswordModal.js

@@ -88,11 +88,6 @@ export class ChangePasswordModal extends React.Component {
 
   canChangePassword() {
     const { serverInfo } = this.props
-    // Password change is not allowed in WORM mode
-    if (serverInfo.info.isWorm) {
-      return false
-    }
-
     // Password change is not allowed for temporary users(STS)
     if(serverInfo.userInfo.isTempUser) {
       return false

browser/app/js/browser/__tests__/ChangePasswordModal.test.js

@@ -64,17 +64,6 @@ describe("ChangePasswordModal", () => {
     shallow(<ChangePasswordModal serverInfo={serverInfo} />)
   })
 
-  it("should not allow changing password when isWorm is true", () => {
-    const newServerInfo = { ...serverInfo, info: { isWorm: true } }
-    const wrapper = shallow(<ChangePasswordModal serverInfo={newServerInfo} />)
-    expect(
-      wrapper
-        .find("ModalBody")
-        .childAt(0)
-        .text()
-    ).toBe("Credentials of this user cannot be updated through MinIO Browser.")
-  })
-
   it("should not allow changing password when not IAM user", () => {
     const newServerInfo = {
       ...serverInfo,

browser/app/js/browser/selectors.js

@@ -0,0 +1,24 @@
+/*
+ * MinIO Cloud Storage (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { createSelector } from "reselect"
+
+export const getServerInfo = state => state.browser.serverInfo
+
+export const hasServerPublicDomain = createSelector(
+  getServerInfo,
+  serverInfo => Boolean(serverInfo.info && serverInfo.info.domains && serverInfo.info.domains.length),
+)

browser/app/js/objects/ObjectActions.js

@@ -47,6 +47,11 @@ export class ObjectActions extends React.Component {
       SHARE_OBJECT_EXPIRY_MINUTES
     )
   }
+  handleDownload(e) {
+    e.preventDefault()
+    const { object, downloadObject } = this.props
+    downloadObject(object.name)
+  }
   deleteObject() {
     const { object, deleteObject } = this.props
     deleteObject(object.name)
@@ -82,6 +87,7 @@ export class ObjectActions extends React.Component {
           <a
             href=""
             className="fiad-action"
+            title="Share"
             onClick={this.shareObject.bind(this)}
           >
             <i className="fas fa-share-alt" />
@@ -90,6 +96,7 @@ export class ObjectActions extends React.Component {
             <a
               href=""
               className="fiad-action"
+              title="Preview"
               onClick={this.showPreviewModal.bind(this)}
             >
               <i className="far fa-file-image" />
@@ -98,6 +105,15 @@ export class ObjectActions extends React.Component {
           <a
             href=""
             className="fiad-action"
+            title="Download"
+            onClick={this.handleDownload.bind(this)}
+          >
+            <i className="fas fa-cloud-download-alt" />
+          </a>
+          <a
+            href=""
+            className="fiad-action"
+            title="Delete"
             onClick={this.showDeleteConfirmModal.bind(this)}
           >
             <i className="fas fa-trash-alt" />
@@ -134,6 +150,7 @@ const mapStateToProps = (state, ownProps) => {
 
 const mapDispatchToProps = (dispatch) => {
   return {
+    downloadObject: object => dispatch(objectsActions.downloadObject(object)),
     shareObject: (object, days, hours, minutes) =>
       dispatch(objectsActions.shareObject(object, days, hours, minutes)),
     deleteObject: (object) => dispatch(objectsActions.deleteObject(object)),

browser/app/js/objects/PrefixActions.js

@@ -0,0 +1,95 @@
+/*
+ * MinIO Cloud Storage (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from "react"
+import { connect } from "react-redux"
+import { Dropdown } from "react-bootstrap"
+import DeleteObjectConfirmModal from "./DeleteObjectConfirmModal"
+import * as actions from "./actions"
+
+export class PrefixActions extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      showDeleteConfirmation: false,
+    }
+  }
+  handleDownload(e) {
+    e.preventDefault()
+    const { object, downloadPrefix } = this.props
+    downloadPrefix(object.name)
+  }
+  deleteObject() {
+    const { object, deleteObject } = this.props
+    deleteObject(object.name)
+  }
+  showDeleteConfirmModal(e) {
+    e.preventDefault()
+    this.setState({ showDeleteConfirmation: true })
+  }
+  hideDeleteConfirmModal() {
+    this.setState({
+      showDeleteConfirmation: false,
+    })
+  }
+  render() {
+    const { object, showShareObjectModal, shareObjectName } = this.props
+    return (
+      <Dropdown id={`obj-actions-${object.name}`}>
+        <Dropdown.Toggle noCaret className="fia-toggle" />
+        <Dropdown.Menu>
+          <a
+            href=""
+            className="fiad-action"
+            title="Download as zip"
+            onClick={this.handleDownload.bind(this)}
+          >
+            <i className="fas fa-cloud-download-alt" />
+          </a>
+          <a
+            href=""
+            className="fiad-action"
+            title="Delete"
+            onClick={this.showDeleteConfirmModal.bind(this)}
+          >
+            <i className="fas fa-trash-alt" />
+          </a>
+        </Dropdown.Menu>
+        {this.state.showDeleteConfirmation && (
+          <DeleteObjectConfirmModal
+            deleteObject={this.deleteObject.bind(this)}
+            hideDeleteConfirmModal={this.hideDeleteConfirmModal.bind(this)}
+          />
+        )}
+      </Dropdown>
+    )
+  }
+}
+
+const mapStateToProps = (state, ownProps) => {
+  return {
+    object: ownProps.object,
+  }
+}
+
+const mapDispatchToProps = (dispatch) => {
+  return {
+    downloadPrefix: object => dispatch(actions.downloadPrefix(object)),
+    deleteObject: (object) => dispatch(actions.deleteObject(object)),
+  }
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(PrefixActions)

browser/app/js/objects/PrefixContainer.js

@@ -17,22 +17,32 @@
 import React from "react"
 import { connect } from "react-redux"
 import ObjectItem from "./ObjectItem"
+import PrefixActions from "./PrefixActions"
 import * as actionsObjects from "./actions"
+import { getCheckedList } from "./selectors"
 
-export const PrefixContainer = ({ object, currentPrefix, selectPrefix }) => {
+export const PrefixContainer = ({
+  object,
+  currentPrefix,
+  checkedObjectsCount,
+  selectPrefix
+}) => {
   const props = {
     name: object.name,
     contentType: object.contentType,
     onClick: () => selectPrefix(`${currentPrefix}${object.name}`)
   }
-
+  if (checkedObjectsCount == 0) {
+    props.actionButtons = <PrefixActions object={object} />
+  }
   return <ObjectItem {...props} />
 }
 
 const mapStateToProps = (state, ownProps) => {
   return {
     object: ownProps.object,
-    currentPrefix: state.objects.currentPrefix
+    currentPrefix: state.objects.currentPrefix,
+    checkedObjectsCount: getCheckedList(state).length
   }
 }
 

browser/app/js/objects/PreviewObjectModal.js

@@ -23,7 +23,10 @@ class PreviewObjectModal extends React.Component {
     this.state = {
       url: "",
     }
-    props.getObjectURL(props.object.name, (url) => {
+  }
+
+  componentDidMount() {
+    this.props.getObjectURL(this.props.object.name, (url) => {
       this.setState({
         url: url,
       })

browser/app/js/objects/ShareObjectModal.js

@@ -77,7 +77,8 @@ export class ShareObjectModal extends React.Component {
     hideShareObject()
   }
   render() {
-    const { shareObjectDetails, shareObject, hideShareObject } = this.props
+    const { shareObjectDetails, hideShareObject } = this.props
+    const url = `${window.location.protocol}//${shareObjectDetails.url}`
     return (
       <Modal
         show={true}
@@ -93,11 +94,12 @@ export class ShareObjectModal extends React.Component {
               type="text"
               ref={node => (this.copyTextInput = node)}
               readOnly="readOnly"
-              value={window.location.protocol + "//" + shareObjectDetails.url}
+              value={url}
               onClick={() => this.copyTextInput.select()}
             />
           </div>
-          <div
+          {shareObjectDetails.showExpiryDate && (
+            <div
             className="input-group"
             style={{ display: web.LoggedIn() ? "block" : "none" }}
           >
@@ -174,10 +176,11 @@ export class ShareObjectModal extends React.Component {
               </div>
             </div>
           </div>
+          )}
         </ModalBody>
         <div className="modal-footer">
           <CopyToClipboard
-            text={window.location.protocol + "//" + shareObjectDetails.url}
+            text={url}
             onCopy={this.onUrlCopied.bind(this)}
           >
             <button className="btn btn-success">Copy Link</button>

browser/app/js/objects/__tests__/ObjectActions.test.js

@@ -67,6 +67,20 @@ describe("ObjectActions", () => {
   })
 
 
+  it("should call downloadObject when single object is selected and download button is clicked", () => {
+    const downloadObject = jest.fn()
+    const wrapper = shallow(
+      <ObjectActions
+        object={{ name: "obj1" }}
+        currentPrefix={"pre1/"}
+        downloadObject={downloadObject} />
+    )
+    wrapper
+      .find("a")
+      .at(1)
+      .simulate("click", { preventDefault: jest.fn() })
+    expect(downloadObject).toHaveBeenCalled()
+  })
 
 
   it("should show PreviewObjectModal when preview action is clicked", () => {
@@ -106,7 +120,7 @@ describe("ObjectActions", () => {
     )
     expect(wrapper
       .find("a")
-      .length).toBe(2) // find only the other 2
+      .length).toBe(3) // find only the other 2
   })
 
   it("should call shareObject with object and expiry", () => {

browser/app/js/objects/__tests__/PrefixActions.test.js

@@ -0,0 +1,84 @@
+/*
+ * MinIO Cloud Storage (C) 2018 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from "react"
+import { shallow } from "enzyme"
+import { PrefixActions } from "../PrefixActions"
+
+describe("PrefixActions", () => {
+  it("should render without crashing", () => {
+    shallow(<PrefixActions object={{ name: "abc/" }} currentPrefix={"pre1/"} />)
+  })
+
+  it("should show DeleteObjectConfirmModal when delete action is clicked", () => {
+    const wrapper = shallow(
+      <PrefixActions object={{ name: "abc/" }} currentPrefix={"pre1/"} />
+    )
+    wrapper
+      .find("a")
+      .last()
+      .simulate("click", { preventDefault: jest.fn() })
+    expect(wrapper.state("showDeleteConfirmation")).toBeTruthy()
+    expect(wrapper.find("DeleteObjectConfirmModal").length).toBe(1)
+  })
+
+  it("should hide DeleteObjectConfirmModal when Cancel button is clicked", () => {
+    const wrapper = shallow(
+      <PrefixActions object={{ name: "abc/" }} currentPrefix={"pre1/"} />
+    )
+    wrapper
+      .find("a")
+      .last()
+      .simulate("click", { preventDefault: jest.fn() })
+    wrapper.find("DeleteObjectConfirmModal").prop("hideDeleteConfirmModal")()
+    wrapper.update()
+    expect(wrapper.state("showDeleteConfirmation")).toBeFalsy()
+    expect(wrapper.find("DeleteObjectConfirmModal").length).toBe(0)
+  })
+
+  it("should call deleteObject with object name", () => {
+    const deleteObject = jest.fn()
+    const wrapper = shallow(
+      <PrefixActions
+        object={{ name: "abc/" }}
+        currentPrefix={"pre1/"}
+        deleteObject={deleteObject}
+      />
+    )
+    wrapper
+      .find("a")
+      .last()
+      .simulate("click", { preventDefault: jest.fn() })
+    wrapper.find("DeleteObjectConfirmModal").prop("deleteObject")()
+    expect(deleteObject).toHaveBeenCalledWith("abc/")
+  })
+
+
+  it("should call downloadPrefix when single object is selected and download button is clicked", () => {
+    const downloadPrefix = jest.fn()
+    const wrapper = shallow(
+      <PrefixActions
+        object={{ name: "abc/" }}
+        currentPrefix={"pre1/"}
+        downloadPrefix={downloadPrefix} />
+    )
+    wrapper
+      .find("a")
+      .first()
+      .simulate("click", { preventDefault: jest.fn() })
+    expect(downloadPrefix).toHaveBeenCalled()
+  })
+})

browser/app/js/objects/__tests__/PrefixContainer.test.js

@@ -41,4 +41,22 @@ describe("PrefixContainer", () => {
     wrapper.find("Connect(ObjectItem)").prop("onClick")()
     expect(selectPrefix).toHaveBeenCalledWith("xyz/abc/")
   })
+
+  it("should pass actions to ObjectItem", () => {
+    const wrapper = shallow(
+      <PrefixContainer object={{ name: "abc/" }} checkedObjectsCount={0} />
+    )
+    expect(wrapper.find("Connect(ObjectItem)").prop("actionButtons")).not.toBe(
+      undefined
+    )
+  })
+
+  it("should pass empty actions to ObjectItem when checkedObjectCount is more than 0", () => {
+    const wrapper = shallow(
+      <PrefixContainer object={{ name: "abc/" }} checkedObjectsCount={1} />
+    )
+    expect(wrapper.find("Connect(ObjectItem)").prop("actionButtons")).toBe(
+      undefined
+    )
+  })
 })

browser/app/js/objects/__tests__/ShareObjectModal.test.js

@@ -34,7 +34,7 @@ describe("ShareObjectModal", () => {
     shallow(
       <ShareObjectModal
         object={{ name: "obj1" }}
-        shareObjectDetails={{ show: true, object: "obj1", url: "test" }}
+        shareObjectDetails={{ show: true, object: "obj1", url: "test", showExpiryDate: true }}
       />
     )
   })
@@ -44,7 +44,7 @@ describe("ShareObjectModal", () => {
     const wrapper = shallow(
       <ShareObjectModal
         object={{ name: "obj1" }}
-        shareObjectDetails={{ show: true, object: "obj1", url: "test" }}
+        shareObjectDetails={{ show: true, object: "obj1", url: "test", showExpiryDate: true }}
         hideShareObject={hideShareObject}
       />
     )
@@ -59,7 +59,7 @@ describe("ShareObjectModal", () => {
     const wrapper = shallow(
       <ShareObjectModal
         object={{ name: "obj1" }}
-        shareObjectDetails={{ show: true, object: "obj1", url: "test" }}
+        shareObjectDetails={{ show: true, object: "obj1", url: "test", showExpiryDate: true }}
       />
     )
     expect(
@@ -76,7 +76,7 @@ describe("ShareObjectModal", () => {
     const wrapper = shallow(
       <ShareObjectModal
         object={{ name: "obj1" }}
-        shareObjectDetails={{ show: true, object: "obj1", url: "test" }}
+        shareObjectDetails={{ show: true, object: "obj1", url: "test", showExpiryDate: true }}
         hideShareObject={hideShareObject}
         showCopyAlert={showCopyAlert}
       />
@@ -89,8 +89,15 @@ describe("ShareObjectModal", () => {
   describe("Update expiry values", () => {
     const props = {
       object: { name: "obj1" },
-      shareObjectDetails: { show: true, object: "obj1", url: "test" }
+      shareObjectDetails: { show: true, object: "obj1", url: "test", showExpiryDate: true }
     }
+
+    it("should not show expiry values if shared with public link", () => {
+      const shareObjectDetails = { show: true, object: "obj1", url: "test", showExpiryDate: false }
+      const wrapper = shallow(<ShareObjectModal {...props} shareObjectDetails={shareObjectDetails} />)
+      expect(wrapper.find('.set-expire').exists()).toEqual(false)
+    })
+
     it("should have default expiry values", () => {
       const wrapper = shallow(<ShareObjectModal {...props} />)
       expect(wrapper.state("expiry")).toEqual({

browser/app/js/objects/__tests__/actions.test.js

@@ -34,6 +34,7 @@ jest.mock("../../web", () => ({
     .mockReturnValueOnce(false)
     .mockReturnValueOnce(true)
     .mockReturnValueOnce(true)
+    .mockReturnValueOnce(true)
     .mockReturnValueOnce(false),
   ListObjects: jest.fn(({ bucketName }) => {
     if (bucketName === "test-deny") {
@@ -70,6 +71,16 @@ jest.mock("../../web", () => ({
     .mockImplementationOnce(() => {
       return Promise.resolve({ token: "test" })
     })
+    .mockImplementationOnce(() => {
+      return Promise.resolve({ token: "test" })
+    }),
+  GetBucketPolicy: jest.fn(({ bucketName, prefix }) => {
+    if (!bucketName) {
+      return Promise.reject({ message: "Invalid bucket" })
+    }
+    if (bucketName === 'test-public') return Promise.resolve({ policy: 'readonly' })
+    return Promise.resolve({})
+  })
 }))
 
 const middlewares = [thunk]
@@ -295,7 +306,8 @@ describe("Objects actions", () => {
         type: "objects/SET_SHARE_OBJECT",
         show: true,
         object: "b.txt",
-        url: "test"
+        url: "test",
+        showExpiryDate: true
       }
     ]
     store.dispatch(actionsObjects.showShareObject("b.txt", "test"))
@@ -321,14 +333,16 @@ describe("Objects actions", () => {
   it("creates objects/SET_SHARE_OBJECT when object is shared", () => {
     const store = mockStore({
       buckets: { currentBucket: "bk1" },
-      objects: { currentPrefix: "pre1/" }
+      objects: { currentPrefix: "pre1/" },
+      browser: { serverInfo: {} },
     })
     const expectedActions = [
       {
         type: "objects/SET_SHARE_OBJECT",
         show: true,
         object: "a.txt",
-        url: "https://test.com/bk1/pre1/b.txt"
+        url: "https://test.com/bk1/pre1/b.txt",
+        showExpiryDate: true
       },
       {
         type: "alert/SET",
@@ -347,10 +361,42 @@ describe("Objects actions", () => {
       })
   })
 
+  it("creates objects/SET_SHARE_OBJECT when object is shared with public link", () => {
+    const store = mockStore({
+      buckets: { currentBucket: "test-public" },
+      objects: { currentPrefix: "pre1/" },
+      browser: { serverInfo: { info: { domains: ['public.com'] }} },
+    })
+    const expectedActions = [
+      {
+        type: "objects/SET_SHARE_OBJECT",
+        show: true,
+        object: "a.txt",
+        url: "public.com/test-public/pre1/a.txt",
+        showExpiryDate: false
+      },
+      {
+        type: "alert/SET",
+        alert: {
+          type: "success",
+          message: "Object shared.",
+          id: alertActions.alertId
+        }
+      }
+    ]
+    return store
+      .dispatch(actionsObjects.shareObject("a.txt", 1, 0, 0))
+      .then(() => {
+        const actions = store.getActions()
+        expect(actions).toEqual(expectedActions)
+      })
+  })
+
   it("creates alert/SET when shareObject is failed", () => {
     const store = mockStore({
       buckets: { currentBucket: "" },
-      objects: { currentPrefix: "pre1/" }
+      objects: { currentPrefix: "pre1/" },
+      browser: { serverInfo: {} },
     })
     const expectedActions = [
       {
@@ -442,6 +488,34 @@ describe("Objects actions", () => {
     })
   })
 
+  it("should download prefix", () => {
+    const open = jest.fn()
+    const send = jest.fn()
+    const xhrMockClass = () => ({
+      open: open,
+      send: send
+    })
+    window.XMLHttpRequest = jest.fn().mockImplementation(xhrMockClass)
+
+    const store = mockStore({
+      buckets: { currentBucket: "bk1" },
+      objects: { currentPrefix: "pre1/" }
+    })
+    return store.dispatch(actionsObjects.downloadPrefix("pre2/")).then(() => {
+      const requestUrl = `${
+        location.origin
+      }${minioBrowserPrefix}/zip?token=test`
+      expect(open).toHaveBeenCalledWith("POST", requestUrl, true)
+      expect(send).toHaveBeenCalledWith(
+        JSON.stringify({
+          bucketName: "bk1",
+          prefix: "pre1/",
+          objects: ["pre2/"]
+        })
+      )
+    })
+  })
+
   it("creates objects/CHECKED_LIST_ADD action", () => {
     const store = mockStore()
     const expectedActions = [

browser/app/js/objects/actions.js

@@ -24,7 +24,6 @@ import {
 import { getCurrentBucket } from "../buckets/selectors"
 import { getCurrentPrefix, getCheckedList } from "./selectors"
 import * as alertActions from "../alert/actions"
-import * as bucketActions from "../buckets/actions"
 import {
   minioBrowserPrefix,
   SORT_BY_NAME,
@@ -33,6 +32,7 @@ import {
   SORT_ORDER_ASC,
   SORT_ORDER_DESC,
 } from "../constants"
+import { getServerInfo, hasServerPublicDomain } from '../browser/selectors'
 
 export const SET_LIST = "objects/SET_LIST"
 export const RESET_LIST = "objects/RESET_LIST"
@@ -222,19 +222,38 @@ export const deleteCheckedObjects = () => {
 
 export const shareObject = (object, days, hours, minutes) => {
   return function (dispatch, getState) {
+    const hasServerDomain = hasServerPublicDomain(getState())
     const currentBucket = getCurrentBucket(getState())
     const currentPrefix = getCurrentPrefix(getState())
     const objectName = `${currentPrefix}${object}`
     const expiry = days * 24 * 60 * 60 + hours * 60 * 60 + minutes * 60
     if (web.LoggedIn()) {
       return web
-        .PresignedGet({
-          host: location.host,
-          bucket: currentBucket,
-          object: objectName,
-          expiry: expiry,
+        .GetBucketPolicy({ bucketName: currentBucket, prefix: currentPrefix })
+        .catch(() => ({ policy: null }))
+        .then(({ policy }) => {
+          if (hasServerDomain && ['readonly', 'readwrite'].includes(policy)) {
+            const domain = getServerInfo(getState()).info.domains[0]
+            const url = `${domain}/${currentBucket}/${encodeURI(objectName)}`
+            dispatch(showShareObject(object, url, false))
+            dispatch(
+              alertActions.set({
+                type: "success",
+                message: "Object shared."
+              })
+            )
+          } else {
+            return web
+              .PresignedGet({
+                host: location.host,
+                bucket: currentBucket,
+                object: objectName,
+                expiry: expiry
+              })
+          }
         })
         .then((obj) => {
+          if (!obj) return
           dispatch(showShareObject(object, obj.url))
           dispatch(
             alertActions.set({
@@ -272,11 +291,12 @@ export const shareObject = (object, days, hours, minutes) => {
   }
 }
 
-export const showShareObject = (object, url) => ({
+export const showShareObject = (object, url, showExpiryDate = true) => ({
   type: SET_SHARE_OBJECT,
   show: true,
   object,
   url,
+  showExpiryDate,
 })
 
 export const hideShareObject = (object, url) => ({
@@ -340,6 +360,19 @@ export const downloadObject = (object) => {
   }
 }
 
+export const downloadPrefix = (object) => {
+  return function (dispatch, getState) {
+    return downloadObjects(
+      getCurrentBucket(getState()),
+      getCurrentPrefix(getState()),
+      [object],
+      `${object.slice(0, -1)}.zip`,
+      dispatch
+    )
+  }
+}
+
+
 export const checkObject = (object) => ({
   type: CHECKED_LIST_ADD,
   object,
@@ -356,21 +389,28 @@ export const resetCheckedList = () => ({
 
 export const downloadCheckedObjects = () => {
   return function (dispatch, getState) {
-    const state = getState()
+    return downloadObjects(
+      getCurrentBucket(getState()),
+      getCurrentPrefix(getState()),
+      getCheckedList(getState()),
+      null,
+      dispatch
+    )
+  }
+}
+
+const downloadObjects = (bucketName, prefix, objects, filename, dispatch) => {
     const req = {
-      bucketName: getCurrentBucket(state),
-      prefix: getCurrentPrefix(state),
-      objects: getCheckedList(state),
+      bucketName: bucketName,
+      prefix: prefix,
+      objects: objects,
     }
-    if (!web.LoggedIn()) {
-      const requestUrl = location.origin + "/minio/zip?token="
-      downloadZip(requestUrl, req, dispatch)
-    } else {
+    if (web.LoggedIn()) {
       return web
         .CreateURLToken()
         .then((res) => {
           const requestUrl = `${location.origin}${minioBrowserPrefix}/zip?token=${res.token}`
-          downloadZip(requestUrl, req, dispatch)
+          downloadZip(requestUrl, req, filename, dispatch)
         })
         .catch((err) =>
           dispatch(
@@ -380,11 +420,13 @@ export const downloadCheckedObjects = () => {
             })
           )
         )
+    } else {
+      const requestUrl = `${location.origin}${minioBrowserPrefix}/zip?token=`
+      downloadZip(requestUrl, req, filename, dispatch)
     }
-  }
 }
 
-const downloadZip = (url, req, dispatch) => {
+const downloadZip = (url, req, filename, dispatch) => {
   var anchor = document.createElement("a")
   document.body.appendChild(anchor)
 
@@ -402,7 +444,7 @@ const downloadZip = (url, req, dispatch) => {
       var separator = req.prefix.length > 1 ? "-" : ""
 
       anchor.href = blobUrl
-      anchor.download =
+      anchor.download = filename ||
         req.bucketName + separator + req.prefix.slice(0, -1) + ".zip"
 
       anchor.click()

browser/app/js/objects/reducer.js

@@ -89,7 +89,8 @@ export default (
         shareObject: {
           show: action.show,
           object: action.object,
-          url: action.url
+          url: action.url,
+          showExpiryDate: action.showExpiryDate
         }
       }
     case actionsObjects.CHECKED_LIST_ADD:

browser/package.json

@@ -28,71 +28,70 @@
   },
   "homepage": "https://github.com/minio/minio",
   "devDependencies": {
-    "async": "^1.5.2",
+    "async": "^3.2.0",
     "babel-cli": "^6.26.0",
     "babel-core": "^6.26.3",
-    "babel-jest": "^22.1.0",
+    "babel-jest": "^23.6.0",
     "babel-loader": "^7.1.2",
     "babel-plugin-syntax-object-rest-spread": "^6.13.0",
-    "babel-plugin-transform-object-rest-spread": "^6.8.0",
-    "babel-polyfill": "^6.23.0",
+    "babel-plugin-transform-object-rest-spread": "^6.26.0",
+    "babel-polyfill": "^6.26.0",
     "babel-preset-es2015": "^6.14.0",
     "babel-preset-react": "^6.11.1",
     "babel-register": "^6.26.0",
-    "copy-webpack-plugin": "^4.6.0",
-    "css-loader": "^0.23.1",
-    "enzyme": "^3.10.0",
-    "enzyme-adapter-react-16": "^1.1.1",
-    "esformatter": "^0.10.0",
-    "esformatter-jsx": "^7.4.1",
+    "copy-webpack-plugin": "^6.0.1",
+    "css-loader": "^3.5.3",
+    "enzyme": "^3.11.0",
+    "enzyme-adapter-react-16": "^1.15.2",
+    "esformatter": "^0.11.3",
+    "esformatter-jsx": "^8.0.1",
     "esformatter-jsx-ignore": "^1.0.6",
-    "html-webpack-plugin": "^3.2.0",
-    "jest": "^22.1.4",
-    "jest-enzyme": "^4.0.2",
-    "json-loader": "^0.5.4",
-    "less": "^3.9.0",
-    "less-loader": "^4.1.0",
-    "purgecss-webpack-plugin": "^1.4.0",
-    "style-loader": "^0.13.1",
-    "url-loader": "^0.5.7",
-    "webpack-cli": "^3.2.0",
-    "webpack-dev-server": "^3.1.14"
+    "html-webpack-plugin": "^4.3.0",
+    "jest": "^23.6.0",
+    "jest-enzyme": "^7.1.2",
+    "json-loader": "^0.5.7",
+    "less": "^3.11.1",
+    "less-loader": "^6.1.0",
+    "purgecss-webpack-plugin": "^2.2.0",
+    "style-loader": "^1.2.1",
+    "url-loader": "^4.1.0",
+    "webpack-cli": "^3.3.11",
+    "webpack-dev-server": "^3.11.0"
   },
   "dependencies": {
-    "@fortawesome/fontawesome-free": "^5.10.0",
+    "@fortawesome/fontawesome-free": "^5.13.0",
     "bootstrap": "^3.4.1",
-    "classnames": "^2.2.3",
-    "core-js": "^3.2.1",
-    "expect": "^1.20.2",
-    "glob-all": "^3.1.0",
-    "history": "^4.7.2",
+    "classnames": "^2.2.6",
+    "core-js": "^3.6.5",
+    "expect": "^26.0.1",
+    "glob-all": "^3.2.1",
+    "history": "^4.10.1",
     "humanize": "0.0.9",
     "identity-obj-proxy": "^3.0.0",
-    "json-loader": "^0.5.4",
     "jwt-decode": "^2.2.0",
-    "local-storage-fallback": "^4.0.2",
+    "local-storage-fallback": "^4.1.1",
     "material-design-iconic-font": "^2.2.0",
-    "mime-db": "^1.25.0",
-    "mime-types": "^2.1.13",
-    "moment": "^2.24.0",
-    "query-string": "^6.8.2",
-    "react": "^16.2.0",
-    "react-addons-test-utils": "^0.14.8",
-    "react-bootstrap": "^0.32.0",
-    "react-copy-to-clipboard": "^5.0.1",
+    "mime-db": "^1.44.0",
+    "mime-types": "^2.1.27",
+    "moment": "^2.26.0",
+    "query-string": "^6.12.1",
+    "react": "^16.13.1",
+    "react-addons-test-utils": "^15.6.2",
+    "react-bootstrap": "^0.32.4",
+    "react-copy-to-clipboard": "^5.0.2",
     "react-custom-scrollbars": "^4.2.1",
-    "react-dom": "^16.2.0",
-    "react-dropzone": "^4.2.3",
-    "react-infinite-scroller": "^1.0.6",
+    "react-dom": "^16.13.1",
+    "react-dropzone": "^4.3.0",
+    "react-infinite-scroller": "^1.2.4",
     "react-onclickout": "^2.0.8",
-    "react-redux": "^5.0.6",
-    "react-router-dom": "^4.2.0",
-    "redux": "^3.7.2",
-    "redux-mock-store": "^1.5.1",
-    "redux-thunk": "^2.2.0",
-    "reselect": "^3.0.1",
-    "superagent": "^3.8.2",
+    "react-redux": "^5.1.2",
+    "react-router-dom": "^5.2.0",
+    "redux": "^4.0.5",
+    "redux-mock-store": "^1.5.4",
+    "redux-thunk": "^2.3.0",
+    "reselect": "^4.0.0",
+    "superagent": "^5.2.2",
     "superagent-es6-promise": "^1.0.0",
-    "webpack": "^4.28.3"
+    "webpack": "^4.43.0"
   }
 }

browser/webpack.config.js

@@ -73,17 +73,17 @@ var exports = {
     },
     proxy: {
       '/minio/webrpc': {
-	target: 'http://localhost:9000',
-  secure: false,
-  headers: {'Host': "localhost:9000"}
+        target: 'http://localhost:9000',
+        secure: false,
+        headers: {'Host': "localhost:9000"}
       },
       '/minio/upload/*': {
-	target: 'http://localhost:9000',
-	secure: false
+        target: 'http://localhost:9000',
+        secure: false
       },
       '/minio/download/*': {
-	target: 'http://localhost:9000',
-	secure: false
+        target: 'http://localhost:9000',
+        secure: false
       },
       '/minio/zip': {
         target: 'http://localhost:9000',
@@ -92,7 +92,7 @@ var exports = {
     }
   },
   plugins: [
-    new CopyWebpackPlugin([
+    new CopyWebpackPlugin({patterns: [
       {from: 'app/css/loader.css'},
       {from: 'app/img/browsers/chrome.png'},
       {from: 'app/img/browsers/firefox.png'},
@@ -102,7 +102,7 @@ var exports = {
       {from: 'app/img/favicon/favicon-32x32.png'},
       {from: 'app/img/favicon/favicon-96x96.png'},
       {from: 'app/index.html'}
-    ]),
+    ]}),
     new webpack.ContextReplacementPlugin(/moment[\\\/]locale$/, /^\.\/(en)$/),
     new PurgecssPlugin({
       paths: glob.sync([

browser/webpack.production.config.js

@@ -67,7 +67,7 @@ var exports = {
     fs:'empty'
   },
   plugins: [
-    new CopyWebpackPlugin([
+    new CopyWebpackPlugin({patterns: [
       {from: 'app/css/loader.css'},
       {from: 'app/img/browsers/chrome.png'},
       {from: 'app/img/browsers/firefox.png'},
@@ -77,7 +77,7 @@ var exports = {
       {from: 'app/img/favicon/favicon-32x32.png'},
       {from: 'app/img/favicon/favicon-96x96.png'},
       {from: 'app/index.html'}
-    ]),
+    ]}),
     new webpack.ContextReplacementPlugin(/moment[\\\/]locale$/, /^\.\/(en)$/),
     new PurgecssPlugin({
       paths: glob.sync([

buildscripts/gateway-tests.sh

@@ -48,7 +48,7 @@ function main()
     SERVER_ENDPOINT=127.0.0.1:24240 ENABLE_HTTPS=0 ACCESS_KEY=minio \
                    SECRET_KEY=minio123 MINT_MODE="full" /mint/entrypoint.sh \
                    aws-sdk-go aws-sdk-java aws-sdk-php aws-sdk-ruby awscli \
-                   healthcheck mc minio-dotnet minio-java minio-js \
+                   healthcheck mc minio-dotnet minio-js \
                    minio-py s3cmd s3select security
     rv=$?
 

buildscripts/verify-build.sh

@@ -154,34 +154,6 @@ function run_test_erasure_sets() {
     return "$rv"
 }
 
-function run_test_dist_erasure_sets_ipv6()
-{
-    minio_pids=( $(start_minio_dist_erasure_sets_ipv6) )
-
-    export SERVER_ENDPOINT="[::1]:9000"
-
-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
-
-    for pid in "${minio_pids[@]}"; do
-        kill "$pid"
-    done
-    sleep 3
-
-    if [ "$rv" -ne 0 ]; then
-        for i in $(seq 0 9); do
-            echo "server$i log:"
-            cat "$WORK_DIR/dist-minio-v6-900$i.log"
-        done
-    fi
-
-    for i in $(seq 0 9); do
-        rm -f "$WORK_DIR/dist-minio-v6-900$i.log"
-    done
-
-    return "$rv"
-}
-
 function run_test_zone_erasure_sets()
 {
     minio_pids=( $(start_minio_zone_erasure_sets) )

cmd/acl-handlers.go

@@ -100,18 +100,18 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
 		}
 
 		if len(acl.AccessControlList.Grants) == 0 {
-			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
 		if acl.AccessControlList.Grants[0].Permission != "FULL_CONTROL" {
-			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
 	if aclHeader != "" && aclHeader != "private" {
-		writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -159,6 +159,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 		},
 		Permission: "FULL_CONTROL",
 	})
+
 	if err := xml.NewEncoder(w).Encode(acl); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
@@ -214,18 +215,18 @@ func (api objectAPIHandlers) PutObjectACLHandler(w http.ResponseWriter, r *http.
 		}
 
 		if len(acl.AccessControlList.Grants) == 0 {
-			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
 		if acl.AccessControlList.Grants[0].Permission != "FULL_CONTROL" {
-			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
 	if aclHeader != "" && aclHeader != "private" {
-		writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/admin-handlers-config-kv.go

@@ -35,50 +35,47 @@ import (
 	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/auth"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
 )
 
-func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *http.Request) ObjectLayer {
+func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *http.Request) (auth.Credentials, ObjectLayer) {
 	// Get current object layer instance.
 	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return nil
+		return auth.Credentials{}, nil
 	}
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.ConfigUpdateAdminAction, "")
+	cred, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.ConfigUpdateAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
-		return nil
+		return cred, nil
 	}
 
-	return objectAPI
+	return cred, objectAPI
 }
 
 // DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
 func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "DelConfigKVHandler")
+	ctx := newContext(r, w, "DeleteConfigKV")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	defer logger.AuditLog(w, r, "DeleteConfigKV", mustGetClaimsFromToken(r))
+
+	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
 		return
 	}
 
-	password := globalActiveCred.SecretKey
+	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		logger.LogIf(ctx, err, logger.Application)
@@ -105,26 +102,22 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 // SetConfigKVHandler - PUT /minio/admin/v3/set-config-kv
 func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetConfigKVHandler")
+	ctx := newContext(r, w, "SetConfigKV")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	defer logger.AuditLog(w, r, "SetConfigKV", mustGetClaimsFromToken(r))
+
+	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
 		return
 	}
 
-	password := globalActiveCred.SecretKey
+	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		logger.LogIf(ctx, err, logger.Application)
@@ -170,9 +163,11 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 // GetConfigKVHandler - GET /minio/admin/v3/get-config-kv?key={key}
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetConfigKVHandler")
+	ctx := newContext(r, w, "GetConfigKV")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	defer logger.AuditLog(w, r, "GetConfigKV", mustGetClaimsFromToken(r))
+
+	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
@@ -195,7 +190,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	password := globalActiveCred.SecretKey
+	password := cred.SecretKey
 	econfigData, err := madmin.EncryptData(password, buf.Bytes())
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -206,9 +201,11 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 }
 
 func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ClearConfigHistoryKVHandler")
+	ctx := newContext(r, w, "ClearConfigHistoryKV")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	defer logger.AuditLog(w, r, "ClearConfigHistoryKV", mustGetClaimsFromToken(r))
+
+	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
@@ -241,9 +238,11 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
 
 // RestoreConfigHistoryKVHandler - restores a config with KV settings for the given KV id.
 func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "RestoreConfigHistoryKVHandler")
+	ctx := newContext(r, w, "RestoreConfigHistoryKV")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	defer logger.AuditLog(w, r, "RestoreConfigHistoryKV", mustGetClaimsFromToken(r))
+
+	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
@@ -287,9 +286,11 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 
 // ListConfigHistoryKVHandler - lists all the KV ids.
 func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListConfigHistoryKVHandler")
+	ctx := newContext(r, w, "ListConfigHistoryKV")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	defer logger.AuditLog(w, r, "ListConfigHistoryKV", mustGetClaimsFromToken(r))
+
+	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
@@ -313,7 +314,7 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	password := globalActiveCred.SecretKey
+	password := cred.SecretKey
 	econfigData, err := madmin.EncryptData(password, data)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -325,9 +326,11 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 
 // HelpConfigKVHandler - GET /minio/admin/v3/help-config-kv?subSys={subSys}&key={key}
 func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "HelpConfigKVHandler")
+	ctx := newContext(r, w, "HelpConfigKV")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	defer logger.AuditLog(w, r, "HelpHistoryKV", mustGetClaimsFromToken(r))
+
+	_, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
@@ -351,26 +354,22 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
 
 // SetConfigHandler - PUT /minio/admin/v3/config
 func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetConfigHandler")
+	ctx := newContext(r, w, "SetConfig")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	defer logger.AuditLog(w, r, "SetConfig", mustGetClaimsFromToken(r))
+
+	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
 		return
 	}
 
-	password := globalActiveCred.SecretKey
+	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		logger.LogIf(ctx, err, logger.Application)
@@ -412,9 +411,11 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 // GetConfigHandler - GET /minio/admin/v3/config
 // Get config.json of this minio setup.
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetConfigHandler")
+	ctx := newContext(r, w, "GetConfig")
 
-	objectAPI := validateAdminReqConfigKV(ctx, w, r)
+	defer logger.AuditLog(w, r, "GetConfig", mustGetClaimsFromToken(r))
+
+	cred, objectAPI := validateAdminReqConfigKV(ctx, w, r)
 	if objectAPI == nil {
 		return
 	}
@@ -471,7 +472,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 		}
 	}
 
-	password := globalActiveCred.SecretKey
+	password := cred.SecretKey
 	econfigData, err := madmin.EncryptData(password, []byte(s.String()))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)

cmd/admin-handlers-users.go

@@ -55,17 +55,13 @@ func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.R
 func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveUser")
 
+	defer logger.AuditLog(w, r, "RemoveUser", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	vars := mux.Vars(r)
 	accessKey := vars["accessKey"]
 
@@ -97,6 +93,8 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListUsers")
 
+	defer logger.AuditLog(w, r, "ListUsers", mustGetClaimsFromToken(r))
+
 	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.ListUsersAdminAction)
 	if objectAPI == nil {
 		return
@@ -129,6 +127,8 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetUserInfo")
 
+	defer logger.AuditLog(w, r, "GetUserInfo", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetUserAdminAction)
 	if objectAPI == nil {
 		return
@@ -156,6 +156,8 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "UpdateGroupMembers")
 
+	defer logger.AuditLog(w, r, "UpdateGroupMembers", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.AddUserToGroupAdminAction)
 	if objectAPI == nil {
 		return
@@ -199,6 +201,8 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetGroup")
 
+	defer logger.AuditLog(w, r, "GetGroup", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetGroupAdminAction)
 	if objectAPI == nil {
 		return
@@ -226,6 +230,8 @@ func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListGroups")
 
+	defer logger.AuditLog(w, r, "ListGroups", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListGroupsAdminAction)
 	if objectAPI == nil {
 		return
@@ -250,6 +256,8 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetGroupStatus")
 
+	defer logger.AuditLog(w, r, "SetGroupStatus", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.EnableGroupAdminAction)
 	if objectAPI == nil {
 		return
@@ -285,17 +293,13 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetUserStatus")
 
+	defer logger.AuditLog(w, r, "SetUserStatus", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.EnableUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	vars := mux.Vars(r)
 	accessKey := vars["accessKey"]
 	status := vars["status"]
@@ -324,17 +328,13 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddUser")
 
+	defer logger.AuditLog(w, r, "AddUser", mustGetClaimsFromToken(r))
+
 	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.CreateUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	vars := mux.Vars(r)
 	accessKey := vars["accessKey"]
 
@@ -383,6 +383,8 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddServiceAccount")
 
+	defer logger.AuditLog(w, r, "AddServiceAccount", mustGetClaimsFromToken(r))
+
 	// Get current object layer instance.
 	objectAPI := newObjectLayerWithoutSafeModeFn()
 	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
@@ -397,55 +399,37 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	}
 
 	password := cred.SecretKey
-	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
 
 	var createReq madmin.AddServiceAccountReq
-	if err = json.Unmarshal(configBytes, &createReq); err != nil {
+	if err = json.Unmarshal(reqBytes, &createReq); err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
 
-	if createReq.Parent == "" {
-		apiErr := APIError{
-			Code:           "XMinioAdminInvalidArgument",
-			Description:    "Service account parent cannot be empty",
-			HTTPStatusCode: http.StatusBadRequest,
-		}
-		writeErrorResponseJSON(ctx, w, apiErr, r.URL)
+	// Disallow creating service accounts by root user.
+	if owner {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminAccountNotEligible), r.URL)
 		return
 	}
 
-	// Disallow creating service accounts by root user as well.
-	if createReq.Parent == globalActiveCred.AccessKey {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddServiceAccountInvalidArgument), r.URL)
-		return
+	parentUser := cred.AccessKey
+	if cred.ParentUser != "" {
+		parentUser = cred.ParentUser
 	}
 
-	// Disallow creating service accounts by users who are not the requested parent.
-	// this restriction is not required for Owner account i.e root user.
-	if createReq.Parent != cred.AccessKey && !owner {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddServiceAccountInvalidParent), r.URL)
-		return
-	}
-
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
-	creds, err := globalIAMSys.NewServiceAccount(ctx, createReq.Parent, createReq.Policy)
+	newCred, err := globalIAMSys.NewServiceAccount(ctx, parentUser, createReq.Policy)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Notify all other Minio peers to reload user
-	for _, nerr := range globalNotificationSys.LoadUser(creds.AccessKey, false) {
+	// Notify all other Minio peers to reload user the service account
+	for _, nerr := range globalNotificationSys.LoadServiceAccount(newCred.AccessKey) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 			logger.LogIf(ctx, nerr.Err)
@@ -454,8 +438,8 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 
 	var createResp = madmin.AddServiceAccountResp{
 		Credentials: auth.Credentials{
-			AccessKey: creds.AccessKey,
-			SecretKey: creds.SecretKey,
+			AccessKey: newCred.AccessKey,
+			SecretKey: newCred.SecretKey,
 		},
 	}
 
@@ -465,19 +449,241 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	econfigData, err := madmin.EncryptData(password, data)
+	encryptedData, err := madmin.EncryptData(password, data)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	writeSuccessResponseJSON(w, econfigData)
+	writeSuccessResponseJSON(w, encryptedData)
+}
+
+// ListServiceAccounts - GET /minio/admin/v3/list-service-accounts
+func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListServiceAccounts")
+
+	defer logger.AuditLog(w, r, "ListServiceAccounts", mustGetClaimsFromToken(r))
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	cred, _, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	// Disallow creating service accounts by root user.
+	if owner {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminAccountNotEligible), r.URL)
+		return
+	}
+
+	parentUser := cred.AccessKey
+	if cred.ParentUser != "" {
+		parentUser = cred.ParentUser
+	}
+
+	serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, parentUser)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	var listResp = madmin.ListServiceAccountsResp{
+		Accounts: serviceAccounts,
+	}
+
+	data, err := json.Marshal(listResp)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, encryptedData)
+}
+
+// DeleteServiceAccount - DELETE /minio/admin/v3/delete-service-account
+func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteServiceAccount")
+
+	defer logger.AuditLog(w, r, "DeleteServiceAccount", mustGetClaimsFromToken(r))
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	cred, _, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	// Disallow creating service accounts by root user.
+	if owner {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminAccountNotEligible), r.URL)
+		return
+	}
+
+	serviceAccount := mux.Vars(r)["accessKey"]
+	if serviceAccount == "" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
+		return
+	}
+
+	user, err := globalIAMSys.GetServiceAccountParent(ctx, serviceAccount)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	parentUser := cred.AccessKey
+	if cred.ParentUser != "" {
+		parentUser = cred.ParentUser
+	}
+
+	if parentUser != user || user == "" {
+		// The service account belongs to another user but return not
+		// found error to mitigate brute force attacks. or the
+		// serviceAccount doesn't exist.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServiceAccountNotFound), r.URL)
+		return
+	}
+
+	err = globalIAMSys.DeleteServiceAccount(ctx, serviceAccount)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessNoContent(w)
+}
+
+// AccountUsageInfoHandler returns usage
+func (a adminAPIHandlers) AccountUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AccountUsageInfo")
+
+	defer logger.AuditLog(w, r, "AccountUsageInfo", mustGetClaimsFromToken(r))
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerWithoutSafeModeFn()
+	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	// Set prefix value for "s3:prefix" policy conditionals.
+	r.Header.Set("prefix", "")
+
+	// Set delimiter value for "s3:delimiter" policy conditionals.
+	r.Header.Set("delimiter", SlashSeparator)
+
+	isAllowedAccess := func(bucketName string) (rd, wr bool) {
+		// Use the following trick to filter in place
+		// https://github.com/golang/go/wiki/SliceTricks#filter-in-place
+		if globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     cred.AccessKey,
+			Action:          iampolicy.ListBucketAction,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			IsOwner:         owner,
+			ObjectName:      "",
+			Claims:          claims,
+		}) {
+			rd = true
+		}
+
+		if globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     cred.AccessKey,
+			Action:          iampolicy.PutObjectAction,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+			IsOwner:         owner,
+			ObjectName:      "",
+			Claims:          claims,
+		}) {
+			wr = true
+		}
+
+		return rd, wr
+	}
+
+	buckets, err := objectAPI.ListBuckets(ctx)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Load the latest calculated data usage
+	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+	if err != nil {
+		// log the error, continue with the accounting response
+		logger.LogIf(ctx, err)
+	}
+
+	accountName := cred.AccessKey
+	if cred.ParentUser != "" {
+		accountName = cred.ParentUser
+	}
+
+	acctInfo := madmin.AccountUsageInfo{
+		AccountName: accountName,
+	}
+
+	for _, bucket := range buckets {
+		rd, wr := isAllowedAccess(bucket.Name)
+		if rd || wr {
+			var size uint64
+			// Fetch the data usage of the current bucket
+			if !dataUsageInfo.LastUpdate.IsZero() {
+				size = dataUsageInfo.BucketsUsage[bucket.Name].Size
+			}
+			acctInfo.Buckets = append(acctInfo.Buckets, madmin.BucketUsageInfo{
+				Name:    bucket.Name,
+				Created: bucket.Created,
+				Size:    size,
+				Access: madmin.AccountAccess{
+					Read:  rd,
+					Write: wr,
+				},
+			})
+		}
+	}
+
+	usageInfoJSON, err := json.Marshal(acctInfo)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, usageInfoJSON)
 }
 
 // InfoCannedPolicyV2 - GET /minio/admin/v2/info-canned-policy?name={policyName}
 func (a adminAPIHandlers) InfoCannedPolicyV2(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicyV2")
 
+	defer logger.AuditLog(w, r, "InfoCannedPolicyV2", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
 	if objectAPI == nil {
 		return
@@ -503,6 +709,8 @@ func (a adminAPIHandlers) InfoCannedPolicyV2(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicy")
 
+	defer logger.AuditLog(w, r, "InfoCannedPolicy", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
 	if objectAPI == nil {
 		return
@@ -522,6 +730,8 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) ListCannedPoliciesV2(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListCannedPoliciesV2")
 
+	defer logger.AuditLog(w, r, "ListCannedPoliciesV2", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
@@ -554,6 +764,8 @@ func (a adminAPIHandlers) ListCannedPoliciesV2(w http.ResponseWriter, r *http.Re
 func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListCannedPolicies")
 
+	defer logger.AuditLog(w, r, "ListCannedPolicies", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
@@ -586,6 +798,8 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveCannedPolicy")
 
+	defer logger.AuditLog(w, r, "RemoveCannedPolicy", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.DeletePolicyAdminAction)
 	if objectAPI == nil {
 		return
@@ -594,12 +808,6 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 	vars := mux.Vars(r)
 	policyName := vars["name"]
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	if err := globalIAMSys.DeletePolicy(policyName); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -618,6 +826,8 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AddCannedPolicy")
 
+	defer logger.AuditLog(w, r, "AddCannedPolicy", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.CreatePolicyAdminAction)
 	if objectAPI == nil {
 		return
@@ -626,12 +836,6 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 	vars := mux.Vars(r)
 	policyName := vars["name"]
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	// Error out if Content-Length is missing.
 	if r.ContentLength <= 0 {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
@@ -674,6 +878,8 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetPolicyForUserOrGroup")
 
+	defer logger.AuditLog(w, r, "SetPolicyForUserOrGroup", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
 	if objectAPI == nil {
 		return
@@ -684,12 +890,6 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 	entityName := vars["userOrGroup"]
 	isGroup := vars["isGroup"] == "true"
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
-		return
-	}
-
 	if !isGroup {
 		ok, err := globalIAMSys.IsTempUser(entityName)
 		if err != nil && err != errNoSuchUser {

cmd/admin-handlers.go

@@ -34,7 +34,6 @@ import (
 	"strings"
 	"time"
 
-	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
 
 	"github.com/minio/minio/cmd/config"
@@ -44,7 +43,6 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/message/log"
 	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/event/target"
 	"github.com/minio/minio/pkg/handlers"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
@@ -54,7 +52,6 @@ import (
 
 const (
 	maxEConfigJSONSize = 262272
-	defaultNetPerfSize = 100 * humanize.MiByte
 )
 
 // Type-safe query params.
@@ -103,6 +100,8 @@ func updateServer(updateURL, sha256Hex string, latestReleaseTime time.Time) (us
 func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerUpdate")
 
+	defer logger.AuditLog(w, r, "ServerUpdate", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerUpdateAdminAction)
 	if objectAPI == nil {
 		return
@@ -176,20 +175,17 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 	}
 }
 
-// ServiceActionHandler - POST /minio/admin/v3/service?action={action}
+// ServiceHandler - POST /minio/admin/v3/service?action={action}
 // ----------
 // restarts/stops minio server gracefully. In a distributed setup,
-func (a adminAPIHandlers) ServiceActionHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ServiceAction")
+func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "Service")
+
+	defer logger.AuditLog(w, r, "Service", mustGetClaimsFromToken(r))
 
 	vars := mux.Vars(r)
 	action := vars["action"]
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, "")
-	if objectAPI == nil {
-		return
-	}
-
 	var serviceSig serviceSignal
 	switch madmin.ServiceAction(action) {
 	case madmin.ServiceActionRestart:
@@ -202,6 +198,16 @@ func (a adminAPIHandlers) ServiceActionHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
+	var objectAPI ObjectLayer
+	if serviceSig == serviceRestart {
+		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceRestartAdminAction)
+	} else {
+		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceStopAdminAction)
+	}
+	if objectAPI == nil {
+		return
+	}
+
 	// Notify all other MinIO peers signal service.
 	for _, nerr := range globalNotificationSys.SignalService(serviceSig) {
 		if nerr.Err != nil {
@@ -270,12 +276,16 @@ type ServerInfo struct {
 // Get server information
 func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StorageInfo")
+
+	defer logger.AuditLog(w, r, "StorageInfo", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.StorageInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
-	storageInfo := objectAPI.StorageInfo(ctx, false)
+	// ignores any errors here.
+	storageInfo, _ := objectAPI.StorageInfo(ctx, false)
 
 	// Marshal API response
 	jsonBytes, err := json.Marshal(storageInfo)
@@ -295,6 +305,9 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 // Get server/cluster data usage info
 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DataUsageInfo")
+
+	defer logger.AuditLog(w, r, "DataUsageInfo", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction)
 	if objectAPI == nil {
 		return
@@ -315,67 +328,6 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
 	writeSuccessResponseJSON(w, dataUsageInfoJSON)
 }
 
-func (a adminAPIHandlers) AccountingUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AccountingUsageInfo")
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AccountingUsageInfoAdminAction)
-	if objectAPI == nil {
-		return
-	}
-
-	var accountingUsageInfo = make(map[string]madmin.BucketAccountingUsage)
-
-	buckets, err := objectAPI.ListBuckets(ctx)
-	if err != nil {
-		// writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	users, err := globalIAMSys.ListUsers()
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Load the latest calculated data usage
-	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
-	if err != nil {
-		logger.LogIf(ctx, err)
-	}
-
-	// Calculate for each bucket, which users are allowed to access to it
-	for _, bucket := range buckets {
-		bucketUsageInfo := madmin.BucketAccountingUsage{}
-
-		// Fetch the data usage of the current bucket
-		if !dataUsageInfo.LastUpdate.IsZero() && dataUsageInfo.BucketsSizes != nil {
-			bucketUsageInfo.Size = dataUsageInfo.BucketsSizes[bucket.Name]
-		}
-
-		for user := range users {
-			rd, wr, custom := globalIAMSys.GetAccountAccess(user, bucket.Name)
-			if rd || wr || custom {
-				bucketUsageInfo.AccessList = append(bucketUsageInfo.AccessList, madmin.AccountAccess{
-					AccountName: user,
-					Read:        rd,
-					Write:       wr,
-					Custom:      custom,
-				})
-			}
-		}
-
-		accountingUsageInfo[bucket.Name] = bucketUsageInfo
-	}
-
-	usageInfoJSON, err := json.Marshal(accountingUsageInfo)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	writeSuccessResponseJSON(w, usageInfoJSON)
-}
-
 func newLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
 	entry := &madmin.LockEntry{
 		Timestamp:  l.Timestamp,
@@ -432,6 +384,8 @@ type PeerLocks struct {
 func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "TopLocks")
 
+	defer logger.AuditLog(w, r, "TopLocks", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.TopLocksAdminAction)
 	if objectAPI == nil {
 		return
@@ -477,6 +431,8 @@ type StartProfilingResult struct {
 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StartProfiling")
 
+	defer logger.AuditLog(w, r, "StartProfiling", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
 	if objectAPI == nil {
 		return
@@ -573,6 +529,8 @@ func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DownloadProfiling")
 
+	defer logger.AuditLog(w, r, "DownloadProfiling", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
 	if objectAPI == nil {
 		return
@@ -665,6 +623,8 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "Heal")
 
+	defer logger.AuditLog(w, r, "Heal", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
 	if objectAPI == nil {
 		return
@@ -690,13 +650,15 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 
 	// Define a closure to start sending whitespace to client
 	// after 10s unless a response item comes in
-	keepConnLive := func(w http.ResponseWriter, respCh chan healResp) {
+	keepConnLive := func(w http.ResponseWriter, r *http.Request, respCh chan healResp) {
 		ticker := time.NewTicker(time.Second * 10)
 		defer ticker.Stop()
 		started := false
 	forLoop:
 		for {
 			select {
+			case <-r.Context().Done():
+				return
 			case <-ticker.C:
 				if !started {
 					// Start writing response to client
@@ -746,8 +708,8 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	// find number of disks in the setup
-	info := objectAPI.StorageInfo(ctx, false)
+	// find number of disks in the setup, ignore any errors here.
+	info, _ := objectAPI.StorageInfo(ctx, false)
 	numDisks := info.Backend.OfflineDisks.Sum() + info.Backend.OnlineDisks.Sum()
 
 	healPath := pathJoin(hip.bucket, hip.objPrefix)
@@ -792,7 +754,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 			respCh <- hr
 		}()
 	case hip.clientToken == "":
-		nh := newHealSequence(hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), numDisks, hip.hs, hip.forceStart)
+		nh := newHealSequence(GlobalContext, hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), numDisks, hip.hs, hip.forceStart)
 		go func() {
 			respBytes, apiErr, errMsg := globalAllHealState.LaunchNewHealSequence(nh)
 			hr := healResp{respBytes, apiErr, errMsg}
@@ -803,12 +765,14 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	// Due to the force-starting functionality, the Launch
 	// call above can take a long time - to keep the
 	// connection alive, we start sending whitespace
-	keepConnLive(w, respCh)
+	keepConnLive(w, r, respCh)
 }
 
 func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HealBackgroundStatus")
 
+	defer logger.AuditLog(w, r, "HealBackgroundStatus", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
 	if objectAPI == nil {
 		return
@@ -935,7 +899,7 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 			HTTPStatusCode: e.StatusCode,
 		}
 	default:
-		if err == errConfigNotFound {
+		if errors.Is(err, errConfigNotFound) {
 			apiErr = APIError{
 				Code:           "XMinioConfigError",
 				Description:    err.Error(),
@@ -970,6 +934,7 @@ func mustTrace(entry interface{}, trcAll, errOnly bool) bool {
 // The handler sends http trace to the connected HTTP client.
 func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HTTPTrace")
+
 	trcAll := r.URL.Query().Get("all") == "true"
 	trcErr := r.URL.Query().Get("err") == "true"
 
@@ -982,16 +947,13 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set(xhttp.ContentType, "text/event-stream")
 
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
 	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
 	traceCh := make(chan interface{}, 4000)
 
-	peers := getRestClients(globalEndpoints)
+	peers := newPeerRestClients(globalEndpoints)
 
-	globalHTTPTrace.Subscribe(traceCh, doneCh, func(entry interface{}) bool {
+	globalHTTPTrace.Subscribe(traceCh, ctx.Done(), func(entry interface{}) bool {
 		return mustTrace(entry, trcAll, trcErr)
 	})
 
@@ -999,7 +961,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		if peer == nil {
 			continue
 		}
-		peer.Trace(traceCh, doneCh, trcAll, trcErr)
+		peer.Trace(traceCh, ctx.Done(), trcAll, trcErr)
 	}
 
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
@@ -1018,7 +980,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			w.(http.Flusher).Flush()
-		case <-GlobalServiceDoneCh:
+		case <-ctx.Done():
 			return
 		}
 	}
@@ -1028,6 +990,8 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ConsoleLog")
 
+	defer logger.AuditLog(w, r, "ConsoleLog", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConsoleLogAdminAction)
 	if objectAPI == nil {
 		return
@@ -1052,20 +1016,18 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 	w.Header().Add("Connection", "close")
 	w.Header().Set(xhttp.ContentType, "text/event-stream")
 
-	doneCh := make(chan struct{})
-	defer close(doneCh)
 	logCh := make(chan interface{}, 4000)
 
-	peers := getRestClients(globalEndpoints)
+	peers := newPeerRestClients(globalEndpoints)
 
-	globalConsoleSys.Subscribe(logCh, doneCh, node, limitLines, logKind, nil)
+	globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil)
 
 	for _, peer := range peers {
 		if peer == nil {
 			continue
 		}
 		if node == "" || strings.EqualFold(peer.host.Name, node) {
-			peer.ConsoleLog(logCh, doneCh)
+			peer.ConsoleLog(logCh, ctx.Done())
 		}
 	}
 
@@ -1089,7 +1051,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 				return
 			}
 			w.(http.Flusher).Flush()
-		case <-GlobalServiceDoneCh:
+		case <-ctx.Done():
 			return
 		}
 	}
@@ -1097,7 +1059,9 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 
 // KMSKeyStatusHandler - GET /minio/admin/v3/kms/key/status?key-id=<master-key-id>
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "KMSKeyStatusHandler")
+	ctx := newContext(r, w, "KMSKeyStatus")
+
+	defer logger.AuditLog(w, r, "KMSKeyStatus", mustGetClaimsFromToken(r))
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
@@ -1169,71 +1133,48 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 // Get server on-board diagnostics
 func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "OBDInfo")
+
+	defer logger.AuditLog(w, r, "OBDInfo", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.OBDInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
 	vars := mux.Vars(r)
-	pulse := make(chan struct{})
-	obdDone := make(chan struct{})
 	obdInfo := madmin.OBDInfo{}
+	obdInfoCh := make(chan madmin.OBDInfo)
 
 	enc := json.NewEncoder(w)
-	doPartialWrite := func() {
-		logger.LogIf(ctx, enc.Encode(obdInfo))
+	partialWrite := func(oinfo madmin.OBDInfo) {
+		obdInfoCh <- oinfo
 	}
 
-	partialWrite := func() {
-		pulse <- struct{}{}
-	}
-
-	finish := func() {
-		obdDone <- struct{}{}
-	}
+	setCommonHeaders(w)
+	w.Header().Set(xhttp.ContentType, "text/event-stream")
+	w.WriteHeader(http.StatusOK)
 
 	errResp := func(err error) {
 		errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(),
 			w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
 		encodedErrorResponse := encodeResponse(errorResponse)
 		obdInfo.Error = string(encodedErrorResponse)
-		finish()
+		logger.LogIf(ctx, enc.Encode(obdInfo))
 	}
 
 	deadline := 3600 * time.Second
-	deadlineStr := r.URL.Query().Get("deadline")
-	if deadlineStr != "" {
+	if dstr := r.URL.Query().Get("deadline"); dstr != "" {
 		var err error
-		deadline, err = time.ParseDuration(deadlineStr)
+		deadline, err = time.ParseDuration(dstr)
 		if err != nil {
 			errResp(err)
 			return
 		}
 	}
-	deadlinedCtx, cancel := context.WithDeadline(ctx, time.Now().Add(deadline))
 
-	setCommonHeaders(w)
-	w.Header().Set(xhttp.ContentType, string(mimeJSON))
-	w.WriteHeader(http.StatusOK)
+	deadlinedCtx, cancel := context.WithTimeout(ctx, deadline)
 
-	ticker := time.NewTicker(30 * time.Second)
-	defer ticker.Stop()
-
-	go func() {
-	loop:
-		for {
-			select {
-			case <-ticker.C:
-				doPartialWrite()
-			case <-pulse:
-				doPartialWrite()
-			case <-obdDone:
-				break loop
-			}
-		}
-		w.(http.Flusher).Flush()
-		cancel()
-	}()
+	defer cancel()
 
 	nsLock := objectAPI.NewNSLock(deadlinedCtx, minioMetaBucket, "obd-in-progress")
 	if err := nsLock.GetLock(newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
@@ -1242,89 +1183,125 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	}
 	defer nsLock.Unlock()
 
-	if cpu, ok := vars["syscpu"]; ok && cpu == "true" {
-		cpuInfo := getLocalCPUOBDInfo(deadlinedCtx)
+	go func() {
+		defer close(obdInfoCh)
 
-		obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo)
-		obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, globalNotificationSys.CPUOBDInfo(deadlinedCtx)...)
-		partialWrite()
-	}
+		if cpu, ok := vars["syscpu"]; ok && cpu == "true" {
+			cpuInfo := getLocalCPUOBDInfo(deadlinedCtx, r)
 
-	if diskHw, ok := vars["sysdiskhw"]; ok && diskHw == "true" {
-		diskHwInfo := getLocalDiskHwOBD(deadlinedCtx)
-
-		obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo)
-		obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, globalNotificationSys.DiskHwOBDInfo(deadlinedCtx)...)
-		partialWrite()
-	}
-
-	if osInfo, ok := vars["sysosinfo"]; ok && osInfo == "true" {
-		osInfo := getLocalOsInfoOBD(deadlinedCtx)
-
-		obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo)
-		obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, globalNotificationSys.OsOBDInfo(deadlinedCtx)...)
-		partialWrite()
-	}
-
-	if mem, ok := vars["sysmem"]; ok && mem == "true" {
-		memInfo := getLocalMemOBD(deadlinedCtx)
-
-		obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo)
-		obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, globalNotificationSys.MemOBDInfo(deadlinedCtx)...)
-		partialWrite()
-	}
-
-	if proc, ok := vars["sysprocess"]; ok && proc == "true" {
-		procInfo := getLocalProcOBD(deadlinedCtx)
-
-		obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo)
-		obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, globalNotificationSys.ProcOBDInfo(deadlinedCtx)...)
-		partialWrite()
-	}
-
-	if config, ok := vars["minioconfig"]; ok && config == "true" {
-		cfg, err := readServerConfig(ctx, objectAPI)
-		logger.LogIf(ctx, err)
-		obdInfo.Minio.Config = cfg
-		partialWrite()
-	}
-
-	if drive, ok := vars["perfdrive"]; ok && drive == "true" {
-		// Get drive obd details from local server's drive(s)
-		driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r)
-		driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r)
-
-		errStr := ""
-		if driveOBDSerial.Error != "" {
-			errStr = "serial: " + driveOBDSerial.Error
-		}
-		if driveOBDParallel.Error != "" {
-			errStr = errStr + " parallel: " + driveOBDParallel.Error
+			obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, cpuInfo)
+			obdInfo.Sys.CPUInfo = append(obdInfo.Sys.CPUInfo, globalNotificationSys.CPUOBDInfo(deadlinedCtx)...)
+			partialWrite(obdInfo)
 		}
 
-		driveOBD := madmin.ServerDrivesOBDInfo{
-			Addr:     driveOBDSerial.Addr,
-			Serial:   driveOBDSerial.Serial,
-			Parallel: driveOBDParallel.Parallel,
-			Error:    errStr,
+		if diskHw, ok := vars["sysdiskhw"]; ok && diskHw == "true" {
+			diskHwInfo := getLocalDiskHwOBD(deadlinedCtx, r)
+
+			obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, diskHwInfo)
+			obdInfo.Sys.DiskHwInfo = append(obdInfo.Sys.DiskHwInfo, globalNotificationSys.DiskHwOBDInfo(deadlinedCtx)...)
+			partialWrite(obdInfo)
 		}
-		obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBD)
 
-		// Notify all other MinIO peers to report drive obd numbers
-		driveOBDs := globalNotificationSys.DriveOBDInfo(deadlinedCtx)
-		obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBDs...)
+		if osInfo, ok := vars["sysosinfo"]; ok && osInfo == "true" {
+			osInfo := getLocalOsInfoOBD(deadlinedCtx, r)
 
-		partialWrite()
+			obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, osInfo)
+			obdInfo.Sys.OsInfo = append(obdInfo.Sys.OsInfo, globalNotificationSys.OsOBDInfo(deadlinedCtx)...)
+			partialWrite(obdInfo)
+		}
+
+		if mem, ok := vars["sysmem"]; ok && mem == "true" {
+			memInfo := getLocalMemOBD(deadlinedCtx, r)
+
+			obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, memInfo)
+			obdInfo.Sys.MemInfo = append(obdInfo.Sys.MemInfo, globalNotificationSys.MemOBDInfo(deadlinedCtx)...)
+			partialWrite(obdInfo)
+		}
+
+		if proc, ok := vars["sysprocess"]; ok && proc == "true" {
+			procInfo := getLocalProcOBD(deadlinedCtx, r)
+
+			obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, procInfo)
+			obdInfo.Sys.ProcInfo = append(obdInfo.Sys.ProcInfo, globalNotificationSys.ProcOBDInfo(deadlinedCtx)...)
+			partialWrite(obdInfo)
+		}
+
+		if config, ok := vars["minioconfig"]; ok && config == "true" {
+			cfg, err := readServerConfig(ctx, objectAPI)
+			logger.LogIf(ctx, err)
+			obdInfo.Minio.Config = cfg
+			partialWrite(obdInfo)
+		}
+
+		if drive, ok := vars["perfdrive"]; ok && drive == "true" {
+			// Get drive obd details from local server's drive(s)
+			driveOBDSerial := getLocalDrivesOBD(deadlinedCtx, false, globalEndpoints, r)
+			driveOBDParallel := getLocalDrivesOBD(deadlinedCtx, true, globalEndpoints, r)
+
+			errStr := ""
+			if driveOBDSerial.Error != "" {
+				errStr = "serial: " + driveOBDSerial.Error
+			}
+			if driveOBDParallel.Error != "" {
+				errStr = errStr + " parallel: " + driveOBDParallel.Error
+			}
+
+			driveOBD := madmin.ServerDrivesOBDInfo{
+				Addr:     driveOBDSerial.Addr,
+				Serial:   driveOBDSerial.Serial,
+				Parallel: driveOBDParallel.Parallel,
+				Error:    errStr,
+			}
+			obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, driveOBD)
+			partialWrite(obdInfo)
+
+			// Notify all other MinIO peers to report drive obd numbers
+			driveOBDs := globalNotificationSys.DriveOBDInfoChan(deadlinedCtx)
+			for obd := range driveOBDs {
+				obdInfo.Perf.DriveInfo = append(obdInfo.Perf.DriveInfo, obd)
+				partialWrite(obdInfo)
+			}
+			partialWrite(obdInfo)
+		}
+
+		if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistXL {
+			obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx))
+			partialWrite(obdInfo)
+
+			netOBDs := globalNotificationSys.DispatchNetOBDChan(deadlinedCtx)
+			for obd := range netOBDs {
+				obdInfo.Perf.Net = append(obdInfo.Perf.Net, obd)
+				partialWrite(obdInfo)
+			}
+			partialWrite(obdInfo)
+
+			obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx)
+			partialWrite(obdInfo)
+		}
+	}()
+
+	ticker := time.NewTicker(30 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case oinfo, ok := <-obdInfoCh:
+			if !ok {
+				return
+			}
+			logger.LogIf(ctx, enc.Encode(oinfo))
+			w.(http.Flusher).Flush()
+		case <-ticker.C:
+			if _, err := w.Write([]byte(" ")); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-deadlinedCtx.Done():
+			w.(http.Flusher).Flush()
+			return
+		}
 	}
 
-	if net, ok := vars["perfnet"]; ok && net == "true" && globalIsDistXL {
-		obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.NetOBDInfo(deadlinedCtx))
-		obdInfo.Perf.Net = append(obdInfo.Perf.Net, globalNotificationSys.DispatchNetOBDInfo(deadlinedCtx)...)
-		obdInfo.Perf.NetParallel = globalNotificationSys.NetOBDParallelInfo(deadlinedCtx)
-		partialWrite()
-	}
-
-	finish()
 }
 
 // ServerInfoHandler - GET /minio/admin/v3/info
@@ -1332,6 +1309,9 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 // Get server information
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerInfo")
+
+	defer logger.AuditLog(w, r, "ServerInfo", mustGetClaimsFromToken(r))
+
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
 	if objectAPI == nil {
 		return
@@ -1350,11 +1330,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
 	if err == nil {
 		buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
-		objects = madmin.Objects{Count: dataUsageInfo.ObjectsCount}
+		objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
 		usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
 	}
 
-	infoMsg := madmin.InfoMessage{}
 	vault := fetchVaultStatus(cfg)
 
 	ldap := madmin.LDAP{}
@@ -1376,8 +1355,8 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	// Get the notification target info
 	notifyTarget := fetchLambdaInfo(cfg)
 
-	// Fetching the Storage information
-	storageInfo := objectAPI.StorageInfo(ctx, false)
+	// Fetching the Storage information, ignore any errors.
+	storageInfo, _ := objectAPI.StorageInfo(ctx, false)
 
 	var OnDisks int
 	var OffDisks int
@@ -1459,11 +1438,11 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		Notifications: notifyTarget,
 	}
 
-	infoMsg = madmin.InfoMessage{
+	infoMsg := madmin.InfoMessage{
 		Mode:         mode,
 		Domain:       domain,
 		Region:       globalServerRegion,
-		SQSARN:       globalNotificationSys.GetARNList(),
+		SQSARN:       globalNotificationSys.GetARNList(false),
 		DeploymentID: globalDeploymentID,
 		Buckets:      buckets,
 		Objects:      objects,
@@ -1488,7 +1467,9 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 func fetchLambdaInfo(cfg config.Config) []map[string][]madmin.TargetIDStatus {
 
 	// Fetch the configured targets
-	targetList, err := notify.FetchRegisteredTargets(cfg, GlobalServiceDoneCh, NewGatewayHTTPTransport(), true, false)
+	tr := NewGatewayHTTPTransport()
+	defer tr.CloseIdleConnections()
+	targetList, err := notify.FetchRegisteredTargets(cfg, GlobalContext.Done(), tr, true, false)
 	if err != nil && err != notify.ErrTargetsOffline {
 		logger.LogIf(GlobalContext, err)
 		return nil
@@ -1538,7 +1519,6 @@ func fetchVaultStatus(cfg config.Config) madmin.Vault {
 	}
 
 	if err := checkConnection(kmsInfo.Endpoint, 15*time.Second); err != nil {
-
 		vault.Status = "offline"
 	} else {
 		vault.Status = "online"
@@ -1606,25 +1586,23 @@ func fetchLoggerInfo(cfg config.Config) ([]madmin.Logger, []madmin.Audit) {
 
 // checkConnection - ping an endpoint , return err in case of no connection
 func checkConnection(endpointStr string, timeout time.Duration) error {
-	u, pErr := xnet.ParseURL(endpointStr)
-	if pErr != nil {
-		return pErr
+	tr := newCustomHTTPTransport(&tls.Config{RootCAs: globalRootCAs}, timeout)()
+	defer tr.CloseIdleConnections()
+
+	ctx, cancel := context.WithTimeout(GlobalContext, timeout)
+	defer cancel()
+
+	req, err := http.NewRequest(http.MethodHead, endpointStr, nil)
+	if err != nil {
+		return err
 	}
 
-	tr := newCustomHTTPTransport(
-		&tls.Config{RootCAs: globalRootCAs},
-		timeout,
-		0, /* Default value */
-	)()
-
-	if dErr := u.DialHTTP(tr); dErr != nil {
-		if urlErr, ok := dErr.(*url.Error); ok {
-			// To treat "connection refused" errors as un reachable endpoint.
-			if target.IsConnRefusedErr(urlErr.Err) {
-				return errors.New("endpoint unreachable, please check your endpoint")
-			}
-		}
-		return dErr
+	client := &http.Client{Transport: tr}
+	resp, err := client.Do(req.WithContext(ctx))
+	if err != nil {
+		return err
 	}
+	defer xhttp.DrainBody(resp.Body)
+	resp.Body.Close()
 	return nil
 }

cmd/admin-handlers_test.go

@@ -68,21 +68,9 @@ func prepareAdminXLTestBed(ctx context.Context) (*adminXLTestBed, error) {
 
 	globalEndpoints = mustGetZoneEndpoints(xlDirs...)
 
-	globalConfigSys = NewConfigSys()
+	newAllSubsystems()
 
-	globalIAMSys = NewIAMSys()
-	globalIAMSys.Init(ctx, objLayer)
-
-	buckets, err := objLayer.ListBuckets(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	globalPolicySys = NewPolicySys()
-	globalPolicySys.Init(buckets, objLayer)
-
-	globalNotificationSys = NewNotificationSys(globalEndpoints)
-	globalNotificationSys.Init(buckets, objLayer)
+	initAllSubsystems(objLayer)
 
 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()
@@ -117,7 +105,7 @@ func initTestXLObjLayer(ctx context.Context) (ObjectLayer, []string, error) {
 	}
 
 	globalPolicySys = NewPolicySys()
-	objLayer, err := newXLSets(ctx, endpoints, storageDisks, format, 1, 16)
+	objLayer, err := newXLSets(ctx, endpoints, storageDisks, format)
 	if err != nil {
 		return nil, nil, err
 	}

cmd/admin-heal-ops.go

@@ -60,9 +60,8 @@ const (
 )
 
 var (
-	errHealIdleTimeout      = fmt.Errorf("healing results were not consumed for too long")
-	errHealPushStopNDiscard = fmt.Errorf("heal push stopped due to heal stop signal")
-	errHealStopSignalled    = fmt.Errorf("heal stop signaled")
+	errHealIdleTimeout   = fmt.Errorf("healing results were not consumed for too long")
+	errHealStopSignalled = fmt.Errorf("heal stop signaled")
 
 	errFnHealFromAPIErr = func(ctx context.Context, err error) error {
 		apiErr := toAPIError(ctx, err)
@@ -73,10 +72,6 @@ var (
 
 // healSequenceStatus - accumulated status of the heal sequence
 type healSequenceStatus struct {
-	// lock to update this structure as it is concurrently
-	// accessed
-	updateLock *sync.RWMutex
-
 	// summary and detail for failures
 	Summary       healStatusSummary `json:"Summary"`
 	FailureDetail string            `json:"Detail,omitempty"`
@@ -114,11 +109,9 @@ func initHealState() *allHealState {
 func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 	// Launch clean-up routine to remove this heal sequence (after
 	// it ends) from the global state after timeout has elapsed.
-	ticker := time.NewTicker(time.Minute * 5)
-	defer ticker.Stop()
 	for {
 		select {
-		case <-ticker.C:
+		case <-time.After(time.Minute * 5):
 			now := UTCNow()
 			ahs.Lock()
 			for path, h := range ahs.healSeqMap {
@@ -202,9 +195,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	existsAndLive := false
 	he, exists := ahs.getHealSequence(h.path)
 	if exists {
-		if !he.hasEnded() || len(he.currentStatus.Items) > 0 {
-			existsAndLive = true
-		}
+		existsAndLive = !he.hasEnded()
 	}
 
 	if existsAndLive {
@@ -230,8 +221,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	// Check if new heal sequence to be started overlaps with any
 	// existing, running sequence
 	for k, hSeq := range ahs.healSeqMap {
-		if !hSeq.hasEnded() && (strings.HasPrefix(k, h.path) ||
-			strings.HasPrefix(h.path, k)) {
+		if !hSeq.hasEnded() && (HasPrefix(k, h.path) || HasPrefix(h.path, k)) {
 
 			errMsg = "The provided heal sequence path overlaps with an existing " +
 				fmt.Sprintf("heal path: %s", k)
@@ -277,8 +267,8 @@ func (ahs *allHealState) PopHealStatusJSON(path string,
 	}
 
 	// Take lock to access and update the heal-sequence
-	h.currentStatus.updateLock.Lock()
-	defer h.currentStatus.updateLock.Unlock()
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
 
 	numItems := len(h.currentStatus.Items)
 
@@ -289,27 +279,25 @@ func (ahs *allHealState) PopHealStatusJSON(path string,
 		lastResultIndex = h.currentStatus.Items[numItems-1].ResultIndex
 	}
 
-	// After sending status to client, and before relinquishing
-	// the updateLock, reset Item to nil and record the result
-	// index sent to the client.
-	defer func(i int64) {
-		h.lastSentResultIndex = i
-		h.currentStatus.Items = nil
-	}(lastResultIndex)
+	h.lastSentResultIndex = lastResultIndex
 
 	jbytes, err := json.Marshal(h.currentStatus)
 	if err != nil {
+		h.currentStatus.Items = nil
+
 		logger.LogIf(h.ctx, err)
 		return nil, ErrInternalError
 	}
 
+	h.currentStatus.Items = nil
+
 	return jbytes, ErrNone
 }
 
 // healSource denotes single entity and heal option.
 type healSource struct {
-	path string           // entity path (format, buckets, objects) to heal
-	opts *madmin.HealOpts // optional heal option overrides default setting
+	path string          // entity path (format, buckets, objects) to heal
+	opts madmin.HealOpts // optional heal option overrides default setting
 }
 
 // healSequence - state for each heal sequence initiated on the
@@ -321,9 +309,12 @@ type healSequence struct {
 	// path is just pathJoin(bucket, objPrefix)
 	path string
 
-	// List of entities (format, buckets, objects) to heal
+	// A channel of entities (format, buckets, objects) to heal
 	sourceCh chan healSource
 
+	// A channel of entities with heal result
+	respCh chan healResult
+
 	// Report healing progress
 	reportProgress bool
 
@@ -349,9 +340,8 @@ type healSequence struct {
 	// completed
 	traverseAndHealDoneCh chan error
 
-	// channel to signal heal sequence to stop (e.g. from the
-	// heal-stop API)
-	stopSignalCh chan struct{}
+	// canceler to cancel heal sequence.
+	cancelCtx context.CancelFunc
 
 	// the last result index sent to client
 	lastSentResultIndex int64
@@ -377,14 +367,15 @@ type healSequence struct {
 
 // NewHealSequence - creates healSettings, assumes bucket and
 // objPrefix are already validated.
-func newHealSequence(bucket, objPrefix, clientAddr string,
+func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
 	numDisks int, hs madmin.HealOpts, forceStart bool) *healSequence {
 
 	reqInfo := &logger.ReqInfo{RemoteHost: clientAddr, API: "Heal", BucketName: bucket}
 	reqInfo.AppendTags("prefix", objPrefix)
-	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
+	ctx, cancel := context.WithCancel(logger.SetReqInfo(ctx, reqInfo))
 
 	return &healSequence{
+		respCh:         make(chan healResult),
 		bucket:         bucket,
 		objPrefix:      objPrefix,
 		path:           pathJoin(bucket, objPrefix),
@@ -398,10 +389,9 @@ func newHealSequence(bucket, objPrefix, clientAddr string,
 			Summary:      healNotStartedStatus,
 			HealSettings: hs,
 			NumDisks:     numDisks,
-			updateLock:   &sync.RWMutex{},
 		},
 		traverseAndHealDoneCh: make(chan error),
-		stopSignalCh:          make(chan struct{}),
+		cancelCtx:             cancel,
 		ctx:                   ctx,
 		scannedItemsMap:       make(map[madmin.HealItemType]int64),
 		healedItemsMap:        make(map[madmin.HealItemType]int64),
@@ -484,7 +474,7 @@ func (h *healSequence) gethealFailedItemsMap() map[string]int64 {
 // external signal)
 func (h *healSequence) isQuitting() bool {
 	select {
-	case <-h.stopSignalCh:
+	case <-h.ctx.Done():
 		return true
 	default:
 		return false
@@ -493,19 +483,15 @@ func (h *healSequence) isQuitting() bool {
 
 // check if the heal sequence has ended
 func (h *healSequence) hasEnded() bool {
-	h.currentStatus.updateLock.RLock()
-	summary := h.currentStatus.Summary
-	h.currentStatus.updateLock.RUnlock()
-	return summary == healStoppedStatus || summary == healFinishedStatus
+	h.mutex.RLock()
+	ended := len(h.currentStatus.Items) == 0 || h.currentStatus.Summary == healStoppedStatus || h.currentStatus.Summary == healFinishedStatus
+	h.mutex.RUnlock()
+	return ended
 }
 
 // stops the heal sequence - safe to call multiple times.
 func (h *healSequence) stop() {
-	select {
-	case <-h.stopSignalCh:
-	default:
-		close(h.stopSignalCh)
-	}
+	h.cancelCtx()
 }
 
 // pushHealResultItem - pushes a heal result item for consumption in
@@ -532,29 +518,27 @@ func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
 
 	var itemsLen int
 	for {
-		h.currentStatus.updateLock.Lock()
+		h.mutex.Lock()
 		itemsLen = len(h.currentStatus.Items)
 		if itemsLen == maxUnconsumedHealResultItems {
-			// unlock and wait to check again if we can push
-			h.currentStatus.updateLock.Unlock()
-
 			// wait for a second, or quit if an external
 			// stop signal is received or the
 			// unconsumedTimer fires.
 			select {
 			// Check after a second
 			case <-time.After(time.Second):
+				h.mutex.Unlock()
 				continue
 
-			case <-h.stopSignalCh:
+			case <-h.ctx.Done():
+				h.mutex.Unlock()
 				// discard result and return.
-				return errHealPushStopNDiscard
+				return errHealStopSignalled
 
-			// Timeout if no results consumed for too
-			// long.
+			// Timeout if no results consumed for too long.
 			case <-unconsumedTimer.C:
+				h.mutex.Unlock()
 				return errHealIdleTimeout
-
 			}
 		}
 		break
@@ -571,13 +555,7 @@ func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
 	h.currentStatus.Items = append(h.currentStatus.Items, r)
 
 	// release lock
-	h.currentStatus.updateLock.Unlock()
-
-	// This is a "safe" point for the heal sequence to quit if
-	// signaled externally.
-	if h.isQuitting() {
-		return errHealStopSignalled
-	}
+	h.mutex.Unlock()
 
 	return nil
 }
@@ -591,10 +569,10 @@ func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
 // the heal-sequence.
 func (h *healSequence) healSequenceStart() {
 	// Set status as running
-	h.currentStatus.updateLock.Lock()
+	h.mutex.Lock()
 	h.currentStatus.Summary = healRunningStatus
 	h.currentStatus.StartTime = UTCNow()
-	h.currentStatus.updateLock.Unlock()
+	h.mutex.Unlock()
 
 	if h.sourceCh == nil {
 		go h.traverseAndHeal()
@@ -604,25 +582,27 @@ func (h *healSequence) healSequenceStart() {
 
 	select {
 	case err, ok := <-h.traverseAndHealDoneCh:
+		if !ok {
+			return
+		}
+		h.mutex.Lock()
 		h.endTime = UTCNow()
-		h.currentStatus.updateLock.Lock()
-		defer h.currentStatus.updateLock.Unlock()
 		// Heal traversal is complete.
-		if ok {
+		if err == nil {
+			// heal traversal succeeded.
+			h.currentStatus.Summary = healFinishedStatus
+		} else {
 			// heal traversal had an error.
 			h.currentStatus.Summary = healStoppedStatus
 			h.currentStatus.FailureDetail = err.Error()
-		} else {
-			// heal traversal succeeded.
-			h.currentStatus.Summary = healFinishedStatus
 		}
-
-	case <-h.stopSignalCh:
+		h.mutex.Unlock()
+	case <-h.ctx.Done():
+		h.mutex.Lock()
 		h.endTime = UTCNow()
-		h.currentStatus.updateLock.Lock()
 		h.currentStatus.Summary = healStoppedStatus
 		h.currentStatus.FailureDetail = errHealStopSignalled.Error()
-		h.currentStatus.updateLock.Unlock()
+		h.mutex.Unlock()
 
 		// drain traverse channel so the traversal
 		// go-routine does not leak.
@@ -636,53 +616,64 @@ func (h *healSequence) healSequenceStart() {
 }
 
 func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
-	var respCh = make(chan healResult)
-	defer close(respCh)
 	// Send heal request
 	task := healTask{
 		path:       source.path,
-		responseCh: respCh,
 		opts:       h.settings,
+		responseCh: h.respCh,
 	}
-	if source.opts != nil {
-		task.opts = *source.opts
+	if !source.opts.Equal(h.settings) {
+		task.opts = source.opts
 	}
 	globalBackgroundHealRoutine.queueHealTask(task)
-	// Wait for answer and push result to the client
-	res := <-respCh
-	if !h.reportProgress {
-		h.mutex.Lock()
-		defer h.mutex.Unlock()
 
-		// Progress is not reported in case of background heal processing.
-		// Instead we increment relevant counter based on the heal result
-		// for prometheus reporting.
-		if res.err != nil && !isErrObjectNotFound(res.err) {
-			for _, d := range res.result.After.Drives {
-				// For failed items we report the endpoint and drive state
-				// This will help users take corrective actions for drives
-				h.healFailedItemsMap[d.Endpoint+","+d.State]++
+	select {
+	case res := <-h.respCh:
+		if !h.reportProgress {
+			// Object might have been deleted, by the time heal
+			// was attempted, we should ignore this object and
+			// return success.
+			if isErrObjectNotFound(res.err) {
+				return nil
 			}
-		} else {
-			// Only object type reported for successful healing
-			h.healedItemsMap[res.result.Type]++
-		}
-		return nil
-	}
-	res.result.Type = healType
-	if res.err != nil {
-		// Object might have been deleted, by the time heal
-		// was attempted, we should ignore this object and return success.
-		if isErrObjectNotFound(res.err) {
-			return nil
-		}
-		// Only report object error
-		if healType != madmin.HealItemObject {
+
+			h.mutex.Lock()
+			defer h.mutex.Unlock()
+
+			// Progress is not reported in case of background heal processing.
+			// Instead we increment relevant counter based on the heal result
+			// for prometheus reporting.
+			if res.err != nil {
+				for _, d := range res.result.After.Drives {
+					// For failed items we report the endpoint and drive state
+					// This will help users take corrective actions for drives
+					h.healFailedItemsMap[d.Endpoint+","+d.State]++
+				}
+			} else {
+				// Only object type reported for successful healing
+				h.healedItemsMap[res.result.Type]++
+			}
+
+			// Report caller of any failure
 			return res.err
 		}
-		res.result.Detail = res.err.Error()
+		res.result.Type = healType
+		if res.err != nil {
+			// Object might have been deleted, by the time heal
+			// was attempted, we should ignore this object and return success.
+			if isErrObjectNotFound(res.err) {
+				return nil
+			}
+			// Only report object error
+			if healType != madmin.HealItemObject {
+				return res.err
+			}
+			res.result.Detail = res.err.Error()
+		}
+		return h.pushHealResultItem(res.result)
+	case <-h.ctx.Done():
+		return nil
 	}
-	return h.pushHealResultItem(res.result)
 }
 
 func (h *healSequence) healItemsFromSourceCh() error {
@@ -693,7 +684,10 @@ func (h *healSequence) healItemsFromSourceCh() error {
 
 	for {
 		select {
-		case source := <-h.sourceCh:
+		case source, ok := <-h.sourceCh:
+			if !ok {
+				return nil
+			}
 			var itemType madmin.HealItemType
 			switch {
 			case source.path == nopHeal:
@@ -712,8 +706,6 @@ func (h *healSequence) healItemsFromSourceCh() error {
 
 			h.scannedItemsMap[itemType]++
 			h.lastHealActivity = UTCNow()
-		case <-h.traverseAndHealDoneCh:
-			return nil
 		case <-h.ctx.Done():
 			return nil
 		}
@@ -722,7 +714,6 @@ func (h *healSequence) healItemsFromSourceCh() error {
 
 func (h *healSequence) healFromSourceCh() {
 	h.healItemsFromSourceCh()
-	close(h.traverseAndHealDoneCh)
 }
 
 func (h *healSequence) healItems(bucketsOnly bool) error {
@@ -754,13 +745,7 @@ func (h *healSequence) healItems(bucketsOnly bool) error {
 // two objects.
 func (h *healSequence) traverseAndHeal() {
 	bucketsOnly := false // Heals buckets and objects also.
-	if err := h.healItems(bucketsOnly); err != nil {
-		if h.isQuitting() {
-			err = errHealStopSignalled
-		}
-		h.traverseAndHealDoneCh <- err
-	}
-
+	h.traverseAndHealDoneCh <- h.healItems(bucketsOnly)
 	close(h.traverseAndHealDoneCh)
 }
 

cmd/admin-quota-handlers.go

@@ -0,0 +1,118 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/env"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
+)
+
+const (
+	bucketQuotaConfigFile = "quota.json"
+)
+
+// PutBucketQuotaConfigHandler - PUT Bucket quota configuration.
+// ----------
+// Places a quota configuration on the specified bucket. The quota
+// specified in the quota configuration will be applied by default
+// to enforce total quota for the specified bucket.
+func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketQuotaConfig")
+
+	defer logger.AuditLog(w, r, "PutBucketQuotaConfig", mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketQuotaAdminAction)
+	if objectAPI == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// Turn off quota commands if data usage info is unavailable.
+	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminBucketQuotaDisabled), r.URL)
+		return
+	}
+
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	data, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	if _, err = parseBucketQuota(bucket, data); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if err = globalBucketMetadataSys.Update(bucket, bucketQuotaConfigFile, data); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetBucketQuotaConfigHandler - gets bucket quota configuration
+func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketQuotaConfig")
+
+	defer logger.AuditLog(w, r, "GetBucketQuotaConfig", mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetBucketQuotaAdminAction)
+	if objectAPI == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	config, err := globalBucketMetadataSys.GetQuotaConfig(bucket)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	configData, err := json.Marshal(config)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseJSON(w, configData)
+}

cmd/admin-router.go

@@ -20,6 +20,8 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
 )
 
@@ -50,7 +52,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 	for _, adminVersion := range adminVersions {
 		// Restart and stop MinIO service.
-		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(httpTraceAll(adminAPI.ServiceActionHandler)).Queries("action", "{action:.*}")
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(httpTraceAll(adminAPI.ServiceHandler)).Queries("action", "{action:.*}")
 		// Update MinIO servers.
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(httpTraceAll(adminAPI.ServerUpdateHandler)).Queries("updateURL", "{updateURL:.*}")
 
@@ -62,8 +64,6 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		// DataUsageInfo operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(httpTraceAll(adminAPI.DataUsageInfoHandler))
 
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountingusageinfo").HandlerFunc(httpTraceAll(adminAPI.AccountingUsageInfoHandler))
-
 		if globalIsDistXL || globalIsXL {
 			/// Heal operations
 
@@ -88,13 +88,19 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKVHandler)).Queries("key", "{key:.*}")
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/set-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKVHandler))
 			adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/del-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.DelConfigKVHandler))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/help-config-kv").HandlerFunc(httpTraceAll(adminAPI.HelpConfigKVHandler)).Queries("subSys", "{subSys:.*}", "key", "{key:.*}")
+		}
+
+		// Enable config help in all modes.
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/help-config-kv").HandlerFunc(httpTraceAll(adminAPI.HelpConfigKVHandler)).Queries("subSys", "{subSys:.*}", "key", "{key:.*}")
+
+		// Config KV history operations.
+		if enableConfigOps {
 			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-config-history-kv").HandlerFunc(httpTraceAll(adminAPI.ListConfigHistoryKVHandler)).Queries("count", "{count:[0-9]+}")
 			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/clear-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.ClearConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/restore-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.RestoreConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
 		}
 
-		/// Config operations
+		/// Config import/export bulk operations
 		if enableConfigOps {
 			// Get config
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
@@ -109,12 +115,17 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
 
 			// Add user IAM
+
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountusageinfo").HandlerFunc(httpTraceAll(adminAPI.AccountUsageInfoHandler))
+
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
 
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
 
 			// Service accounts ops
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/add-service-account").HandlerFunc(httpTraceHdrs(adminAPI.AddServiceAccount))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-service-accounts").HandlerFunc(httpTraceHdrs(adminAPI.ListServiceAccounts))
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/delete-service-account").HandlerFunc(httpTraceHdrs(adminAPI.DeleteServiceAccount)).Queries("accessKey", "{accessKey:.*}")
 
 			if adminVersion == adminAPIVersionV2Prefix {
 				// Info policy IAM v2
@@ -158,7 +169,18 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 			// Set Group Status
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(httpTraceHdrs(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
+		}
 
+		// Quota operations
+		if globalIsXL || globalIsDistXL {
+			if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
+				// GetBucketQuotaConfig
+				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
+					httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
+				// PutBucketQuotaConfig
+				adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
+					httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
+			}
 		}
 
 		// -- Top APIs --

cmd/api-errors.go

@@ -25,17 +25,16 @@ import (
 	"strings"
 
 	"github.com/Azure/azure-storage-blob-go/azblob"
-	"github.com/aliyun/aliyun-oss-go-sdk/oss"
 	"google.golang.org/api/googleapi"
 
 	minio "github.com/minio/minio-go/v6"
+	"github.com/minio/minio-go/v6/pkg/tags"
 	"github.com/minio/minio/cmd/config/etcd/dns"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
-	"github.com/minio/minio/pkg/bucket/object/tagging"
 	"github.com/minio/minio/pkg/bucket/policy"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
@@ -101,6 +100,9 @@ const (
 	ErrNoSuchBucketLifecycle
 	ErrNoSuchLifecycleConfiguration
 	ErrNoSuchBucketSSEConfig
+	ErrNoSuchCORSConfiguration
+	ErrNoSuchWebsiteConfiguration
+	ErrReplicationConfigurationNotFoundError
 	ErrNoSuchKey
 	ErrNoSuchUpload
 	ErrNoSuchVersion
@@ -158,6 +160,7 @@ const (
 	ErrInvalidRetentionDate
 	ErrPastObjectLockRetainDate
 	ErrUnknownWORMModeDirective
+	ErrBucketTaggingNotFound
 	ErrObjectLockInvalidHeaders
 	ErrInvalidTagDirective
 	// Add new error codes here.
@@ -236,6 +239,10 @@ const (
 	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
 	ErrObjectTampered
+	// Bucket Quota error codes
+	ErrAdminBucketQuotaExceeded
+	ErrAdminNoSuchQuotaConfiguration
+	ErrAdminBucketQuotaDisabled
 
 	ErrHealNotImplemented
 	ErrHealNoSuchProcess
@@ -335,8 +342,8 @@ const (
 	ErrAdminProfilerNotEnabled
 	ErrInvalidDecompressedSize
 	ErrAddUserInvalidArgument
-	ErrAddServiceAccountInvalidArgument
-	ErrAddServiceAccountInvalidParent
+	ErrAdminAccountNotEligible
+	ErrServiceAccountNotFound
 	ErrPostPolicyConditionInvalidFormat
 )
 
@@ -452,7 +459,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrInvalidAccessKeyID: {
 		Code:           "InvalidAccessKeyId",
-		Description:    "The access key ID you provided does not exist in our records.",
+		Description:    "The Access Key Id you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrInvalidBucketName: {
@@ -774,6 +781,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Bucket is missing ObjectLockConfiguration",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrBucketTaggingNotFound: {
+		Code:           "NoSuchTagSet",
+		Description:    "The TagSet does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrObjectLockConfigurationNotFound: {
 		Code:           "ObjectLockConfigurationNotFoundError",
 		Description:    "Object Lock configuration does not exist for this bucket",
@@ -784,6 +796,21 @@ var errorCodes = errorCodeMap{
 		Description:    "Object Lock configuration cannot be enabled on existing buckets",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrNoSuchCORSConfiguration: {
+		Code:           "NoSuchCORSConfiguration",
+		Description:    "The CORS configuration does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchWebsiteConfiguration: {
+		Code:           "NoSuchWebsiteConfiguration",
+		Description:    "The specified bucket does not have a website configuration",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrReplicationConfigurationNotFoundError: {
+		Code:           "ReplicationConfigurationNotFoundError",
+		Description:    "The replication configuration was not found",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrNoSuchObjectLockConfiguration: {
 		Code:           "NoSuchObjectLockConfiguration",
 		Description:    "The specified object does not have a ObjectLock configuration",
@@ -1090,6 +1117,21 @@ var errorCodes = errorCodeMap{
 		Description:    "Credentials in config mismatch with server environment variables",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrAdminBucketQuotaExceeded: {
+		Code:           "XMinioAdminBucketQuotaExceeded",
+		Description:    "Bucket quota exceeded",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminNoSuchQuotaConfiguration: {
+		Code:           "XMinioAdminNoSuchQuotaConfiguration",
+		Description:    "The quota configuration does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAdminBucketQuotaDisabled: {
+		Code:           "XMinioAdminBucketQuotaDisabled",
+		Description:    "Quota specified but disk usage crawl is disabled on MinIO server",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInsecureClientRequest: {
 		Code:           "XMinioInsecureClientRequest",
 		Description:    "Cannot respond to plain-text request from TLS-encrypted server",
@@ -1600,17 +1642,16 @@ var errorCodes = errorCodeMap{
 		Description:    "User is not allowed to be same as admin access key",
 		HTTPStatusCode: http.StatusConflict,
 	},
-	ErrAddServiceAccountInvalidArgument: {
+	ErrAdminAccountNotEligible: {
 		Code:           "XMinioInvalidIAMCredentials",
-		Description:    "Creating service accounts for admin access key is not allowed",
+		Description:    "The administrator key is not eligible for this operation",
 		HTTPStatusCode: http.StatusConflict,
 	},
-	ErrAddServiceAccountInvalidParent: {
-		Code:           "XMinioInvalidIAMCredentialsParent",
-		Description:    "Creating service accounts for other users is not allowed",
-		HTTPStatusCode: http.StatusConflict,
+	ErrServiceAccountNotFound: {
+		Code:           "XMinioInvalidIAMCredentials",
+		Description:    "The specified service account is not found",
+		HTTPStatusCode: http.StatusNotFound,
 	},
-
 	ErrPostPolicyConditionInvalidFormat: {
 		Code:           "PostPolicyInvalidKeyName",
 		Description:    "Invalid according to Policy: Policy Condition failed",
@@ -1626,7 +1667,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	if err == nil {
 		return ErrNone
 	}
-	// Verify if the underlying error is signature mismatch.
+
 	switch err {
 	case errInvalidArgument:
 		apiErr = ErrAdminInvalidArgument
@@ -1785,6 +1826,14 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrNoSuchLifecycleConfiguration
 	case BucketSSEConfigNotFound:
 		apiErr = ErrNoSuchBucketSSEConfig
+	case BucketTaggingNotFound:
+		apiErr = ErrBucketTaggingNotFound
+	case BucketObjectLockConfigNotFound:
+		apiErr = ErrObjectLockConfigurationNotFound
+	case BucketQuotaConfigNotFound:
+		apiErr = ErrAdminNoSuchQuotaConfiguration
+	case BucketQuotaExceeded:
+		apiErr = ErrAdminBucketQuotaExceeded
 	case *event.ErrInvalidEventName:
 		apiErr = ErrEventNotification
 	case *event.ErrInvalidARN:
@@ -1875,7 +1924,7 @@ func toAPIError(ctx context.Context, err error) APIError {
 				Description:    e.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
-		case tagging.Error:
+		case tags.Error:
 			apiErr = APIError{
 				Code:           e.Code(),
 				Description:    e.Error(),
@@ -1917,12 +1966,6 @@ func toAPIError(ctx context.Context, err error) APIError {
 				Description:    e.Error(),
 				HTTPStatusCode: e.Response().StatusCode,
 			}
-		case oss.ServiceError:
-			apiErr = APIError{
-				Code:           e.Code,
-				Description:    e.Message,
-				HTTPStatusCode: e.StatusCode,
-			}
 			// Add more Gateway SDKs here if any in future.
 		}
 	}

cmd/api-headers.go

@@ -83,7 +83,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	}
 
 	if strings.Contains(objInfo.ETag, "-") && len(objInfo.Parts) > 0 {
-		w.Header().Set(xhttp.AmzMpPartsCount, strconv.Itoa(len(objInfo.Parts)))
+		w.Header()[xhttp.AmzMpPartsCount] = []string{strconv.Itoa(len(objInfo.Parts))}
 	}
 
 	if objInfo.ContentType != "" {
@@ -97,6 +97,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	if !objInfo.Expires.IsZero() {
 		w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
 	}
+
 	if globalCacheConfig.Enabled {
 		w.Header().Set(xhttp.XCache, objInfo.CacheStatus.String())
 		w.Header().Set(xhttp.XCacheLookup, objInfo.CacheLookupStatus.String())
@@ -105,34 +106,34 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	// Set tag count if object has tags
 	tags, _ := url.ParseQuery(objInfo.UserTags)
 	tagCount := len(tags)
-	if tagCount != 0 {
-		w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount))
+	if tagCount > 0 {
+		w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(tagCount)}
 	}
 
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
-		if HasPrefix(k, ReservedMetadataPrefix) {
+		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 			// Do not need to send any internal metadata
 			// values to client.
 			continue
 		}
-		w.Header().Set(k, v)
+		var isSet bool
+		for _, userMetadataPrefix := range userMetadataKeyPrefixes {
+			if !strings.HasPrefix(k, userMetadataPrefix) {
+				continue
+			}
+			w.Header()[strings.ToLower(k)] = []string{v}
+			isSet = true
+			break
+		}
+		if !isSet {
+			w.Header().Set(k, v)
+		}
 	}
 
-	var totalObjectSize int64
-	switch {
-	case crypto.IsEncrypted(objInfo.UserDefined):
-		totalObjectSize, err = objInfo.DecryptedSize()
-		if err != nil {
-			return err
-		}
-	case objInfo.IsCompressed():
-		totalObjectSize = objInfo.GetActualSize()
-		if totalObjectSize < 0 {
-			return errInvalidDecompressedSize
-		}
-	default:
-		totalObjectSize = objInfo.Size
+	totalObjectSize, err := objInfo.GetActualSize()
+	if err != nil {
+		return err
 	}
 
 	// for providing ranged content

cmd/api-response.go

@@ -33,8 +33,10 @@ import (
 )
 
 const (
-	timeFormatAMZLong = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
-	maxObjectList     = 10000                      // Limit number of objects in a listObjectsResponse.
+	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
+	iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
+	maxObjectList     = 50000                      // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
+	maxDeleteList     = 10000                      // Limit number of objects deleted in a delete call.
 	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
 	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.
 )
@@ -400,7 +402,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	for _, bucket := range buckets {
 		var listbucket = Bucket{}
 		listbucket.Name = bucket.Name
-		listbucket.CreationDate = bucket.Created.UTC().Format(timeFormatAMZLong)
+		listbucket.CreationDate = bucket.Created.UTC().Format(iso8601TimeFormat)
 		listbuckets = append(listbuckets, listbucket)
 	}
 
@@ -424,7 +426,7 @@ func generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingTyp
 			continue
 		}
 		content.Key = s3EncodeName(object.Name, encodingType)
-		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
+		content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
 		if object.ETag != "" {
 			content.ETag = "\"" + object.ETag + "\""
 		}
@@ -440,9 +442,9 @@ func generateListVersionsResponse(bucket, prefix, marker, delimiter, encodingTyp
 		content.IsLatest = true
 		versions = append(versions, content)
 	}
+
 	data.Name = bucket
 	data.Versions = versions
-
 	data.EncodingType = encodingType
 	data.Prefix = s3EncodeName(prefix, encodingType)
 	data.KeyMarker = s3EncodeName(marker, encodingType)
@@ -475,7 +477,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 			continue
 		}
 		content.Key = s3EncodeName(object.Name, encodingType)
-		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
+		content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
 		if object.ETag != "" {
 			content.ETag = "\"" + object.ETag + "\""
 		}
@@ -525,7 +527,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 			continue
 		}
 		content.Key = s3EncodeName(object.Name, encodingType)
-		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
+		content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
 		if object.ETag != "" {
 			content.ETag = "\"" + object.ETag + "\""
 		}
@@ -539,7 +541,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 		if metadata {
 			content.UserMetadata = make(StringMap)
 			for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) {
-				if HasPrefix(k, ReservedMetadataPrefix) {
+				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 					// Do not need to send any internal metadata
 					// values to client.
 					continue
@@ -574,7 +576,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
 	return CopyObjectResponse{
 		ETag:         "\"" + etag + "\"",
-		LastModified: lastModified.UTC().Format(timeFormatAMZLong),
+		LastModified: lastModified.UTC().Format(iso8601TimeFormat),
 	}
 }
 
@@ -582,7 +584,7 @@ func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectR
 func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse {
 	return CopyObjectPartResponse{
 		ETag:         "\"" + etag + "\"",
-		LastModified: lastModified.UTC().Format(timeFormatAMZLong),
+		LastModified: lastModified.UTC().Format(iso8601TimeFormat),
 	}
 }
 
@@ -601,7 +603,8 @@ func generateCompleteMultpartUploadResponse(bucket, key, location, etag string)
 		Location: location,
 		Bucket:   bucket,
 		Key:      key,
-		ETag:     etag,
+		// AWS S3 quotes the ETag in XML, make sure we are compatible here.
+		ETag: "\"" + etag + "\"",
 	}
 }
 
@@ -626,7 +629,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
 		newPart.PartNumber = part.PartNumber
 		newPart.ETag = "\"" + part.ETag + "\""
 		newPart.Size = part.Size
-		newPart.LastModified = part.LastModified.UTC().Format(timeFormatAMZLong)
+		newPart.LastModified = part.LastModified.UTC().Format(iso8601TimeFormat)
 		listPartsResponse.Parts[index] = newPart
 	}
 	return listPartsResponse
@@ -656,7 +659,7 @@ func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMult
 		newUpload := Upload{}
 		newUpload.UploadID = upload.UploadID
 		newUpload.Key = s3EncodeName(upload.Object, encodingType)
-		newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZLong)
+		newUpload.Initiated = upload.Initiated.UTC().Format(iso8601TimeFormat)
 		listMultipartUploadsResponse.Uploads[index] = newUpload
 	}
 	return listMultipartUploadsResponse
@@ -766,17 +769,6 @@ func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIE
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }
 
-// writeVersionMismatchResponse - writes custom error responses for version mismatches.
-func writeVersionMismatchResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL, isJSON bool) {
-	if isJSON {
-		// Generate error response.
-		errorResponse := getAPIErrorResponse(ctx, err, reqURL.String(), w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
-		writeResponse(w, err.HTTPStatusCode, encodeResponseJSON(errorResponse), mimeJSON)
-	} else {
-		writeResponse(w, err.HTTPStatusCode, []byte(err.Description), mimeNone)
-	}
-}
-
 // writeCustomErrorResponseJSON - similar to writeErrorResponseJSON,
 // but accepts the error message directly (this allows messages to be
 // dynamically generated.)

cmd/api-router.go

@@ -194,7 +194,7 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		// GetBucketReplicationHandler - this is a dummy call.
 		bucket.Methods(http.MethodGet).HandlerFunc(
 			maxClients(collectAPIStats("getbucketreplication", httpTraceAll(api.GetBucketReplicationHandler)))).Queries("replication", "")
-		// GetBucketTaggingHandler - this is a dummy call.
+		// GetBucketTaggingHandler
 		bucket.Methods(http.MethodGet).HandlerFunc(
 			maxClients(collectAPIStats("getbuckettagging", httpTraceAll(api.GetBucketTaggingHandler)))).Queries("tagging", "")
 		//DeleteBucketWebsiteHandler
@@ -244,6 +244,9 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		// PutBucketObjectLockConfig
 		bucket.Methods(http.MethodPut).HandlerFunc(
 			maxClients(collectAPIStats("putbucketobjectlockconfig", httpTraceAll(api.PutBucketObjectLockConfigHandler)))).Queries("object-lock", "")
+		// PutBucketTaggingHandler
+		bucket.Methods(http.MethodPut).HandlerFunc(
+			maxClients(collectAPIStats("putbuckettagging", httpTraceAll(api.PutBucketTaggingHandler)))).Queries("tagging", "")
 		// PutBucketVersioning
 		bucket.Methods(http.MethodPut).HandlerFunc(
 			maxClients(collectAPIStats("putbucketversioning", httpTraceAll(api.PutBucketVersioningHandler)))).Queries("versioning", "")
@@ -282,6 +285,11 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
 		maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler))))
 
+	// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
+	// than failing with UnknownAPIRequest we simply handle it for now.
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
+		maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler))))
+
 	// If none of the routes match add default error handler routes
 	apiRouter.NotFoundHandler = http.HandlerFunc(collectAPIStats("notfound", httpTraceAll(errorResponseHandler)))
 	apiRouter.MethodNotAllowedHandler = http.HandlerFunc(collectAPIStats("methodnotallowed", httpTraceAll(errorResponseHandler)))

cmd/auth-handler.go

@@ -140,6 +140,7 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 		reqInfo := (&logger.ReqInfo{}).AppendTags("requestHeaders", dumpRequest(r))
 		ctx := logger.SetReqInfo(ctx, reqInfo)
 		logger.LogIf(ctx, errors.New(getAPIError(s3Err).Description), logger.Application)
+		return cred, nil, owner, s3Err
 	}
 
 	claims, s3Err := checkClaimsFromToken(r, cred)
@@ -422,7 +423,7 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 	if err != nil {
 		return toAPIErrorCode(ctx, err)
 	}
-	r.Body = ioutil.NopCloser(reader)
+	r.Body = reader
 	return ErrNone
 }
 

cmd/auth-handler_test.go

@@ -391,6 +391,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 		}
 	}
 }
+
 func TestCheckAdminRequestAuthType(t *testing.T) {
 	objLayer, fsDir, err := prepareFS()
 	if err != nil {
@@ -425,3 +426,48 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 		}
 	}
 }
+
+func TestValidateAdminSignature(t *testing.T) {
+
+	ctx := context.Background()
+
+	objLayer, fsDir, err := prepareFS()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(fsDir)
+
+	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+
+	creds, err := auth.CreateCredentials("admin", "mypassword")
+	if err != nil {
+		t.Fatalf("unable create credential, %s", err)
+	}
+	globalActiveCred = creds
+
+	testCases := []struct {
+		AccessKey string
+		SecretKey string
+		ErrCode   APIErrorCode
+	}{
+		{"", "", ErrInvalidAccessKeyID},
+		{"admin", "", ErrSignatureDoesNotMatch},
+		{"admin", "wrongpassword", ErrSignatureDoesNotMatch},
+		{"wronguser", "mypassword", ErrInvalidAccessKeyID},
+		{"", "mypassword", ErrInvalidAccessKeyID},
+		{"admin", "mypassword", ErrNone},
+	}
+
+	for i, testCase := range testCases {
+		req := mustNewRequest("GET", "http://localhost:9000/", 0, nil, t)
+		if err := signRequestV4(req, testCase.AccessKey, testCase.SecretKey); err != nil {
+			t.Fatalf("Unable to inititalized new signed http request %s", err)
+		}
+		_, _, _, s3Error := validateAdminSignature(ctx, req, globalMinioDefaultRegion)
+		if s3Error != testCase.ErrCode {
+			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i+1, testCase.ErrCode, s3Error)
+		}
+	}
+}

cmd/background-heal-ops.go

@@ -88,9 +88,10 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 			case bucket != "" && object != "":
 				res, err = objAPI.HealObject(ctx, bucket, object, task.opts)
 			}
-			if task.responseCh != nil {
-				task.responseCh <- healResult{result: res, err: err}
+			if task.path != slashSeparator && task.path != nopHeal {
+				ObjectPathUpdated(task.path)
 			}
+			task.responseCh <- healResult{result: res, err: err}
 		case <-h.doneCh:
 			return
 		case <-ctx.Done():
@@ -113,8 +114,9 @@ func startBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 	go globalBackgroundHealRoutine.run(ctx, objAPI)
 
 	// Launch the background healer sequence to track
-	// background healing operations
-	info := objAPI.StorageInfo(ctx, false)
+	// background healing operations, ignore errors
+	// errors are handled into offline disks already.
+	info, _ := objAPI.StorageInfo(ctx, false)
 	numDisks := info.Backend.OnlineDisks.Sum() + info.Backend.OfflineDisks.Sum()
 	nh := newBgHealSequence(numDisks)
 	globalBackgroundHealState.LaunchNewHealSequence(nh)

cmd/background-newdisks-heal-ops.go

@@ -57,6 +57,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 		case <-time.After(defaultMonitorNewDiskInterval):
 			// Attempt a heal as the server starts-up first.
 			localDisksInZoneHeal := make([]Endpoints, len(z.zones))
+			var healNewDisks bool
 			for i, ep := range globalEndpoints {
 				localDisksToHeal := Endpoints{}
 				for _, endpoint := range ep.Endpoints {
@@ -74,6 +75,12 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 					continue
 				}
 				localDisksInZoneHeal[i] = localDisksToHeal
+				healNewDisks = true
+			}
+
+			// Reformat disks only if needed.
+			if !healNewDisks {
+				continue
 			}
 
 			// Reformat disks

cmd/benchmark-utils_test.go

@@ -35,7 +35,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err = obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -76,7 +76,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	object := getRandomObjectName()
 
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err = obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -181,7 +181,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -278,7 +278,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -322,7 +322,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
+	err := obj.MakeBucketWithLocation(context.Background(), bucket, "", false)
 	if err != nil {
 		b.Fatal(err)
 	}

cmd/bitrot_test.go

@@ -34,7 +34,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	volume := "testvol"
 	filePath := "testfile"
 
-	disk, err := newPosix(tmpDir)
+	disk, err := newPosix(tmpDir, "")
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/bootstrap-peer-server.go

@@ -247,7 +247,7 @@ func newBootstrapRESTClient(endpoint Endpoint) (*bootstrapRESTClient, error) {
 		}
 	}
 
-	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout, rest.DefaultRESTTimeout)
+	trFn := newCustomHTTPTransport(tlsConfig, rest.DefaultRESTTimeout)
 	restClient, err := rest.NewClient(serverURL, trFn, newAuthToken)
 	if err != nil {
 		return nil, err

cmd/bucket-encryption-handlers.go

@@ -18,13 +18,13 @@ package cmd
 
 import (
 	"encoding/xml"
+	"fmt"
 	"io"
 	"net/http"
 
 	"github.com/gorilla/mux"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/policy"
 )
 
@@ -69,7 +69,12 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 	// Parse bucket encryption xml
 	encConfig, err := validateBucketSSEConfig(io.LimitReader(r.Body, maxBucketSSEConfigSize))
 	if err != nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+		apiErr := APIError{
+			Code:           "MalformedXML",
+			Description:    fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description, err),
+			HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode,
+		}
+		writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -79,17 +84,17 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// Store the bucket encryption configuration in the object layer
-	if err = objAPI.SetBucketSSEConfig(ctx, bucket, encConfig); err != nil {
+	configData, err := xml.Marshal(encConfig)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	// Update the in-memory bucket encryption cache
-	globalBucketSSEConfigSys.Set(bucket, *encConfig)
-
-	// Update peer MinIO servers of the updated bucket encryption config
-	globalNotificationSys.SetBucketSSEConfig(ctx, bucket, encConfig)
+	// Store the bucket encryption configuration in the object layer
+	if err = globalBucketMetadataSys.Update(bucket, bucketSSEConfig, configData); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
 
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -122,21 +127,20 @@ func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// Fetch bucket encryption configuration from object layer
-	var encConfig *bucketsse.BucketSSEConfig
-	if encConfig, err = objAPI.GetBucketSSEConfig(ctx, bucket); err != nil {
+	config, err := globalBucketMetadataSys.GetSSEConfig(bucket)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	var encConfigData []byte
-	if encConfigData, err = xml.Marshal(encConfig); err != nil {
+	configData, err := xml.Marshal(config)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Write bucket encryption configuration to client
-	writeSuccessResponseXML(w, encConfigData)
+	writeSuccessResponseXML(w, configData)
 }
 
 // DeleteBucketEncryptionHandler - Removes bucket encryption configuration
@@ -167,15 +171,10 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 	}
 
 	// Delete bucket encryption config from object layer
-	if err = objAPI.DeleteBucketSSEConfig(ctx, bucket); err != nil {
+	if err = globalBucketMetadataSys.Update(bucket, bucketSSEConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	// Remove entry from the in-memory bucket encryption cache
-	globalBucketSSEConfigSys.Remove(bucket)
-	// Update peer MinIO servers of the updated bucket encryption config
-	globalNotificationSys.RemoveBucketSSEConfig(ctx, bucket)
-
 	writeSuccessNoContent(w)
 }

cmd/bucket-encryption.go

@@ -17,142 +17,32 @@
 package cmd
 
 import (
-	"bytes"
-	"context"
-	"encoding/xml"
 	"errors"
 	"io"
-	"path"
-	"sync"
 
 	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 )
 
 // BucketSSEConfigSys - in-memory cache of bucket encryption config
-type BucketSSEConfigSys struct {
-	sync.RWMutex
-	bucketSSEConfigMap map[string]bucketsse.BucketSSEConfig
-}
+type BucketSSEConfigSys struct{}
 
 // NewBucketSSEConfigSys - Creates an empty in-memory bucket encryption configuration cache
 func NewBucketSSEConfigSys() *BucketSSEConfigSys {
-	return &BucketSSEConfigSys{
-		bucketSSEConfigMap: make(map[string]bucketsse.BucketSSEConfig),
-	}
-}
-
-// load - Loads the bucket encryption configuration for the given list of buckets
-func (sys *BucketSSEConfigSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
-	for _, bucket := range buckets {
-		config, err := objAPI.GetBucketSSEConfig(GlobalContext, bucket.Name)
-		if err != nil {
-			if _, ok := err.(BucketSSEConfigNotFound); ok {
-				sys.Remove(bucket.Name)
-			}
-			continue
-		}
-		sys.Set(bucket.Name, *config)
-	}
-
-	return nil
-}
-
-// Init - Initializes in-memory bucket encryption config cache for the given list of buckets
-func (sys *BucketSSEConfigSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	// We don't cache bucket encryption config in gateway mode, nothing to do.
-	if globalIsGateway {
-		return nil
-	}
-
-	// Load bucket encryption config cache once during boot.
-	return sys.load(buckets, objAPI)
+	return &BucketSSEConfigSys{}
 }
 
 // Get - gets bucket encryption config for the given bucket.
-func (sys *BucketSSEConfigSys) Get(bucket string) (config bucketsse.BucketSSEConfig, ok bool) {
-	// We don't cache bucket encryption config in gateway mode.
+func (sys *BucketSSEConfigSys) Get(bucket string) (*bucketsse.BucketSSEConfig, error) {
 	if globalIsGateway {
 		objAPI := newObjectLayerWithoutSafeModeFn()
 		if objAPI == nil {
-			return
+			return nil, errServerNotInitialized
 		}
 
-		cfg, err := objAPI.GetBucketSSEConfig(GlobalContext, bucket)
-		if err != nil {
-			return
-		}
-		return *cfg, true
+		return nil, BucketSSEConfigNotFound{Bucket: bucket}
 	}
 
-	sys.Lock()
-	defer sys.Unlock()
-	config, ok = sys.bucketSSEConfigMap[bucket]
-	return
-}
-
-// Set - sets bucket encryption config to given bucket name.
-func (sys *BucketSSEConfigSys) Set(bucket string, config bucketsse.BucketSSEConfig) {
-	// We don't cache bucket encryption config in gateway mode.
-	if globalIsGateway {
-		return
-	}
-
-	sys.Lock()
-	defer sys.Unlock()
-	sys.bucketSSEConfigMap[bucket] = config
-}
-
-// Remove - removes bucket encryption config for given bucket.
-func (sys *BucketSSEConfigSys) Remove(bucket string) {
-	sys.Lock()
-	defer sys.Unlock()
-
-	delete(sys.bucketSSEConfigMap, bucket)
-}
-
-// saveBucketSSEConfig - save bucket encryption config for given bucket.
-func saveBucketSSEConfig(ctx context.Context, objAPI ObjectLayer, bucket string, config *bucketsse.BucketSSEConfig) error {
-	data, err := xml.Marshal(config)
-	if err != nil {
-		return err
-	}
-
-	// Path to store bucket encryption config for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
-	return saveConfig(ctx, objAPI, configFile, data)
-}
-
-// getBucketSSEConfig - get bucket encryption config for given bucket.
-func getBucketSSEConfig(objAPI ObjectLayer, bucket string) (*bucketsse.BucketSSEConfig, error) {
-	// Path to bucket-encryption.xml for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
-	configData, err := readConfig(GlobalContext, objAPI, configFile)
-	if err != nil {
-		if err == errConfigNotFound {
-			err = BucketSSEConfigNotFound{Bucket: bucket}
-		}
-		return nil, err
-	}
-
-	return bucketsse.ParseBucketSSEConfig(bytes.NewReader(configData))
-}
-
-// removeBucketSSEConfig - removes bucket encryption config for given bucket.
-func removeBucketSSEConfig(ctx context.Context, objAPI ObjectLayer, bucket string) error {
-	// Path to bucket-encryption.xml for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
-
-	if err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile); err != nil {
-		if _, ok := err.(ObjectNotFound); ok {
-			return BucketSSEConfigNotFound{Bucket: bucket}
-		}
-		return err
-	}
-	return nil
+	return globalBucketMetadataSys.GetSSEConfig(bucket)
 }
 
 // validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO.

cmd/bucket-handlers.go

@@ -30,6 +30,7 @@ import (
 	"github.com/gorilla/mux"
 
 	"github.com/minio/minio-go/v6/pkg/set"
+	"github.com/minio/minio-go/v6/pkg/tags"
 	"github.com/minio/minio/cmd/config/etcd/dns"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -44,10 +45,9 @@ import (
 )
 
 const (
-	getBucketVersioningResponse       = `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`
-	objectLockConfig                  = "object-lock.xml"
-	bucketObjectLockEnabledConfigFile = "object-lock-enabled.json"
-	bucketObjectLockEnabledConfig     = `{"x-amz-bucket-object-lock-enabled":true}`
+	getBucketVersioningResponse = `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`
+	objectLockConfig            = "object-lock.xml"
+	bucketTaggingConfigFile     = "tagging.xml"
 )
 
 // Check if there are buckets on server without corresponding entry in etcd backend and
@@ -399,9 +399,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			continue
 		}
 
-		if _, ok := globalBucketObjectLockConfig.Get(bucket); ok {
-			if err := enforceRetentionBypassForDelete(ctx, r, bucket, object.ObjectName, getObjectInfoFn); err != ErrNone {
-				dErrs[index] = err
+		if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
+			if apiErr := enforceRetentionBypassForDelete(ctx, r, bucket, object.ObjectName, getObjectInfoFn); apiErr != ErrNone {
+				dErrs[index] = apiErr
 				continue
 			}
 		}
@@ -527,25 +527,20 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		if err != nil {
 			if err == dns.ErrNoEntriesFound {
 				// Proceed to creating a bucket.
-				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
+				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location, objectLockEnabled); err != nil {
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 					return
 				}
 
-				if objectLockEnabled {
-					configFile := path.Join(bucketConfigPrefix, bucket, bucketObjectLockEnabledConfigFile)
-					if err = saveConfig(ctx, objectAPI, configFile, []byte(bucketObjectLockEnabledConfig)); err != nil {
-						writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-						return
-					}
-				}
-
 				if err = globalDNSConfig.Put(bucket); err != nil {
 					objectAPI.DeleteBucket(ctx, bucket, false)
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 					return
 				}
 
+				// Load updated bucket metadata into memory.
+				globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
+
 				// Make sure to add Location information here only for bucket
 				w.Header().Set(xhttp.Location,
 					getObjectLocation(r, globalDomainNames, bucket, ""))
@@ -570,21 +565,14 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Proceed to creating a bucket.
-	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
+	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location, objectLockEnabled)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	if objectLockEnabled && !globalIsGateway {
-		configFile := path.Join(bucketConfigPrefix, bucket, bucketObjectLockEnabledConfigFile)
-		if err = saveConfig(ctx, objectAPI, configFile, []byte(bucketObjectLockEnabledConfig)); err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-		globalBucketObjectLockConfig.Set(bucket, objectlock.Retention{})
-		globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, objectlock.Retention{})
-	}
+	// Load updated bucket metadata into memory.
+	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 
 	// Make sure to add Location information here only for bucket
 	w.Header().Set(xhttp.Location, path.Clean(r.URL.Path)) // Clean any trailing slashes.
@@ -619,7 +607,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	bucket := mux.Vars(r)["bucket"]
 
 	// To detect if the client has disconnected.
-	r.Body = &detectDisconnect{r.Body, r.Context().Done()}
+	r.Body = &contextReader{r.Body, r.Context()}
 
 	// Require Content-Length to be set in the request
 	size := r.ContentLength
@@ -758,11 +746,13 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	var objectEncryptionKey crypto.ObjectKey
 
 	// Check if bucket encryption is enabled
-	_, encEnabled := globalBucketSSEConfigSys.Get(bucket)
-	// This request header needs to be set prior to setting ObjectOptions
-	if (globalAutoEncryption || encEnabled) && !crypto.SSEC.IsRequested(r.Header) {
-		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
+	if _, err = globalBucketSSEConfigSys.Get(bucket); err == nil || globalAutoEncryption {
+		// This request header needs to be set prior to setting ObjectOptions
+		if !crypto.SSEC.IsRequested(r.Header) {
+			r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
+		}
 	}
+
 	// get gateway encryption options
 	var opts ObjectOptions
 	opts, err = putOpts(ctx, r, bucket, object, metadata)
@@ -920,9 +910,11 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		}
 	}
 
-	if _, ok := globalBucketObjectLockConfig.Get(bucket); (ok || globalWORMEnabled) && forceDelete {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
-		return
+	if forceDelete {
+		if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
+			return
+		}
 	}
 
 	deleteBucket := objectAPI.DeleteBucket
@@ -936,13 +928,13 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	if globalDNSConfig != nil {
 		if err := globalDNSConfig.Delete(bucket); err != nil {
 			// Deleting DNS entry failed, attempt to create the bucket again.
-			objectAPI.MakeBucketWithLocation(ctx, bucket, "")
+			objectAPI.MakeBucketWithLocation(ctx, bucket, "", false)
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
-	globalNotificationSys.DeleteBucket(ctx, bucket)
+	globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
 
 	// Write success response.
 	writeSuccessNoContent(w)
@@ -1021,15 +1013,11 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		return
 	}
 
-	// Deny if WORM is enabled
-	if globalWORMEnabled {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
-		return
-	}
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketObjectLockConfigurationAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	config, err := objectlock.ParseObjectLockConfig(r.Body)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
@@ -1037,39 +1025,23 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
 		return
 	}
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketObjectLockEnabledConfigFile)
-	configData, err := readConfig(ctx, objectAPI, configFile)
+
+	configData, err := xml.Marshal(config)
 	if err != nil {
-		aerr := toAPIError(ctx, err)
-		if err == errConfigNotFound {
-			aerr = errorCodes.ToAPIErr(ErrObjectLockConfigurationNotAllowed)
-		}
-		writeErrorResponse(ctx, w, aerr, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	if string(configData) != bucketObjectLockEnabledConfig {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL, guessIsBrowserReq(r))
-		return
-	}
-	data, err := xml.Marshal(config)
-	if err != nil {
+	// Deny object locking configuration settings on existing buckets without object lock enabled.
+	if _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	configFile = path.Join(bucketConfigPrefix, bucket, objectLockConfig)
-	if err = saveConfig(ctx, objectAPI, configFile, data); err != nil {
+
+	if err = globalBucketMetadataSys.Update(bucket, objectLockConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if config.Rule != nil {
-		retention := config.ToRetention()
-		globalBucketObjectLockConfig.Set(bucket, retention)
-		globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, retention)
-	} else {
-		globalBucketObjectLockConfig.Remove(bucket)
-		globalNotificationSys.RemoveBucketObjectLockConfig(ctx, bucket)
-	}
 
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
@@ -1093,43 +1065,137 @@ func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWri
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	// check if user has permissions to perform this operation
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketObjectLockConfigurationAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	configFile := path.Join(bucketConfigPrefix, bucket, bucketObjectLockEnabledConfigFile)
-	configData, err := readConfig(ctx, objectAPI, configFile)
+
+	config, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
 	if err != nil {
-		var aerr APIError
-		if err == errConfigNotFound {
-			aerr = errorCodes.ToAPIErr(ErrObjectLockConfigurationNotFound)
-		} else {
-			aerr = toAPIError(ctx, err)
-		}
-		writeErrorResponse(ctx, w, aerr, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	if string(configData) != bucketObjectLockEnabledConfig {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	configFile = path.Join(bucketConfigPrefix, bucket, objectLockConfig)
-	configData, err = readConfig(ctx, objectAPI, configFile)
+	configData, err := xml.Marshal(config)
 	if err != nil {
-		if err != errConfigNotFound {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-
-		if configData, err = xml.Marshal(objectlock.NewObjectLockConfig()); err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
 	}
 
 	// Write success response.
 	writeSuccessResponseXML(w, configData)
 }
+
+// PutBucketTaggingHandler - PUT Bucket tagging.
+// ----------
+func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketTagging")
+
+	defer logger.AuditLog(w, r, "PutBucketTagging", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketTaggingAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	tags, err := tags.ParseBucketXML(io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
+		apiErr.Description = err.Error()
+		writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	configData, err := xml.Marshal(tags)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if err = globalBucketMetadataSys.Update(bucket, bucketTaggingConfigFile, configData); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// GetBucketTaggingHandler - GET Bucket tagging.
+// ----------
+func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketTagging")
+
+	defer logger.AuditLog(w, r, "GetBucketTagging", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// check if user has permissions to perform this operation
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketTaggingAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	config, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	configData, err := xml.Marshal(config)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseXML(w, configData)
+}
+
+// DeleteBucketTaggingHandler - DELETE Bucket tagging.
+// ----------
+func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteBucketTagging")
+
+	defer logger.AuditLog(w, r, "DeleteBucketTagging", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketTaggingAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if err := globalBucketMetadataSys.Update(bucket, bucketTaggingConfigFile, nil); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write success response.
+	writeSuccessResponseHeadersOnly(w)
+}

cmd/bucket-handlers_test.go

@@ -115,7 +115,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 			errorResponse: APIErrorResponse{
 				Resource: SlashSeparator + bucketName + SlashSeparator,
 				Code:     "InvalidAccessKeyId",
-				Message:  "The access key ID you provided does not exist in our records.",
+				Message:  "The Access Key Id you provided does not exist in our records.",
 			},
 			shouldPass: false,
 		},

cmd/bucket-lifecycle-handlers.go

@@ -72,13 +72,22 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err = objAPI.SetBucketLifecycle(ctx, bucket, bucketLifecycle); err != nil {
+	// Validate the received bucket policy document
+	if err = bucketLifecycle.Validate(); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	globalLifecycleSys.Set(bucket, *bucketLifecycle)
-	globalNotificationSys.SetBucketLifecycle(ctx, bucket, bucketLifecycle)
+	configData, err := xml.Marshal(bucketLifecycle)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if err = globalBucketMetadataSys.Update(bucket, bucketLifecycleConfig, configData); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
 
 	// Success.
 	writeSuccessResponseHeadersOnly(w)
@@ -110,21 +119,20 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// Read bucket access lifecycle.
-	bucketLifecycle, err := objAPI.GetBucketLifecycle(ctx, bucket)
+	config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	lifecycleData, err := xml.Marshal(bucketLifecycle)
+	configData, err := xml.Marshal(config)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Write lifecycle configuration to client.
-	writeSuccessResponseXML(w, lifecycleData)
+	writeSuccessResponseXML(w, configData)
 }
 
 // DeleteBucketLifecycleHandler - This HTTP handler removes bucket lifecycle configuration.
@@ -153,14 +161,11 @@ func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter,
 		return
 	}
 
-	if err := objAPI.DeleteBucketLifecycle(ctx, bucket); err != nil {
+	if err := globalBucketMetadataSys.Update(bucket, bucketLifecycleConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	globalLifecycleSys.Remove(bucket)
-	globalNotificationSys.RemoveBucketLifecycle(ctx, bucket)
-
 	// Success.
 	writeSuccessNoContent(w)
 }

cmd/bucket-lifecycle-handlers_test.go

@@ -72,7 +72,7 @@ func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType,
 			errorResponse: APIErrorResponse{
 				Resource: SlashSeparator + bucketName + SlashSeparator,
 				Code:     "InvalidAccessKeyId",
-				Message:  "The access key ID you provided does not exist in our records.",
+				Message:  "The Access Key Id you provided does not exist in our records.",
 			},
 			shouldPass: false,
 		},
@@ -102,7 +102,7 @@ func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType,
 			errorResponse: APIErrorResponse{
 				Resource: SlashSeparator + bucketName + SlashSeparator,
 				Code:     "InvalidAccessKeyId",
-				Message:  "The access key ID you provided does not exist in our records.",
+				Message:  "The Access Key Id you provided does not exist in our records.",
 			},
 			shouldPass: false,
 		},
@@ -132,7 +132,7 @@ func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType,
 			errorResponse: APIErrorResponse{
 				Resource: SlashSeparator + bucketName + SlashSeparator,
 				Code:     "InvalidAccessKeyId",
-				Message:  "The access key ID you provided does not exist in our records.",
+				Message:  "The Access Key Id you provided does not exist in our records.",
 			},
 			shouldPass: false,
 		},

cmd/bucket-lifecycle.go

@@ -0,0 +1,49 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+)
+
+const (
+
+	// Disabled means the lifecycle rule is inactive
+	Disabled = "Disabled"
+)
+
+// LifecycleSys - Bucket lifecycle subsystem.
+type LifecycleSys struct{}
+
+// Get - gets lifecycle config associated to a given bucket name.
+func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) {
+	if globalIsGateway {
+		objAPI := newObjectLayerWithoutSafeModeFn()
+		if objAPI == nil {
+			return nil, errServerNotInitialized
+		}
+
+		return nil, BucketLifecycleNotFound{Bucket: bucketName}
+	}
+
+	return globalBucketMetadataSys.GetLifecycleConfig(bucketName)
+}
+
+// NewLifecycleSys - creates new lifecycle system.
+func NewLifecycleSys() *LifecycleSys {
+	return &LifecycleSys{}
+}

cmd/bucket-listobjects-handlers.go

@@ -99,24 +99,13 @@ func (api objectAPIHandlers) ListBucketObjectVersionsHandler(w http.ResponseWrit
 	}
 
 	for i := range listObjectsInfo.Objects {
-		var actualSize int64
-		if listObjectsInfo.Objects[i].IsCompressed() {
-			// Read the decompressed size from the meta.json.
-			actualSize = listObjectsInfo.Objects[i].GetActualSize()
-			if actualSize < 0 {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize),
-					r.URL, guessIsBrowserReq(r))
-				return
-			}
-			// Set the info.Size to the actualSize.
-			listObjectsInfo.Objects[i].Size = actualSize
-		} else if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
+		if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
 			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
-			listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
-			if err != nil {
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-				return
-			}
+		}
+		listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].GetActualSize()
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
 		}
 	}
 
@@ -181,23 +170,13 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 	}
 
 	for i := range listObjectsV2Info.Objects {
-		var actualSize int64
-		if listObjectsV2Info.Objects[i].IsCompressed() {
-			// Read the decompressed size from the meta.json.
-			actualSize = listObjectsV2Info.Objects[i].GetActualSize()
-			if actualSize < 0 {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
-				return
-			}
-			// Set the info.Size to the actualSize.
-			listObjectsV2Info.Objects[i].Size = actualSize
-		} else if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
+		if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
 			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
-			listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
-			if err != nil {
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-				return
-			}
+		}
+		listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].GetActualSize()
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
 		}
 	}
 
@@ -265,23 +244,13 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	}
 
 	for i := range listObjectsV2Info.Objects {
-		var actualSize int64
-		if listObjectsV2Info.Objects[i].IsCompressed() {
-			// Read the decompressed size from the meta.json.
-			actualSize = listObjectsV2Info.Objects[i].GetActualSize()
-			if actualSize < 0 {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
-				return
-			}
-			// Set the info.Size to the actualSize.
-			listObjectsV2Info.Objects[i].Size = actualSize
-		} else if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
+		if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
 			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
-			listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
-			if err != nil {
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-				return
-			}
+		}
+		listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].GetActualSize()
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
 		}
 	}
 
@@ -344,25 +313,16 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 
 	for i := range listObjectsInfo.Objects {
-		var actualSize int64
-		if listObjectsInfo.Objects[i].IsCompressed() {
-			// Read the decompressed size from the meta.json.
-			actualSize = listObjectsInfo.Objects[i].GetActualSize()
-			if actualSize < 0 {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
-				return
-			}
-			// Set the info.Size to the actualSize.
-			listObjectsInfo.Objects[i].Size = actualSize
-		} else if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
+		if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
 			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
-			listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
-			if err != nil {
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-				return
-			}
+		}
+		listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].GetActualSize()
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
 		}
 	}
+
 	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo)
 
 	// Write success response.

cmd/bucket-metadata-sys.go

@@ -0,0 +1,314 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/minio/minio-go/v6/pkg/tags"
+	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
+	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+// BucketMetadataSys captures all bucket metadata for a given cluster.
+type BucketMetadataSys struct {
+	sync.RWMutex
+	metadataMap map[string]BucketMetadata
+}
+
+// Remove bucket metadata from memory.
+func (sys *BucketMetadataSys) Remove(bucket string) {
+	if globalIsGateway && globalGatewayName != "nas" {
+		return
+	}
+	sys.Lock()
+	delete(sys.metadataMap, bucket)
+	sys.Unlock()
+}
+
+// Set - sets a new metadata in-memory.
+// Only a shallow copy is saved and fields with references
+// cannot be modified without causing a race condition,
+// so they should be replaced atomically and not appended to, etc.
+// Data is not persisted to disk.
+func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
+	if globalIsGateway {
+		return
+	}
+
+	if bucket != minioMetaBucket {
+		sys.Lock()
+		sys.metadataMap[bucket] = meta
+		sys.Unlock()
+	}
+}
+
+// Update update bucket metadata for the specified config file.
+// The configData data should not be modified after being sent here.
+func (sys *BucketMetadataSys) Update(bucket string, configFile string, configData []byte) error {
+	objAPI := newObjectLayerWithoutSafeModeFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	if globalIsGateway && globalGatewayName != "nas" {
+		// This code is needed only for gateway implementations.
+		if configFile == bucketPolicyConfig {
+			config, err := policy.ParseConfig(bytes.NewReader(configData), bucket)
+			if err != nil {
+				return err
+			}
+			return objAPI.SetBucketPolicy(GlobalContext, bucket, config)
+		}
+		return NotImplemented{}
+	}
+
+	if bucket == minioMetaBucket {
+		return errInvalidArgument
+	}
+
+	meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+	if err != nil {
+		return err
+	}
+
+	switch configFile {
+	case bucketPolicyConfig:
+		meta.PolicyConfigJSON = configData
+	case bucketNotificationConfig:
+		meta.NotificationConfigXML = configData
+	case bucketLifecycleConfig:
+		meta.LifecycleConfigXML = configData
+	case bucketSSEConfig:
+		meta.EncryptionConfigXML = configData
+	case bucketTaggingConfigFile:
+		meta.TaggingConfigXML = configData
+	case objectLockConfig:
+		meta.ObjectLockConfigXML = configData
+	case bucketQuotaConfigFile:
+		meta.QuotaConfigJSON = configData
+	default:
+		return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
+	}
+
+	if err := meta.Save(GlobalContext, objAPI); err != nil {
+		return err
+	}
+
+	sys.Set(bucket, meta)
+	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
+
+	return nil
+}
+
+// Get metadata for a bucket.
+// If no metadata exists errConfigNotFound is returned and a new metadata is returned.
+// Only a shallow copy is returned, so referenced data should not be modified,
+// but can be replaced atomically.
+func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
+	if globalIsGateway || bucket == minioMetaBucket {
+		return newBucketMetadata(bucket), errConfigNotFound
+	}
+
+	sys.RLock()
+	defer sys.RUnlock()
+
+	meta, ok := sys.metadataMap[bucket]
+	if !ok {
+		return newBucketMetadata(bucket), errConfigNotFound
+	}
+
+	return meta, nil
+}
+
+// GetTaggingConfig returns configured tagging config
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, error) {
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, BucketTaggingNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+	if meta.taggingConfig == nil {
+		return nil, BucketTaggingNotFound{Bucket: bucket}
+	}
+	return meta.taggingConfig, nil
+}
+
+// GetObjectLockConfig returns configured object lock config
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Config, error) {
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, BucketObjectLockConfigNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+	if meta.objectLockConfig == nil {
+		return nil, BucketObjectLockConfigNotFound{Bucket: bucket}
+	}
+	return meta.objectLockConfig, nil
+}
+
+// GetLifecycleConfig returns configured lifecycle config
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, error) {
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, BucketLifecycleNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+	if meta.lifecycleConfig == nil {
+		return nil, BucketLifecycleNotFound{Bucket: bucket}
+	}
+	return meta.lifecycleConfig, nil
+}
+
+// GetNotificationConfig returns configured notification config
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		return nil, err
+	}
+	return meta.notificationConfig, nil
+}
+
+// GetSSEConfig returns configured SSE config
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEConfig, error) {
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, BucketSSEConfigNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+	if meta.sseConfig == nil {
+		return nil, BucketSSEConfigNotFound{Bucket: bucket}
+	}
+	return meta.sseConfig, nil
+}
+
+// GetPolicyConfig returns configured bucket policy
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, error) {
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, BucketPolicyNotFound{Bucket: bucket}
+		}
+		return nil, err
+	}
+	if meta.policyConfig == nil {
+		return nil, BucketPolicyNotFound{Bucket: bucket}
+	}
+	return meta.policyConfig, nil
+}
+
+// GetQuotaConfig returns configured bucket quota
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetQuotaConfig(bucket string) (*madmin.BucketQuota, error) {
+	meta, err := sys.GetConfig(bucket)
+	if err != nil {
+		return nil, err
+	}
+	return meta.quotaConfig, nil
+}
+
+// GetConfig returns a specific configuration from the bucket metadata.
+// The returned object may not be modified.
+func (sys *BucketMetadataSys) GetConfig(bucket string) (BucketMetadata, error) {
+	objAPI := newObjectLayerWithoutSafeModeFn()
+	if objAPI == nil {
+		return newBucketMetadata(bucket), errServerNotInitialized
+	}
+
+	if globalIsGateway && globalGatewayName != "nas" {
+		return newBucketMetadata(bucket), NotImplemented{}
+	}
+
+	if bucket == minioMetaBucket {
+		return newBucketMetadata(bucket), errInvalidArgument
+	}
+
+	sys.RLock()
+	meta, ok := sys.metadataMap[bucket]
+	sys.RUnlock()
+	if ok {
+		return meta, nil
+	}
+	meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+	if err != nil {
+		return meta, err
+	}
+	sys.Lock()
+	sys.metadataMap[bucket] = meta
+	sys.Unlock()
+	return meta, nil
+}
+
+// Init - initializes bucket metadata system for all buckets.
+func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	// In gateway mode, we don't need to load the policies
+	// from the backend.
+	if globalIsGateway {
+		return nil
+	}
+
+	// Load PolicySys once during boot.
+	return sys.load(ctx, buckets, objAPI)
+}
+
+// Loads bucket metadata for all buckets into BucketMetadataSys.
+func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+	sys.Lock()
+	defer sys.Unlock()
+
+	for _, bucket := range buckets {
+		meta, err := loadBucketMetadata(ctx, objAPI, bucket.Name)
+		if err != nil {
+			return err
+		}
+		sys.metadataMap[bucket.Name] = meta
+	}
+	return nil
+}
+
+// NewBucketMetadataSys - creates new policy system.
+func NewBucketMetadataSys() *BucketMetadataSys {
+	return &BucketMetadataSys{
+		metadataMap: make(map[string]BucketMetadata),
+	}
+}

cmd/bucket-metadata.go

@@ -0,0 +1,311 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"path"
+	"time"
+
+	"github.com/minio/minio-go/v6/pkg/tags"
+	"github.com/minio/minio/cmd/logger"
+	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
+	"github.com/minio/minio/pkg/bucket/lifecycle"
+	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
+	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+const (
+	legacyBucketObjectLockEnabledConfigFile = "object-lock-enabled.json"
+	legacyBucketObjectLockEnabledConfig     = `{"x-amz-bucket-object-lock-enabled":true}`
+
+	bucketMetadataFile    = ".metadata.bin"
+	bucketMetadataFormat  = 1
+	bucketMetadataVersion = 1
+)
+
+var (
+	enabledBucketObjectLockConfig = []byte(`<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled></ObjectLockConfiguration>`)
+)
+
+//go:generate msgp -file $GOFILE
+
+// BucketMetadata contains bucket metadata.
+// When adding/removing fields, regenerate the marshal code using the go generate above.
+// Only changing meaning of fields requires a version bump.
+// bucketMetadataFormat refers to the format.
+// bucketMetadataVersion can be used to track a rolling upgrade of a field.
+type BucketMetadata struct {
+	Name                  string
+	Created               time.Time
+	LockEnabled           bool // legacy not used anymore.
+	PolicyConfigJSON      []byte
+	NotificationConfigXML []byte
+	LifecycleConfigXML    []byte
+	ObjectLockConfigXML   []byte
+	EncryptionConfigXML   []byte
+	TaggingConfigXML      []byte
+	QuotaConfigJSON       []byte
+
+	// Unexported fields. Must be updated atomically.
+	policyConfig       *policy.Policy
+	notificationConfig *event.Config
+	lifecycleConfig    *lifecycle.Lifecycle
+	objectLockConfig   *objectlock.Config
+	sseConfig          *bucketsse.BucketSSEConfig
+	taggingConfig      *tags.Tags
+	quotaConfig        *madmin.BucketQuota
+}
+
+// newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
+func newBucketMetadata(name string) BucketMetadata {
+	return BucketMetadata{
+		Name:    name,
+		Created: UTCNow(),
+		notificationConfig: &event.Config{
+			XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
+		},
+		quotaConfig: &madmin.BucketQuota{},
+	}
+}
+
+// Load - loads the metadata of bucket by name from ObjectLayer api.
+// If an error is returned the returned metadata will be default initialized.
+func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
+	configFile := path.Join(bucketConfigPrefix, name, bucketMetadataFile)
+	data, err := readConfig(ctx, api, configFile)
+	if err != nil {
+		return err
+	}
+	if len(data) <= 4 {
+		return fmt.Errorf("loadBucketMetadata: no data")
+	}
+	// Read header
+	switch binary.LittleEndian.Uint16(data[0:2]) {
+	case bucketMetadataFormat:
+	default:
+		return fmt.Errorf("loadBucketMetadata: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+	}
+	switch binary.LittleEndian.Uint16(data[2:4]) {
+	case bucketMetadataVersion:
+	default:
+		return fmt.Errorf("loadBucketMetadata: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+	}
+
+	// OK, parse data.
+	_, err = b.UnmarshalMsg(data[4:])
+	return err
+}
+
+// loadBucketMetadata loads and migrates to bucket metadata.
+func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
+	b := newBucketMetadata(bucket)
+	err := b.Load(ctx, objectAPI, bucket)
+	if err == nil {
+		return b, b.convertLegacyConfigs(ctx, objectAPI)
+	}
+
+	if err != errConfigNotFound {
+		return b, err
+	}
+
+	// Old bucket without bucket metadata. Hence we migrate existing settings.
+	return b, b.convertLegacyConfigs(ctx, objectAPI)
+}
+
+// parseAllConfigs will parse all configs and populate the private fields.
+// The first error encountered is returned.
+func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLayer) (err error) {
+	if len(b.PolicyConfigJSON) != 0 {
+		b.policyConfig, err = policy.ParseConfig(bytes.NewReader(b.PolicyConfigJSON), b.Name)
+		if err != nil {
+			return err
+		}
+	} else {
+		b.policyConfig = nil
+	}
+
+	if len(b.NotificationConfigXML) != 0 {
+		if err = xml.Unmarshal(b.NotificationConfigXML, b.notificationConfig); err != nil {
+			return err
+		}
+	}
+
+	if len(b.LifecycleConfigXML) != 0 {
+		b.lifecycleConfig, err = lifecycle.ParseLifecycleConfig(bytes.NewReader(b.LifecycleConfigXML))
+		if err != nil {
+			return err
+		}
+	} else {
+		b.lifecycleConfig = nil
+	}
+
+	if len(b.EncryptionConfigXML) != 0 {
+		b.sseConfig, err = bucketsse.ParseBucketSSEConfig(bytes.NewReader(b.EncryptionConfigXML))
+		if err != nil {
+			return err
+		}
+	} else {
+		b.sseConfig = nil
+	}
+
+	if len(b.TaggingConfigXML) != 0 {
+		b.taggingConfig, err = tags.ParseBucketXML(bytes.NewReader(b.TaggingConfigXML))
+		if err != nil {
+			return err
+		}
+	} else {
+		b.taggingConfig = nil
+	}
+
+	if len(b.ObjectLockConfigXML) != 0 {
+		b.objectLockConfig, err = objectlock.ParseObjectLockConfig(bytes.NewReader(b.ObjectLockConfigXML))
+		if err != nil {
+			return err
+		}
+	} else {
+		b.objectLockConfig = nil
+	}
+
+	if len(b.QuotaConfigJSON) != 0 {
+		b.quotaConfig, err = parseBucketQuota(b.Name, b.QuotaConfigJSON)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer) error {
+	legacyConfigs := []string{
+		legacyBucketObjectLockEnabledConfigFile,
+		bucketPolicyConfig,
+		bucketNotificationConfig,
+		bucketLifecycleConfig,
+		bucketQuotaConfigFile,
+		bucketSSEConfig,
+		bucketTaggingConfigFile,
+		objectLockConfig,
+	}
+
+	configs := make(map[string][]byte)
+
+	// Handle migration from lockEnabled to newer format.
+	if b.LockEnabled {
+		configs[objectLockConfig] = enabledBucketObjectLockConfig
+		b.LockEnabled = false // legacy value unset it
+		// we are only interested in b.ObjectLockConfigXML or objectLockConfig value
+	}
+
+	for _, legacyFile := range legacyConfigs {
+		configFile := path.Join(bucketConfigPrefix, b.Name, legacyFile)
+
+		configData, err := readConfig(ctx, objectAPI, configFile)
+		if err != nil {
+			if errors.Is(err, errConfigNotFound) {
+				// legacy file config not found, proceed to look for new metadata.
+				continue
+			}
+
+			return err
+		}
+		configs[legacyFile] = configData
+	}
+
+	if len(configs) == 0 {
+		// nothing to update, return right away.
+		return b.parseAllConfigs(ctx, objectAPI)
+	}
+
+	for legacyFile, configData := range configs {
+		switch legacyFile {
+		case legacyBucketObjectLockEnabledConfigFile:
+			if string(configData) == legacyBucketObjectLockEnabledConfig {
+				b.ObjectLockConfigXML = enabledBucketObjectLockConfig
+				b.LockEnabled = false // legacy value unset it
+				// we are only interested in b.ObjectLockConfigXML
+			}
+		case bucketPolicyConfig:
+			b.PolicyConfigJSON = configData
+		case bucketNotificationConfig:
+			b.NotificationConfigXML = configData
+		case bucketLifecycleConfig:
+			b.LifecycleConfigXML = configData
+		case bucketSSEConfig:
+			b.EncryptionConfigXML = configData
+		case bucketTaggingConfigFile:
+			b.TaggingConfigXML = configData
+		case objectLockConfig:
+			b.ObjectLockConfigXML = configData
+		case bucketQuotaConfigFile:
+			b.QuotaConfigJSON = configData
+		}
+	}
+
+	if err := b.Save(ctx, objectAPI); err != nil {
+		return err
+	}
+
+	for legacyFile := range configs {
+		configFile := path.Join(bucketConfigPrefix, b.Name, legacyFile)
+		if err := deleteConfig(ctx, objectAPI, configFile); err != nil && !errors.Is(err, errConfigNotFound) {
+			logger.LogIf(ctx, err)
+		}
+	}
+
+	return nil
+}
+
+// Save config to supplied ObjectLayer api.
+func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
+	if err := b.parseAllConfigs(ctx, api); err != nil {
+		return err
+	}
+
+	data := make([]byte, 4, b.Msgsize()+4)
+
+	// Initialize the header.
+	binary.LittleEndian.PutUint16(data[0:2], bucketMetadataFormat)
+	binary.LittleEndian.PutUint16(data[2:4], bucketMetadataVersion)
+
+	// Marshal the bucket metadata
+	data, err := b.MarshalMsg(data)
+	if err != nil {
+		return err
+	}
+
+	configFile := path.Join(bucketConfigPrefix, b.Name, bucketMetadataFile)
+	return saveConfig(ctx, api, configFile, data)
+}
+
+// deleteBucketMetadata deletes bucket metadata
+// If config does not exist no error is returned.
+func deleteBucketMetadata(ctx context.Context, obj ObjectLayer, bucket string) error {
+	configFile := path.Join(bucketConfigPrefix, bucket, bucketMetadataFile)
+	if err := deleteConfig(ctx, obj, configFile); err != nil && err != errConfigNotFound {
+		return err
+	}
+	return nil
+}

cmd/bucket-metadata_gen.go

@@ -0,0 +1,335 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Name":
+			z.Name, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "Created":
+			z.Created, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "Created")
+				return
+			}
+		case "LockEnabled":
+			z.LockEnabled, err = dc.ReadBool()
+			if err != nil {
+				err = msgp.WrapError(err, "LockEnabled")
+				return
+			}
+		case "PolicyConfigJSON":
+			z.PolicyConfigJSON, err = dc.ReadBytes(z.PolicyConfigJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "PolicyConfigJSON")
+				return
+			}
+		case "NotificationConfigXML":
+			z.NotificationConfigXML, err = dc.ReadBytes(z.NotificationConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "NotificationConfigXML")
+				return
+			}
+		case "LifecycleConfigXML":
+			z.LifecycleConfigXML, err = dc.ReadBytes(z.LifecycleConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "LifecycleConfigXML")
+				return
+			}
+		case "ObjectLockConfigXML":
+			z.ObjectLockConfigXML, err = dc.ReadBytes(z.ObjectLockConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "ObjectLockConfigXML")
+				return
+			}
+		case "EncryptionConfigXML":
+			z.EncryptionConfigXML, err = dc.ReadBytes(z.EncryptionConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "EncryptionConfigXML")
+				return
+			}
+		case "TaggingConfigXML":
+			z.TaggingConfigXML, err = dc.ReadBytes(z.TaggingConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "TaggingConfigXML")
+				return
+			}
+		case "QuotaConfigJSON":
+			z.QuotaConfigJSON, err = dc.ReadBytes(z.QuotaConfigJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "QuotaConfigJSON")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 10
+	// write "Name"
+	err = en.Append(0x8a, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Name)
+	if err != nil {
+		err = msgp.WrapError(err, "Name")
+		return
+	}
+	// write "Created"
+	err = en.Append(0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.Created)
+	if err != nil {
+		err = msgp.WrapError(err, "Created")
+		return
+	}
+	// write "LockEnabled"
+	err = en.Append(0xab, 0x4c, 0x6f, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteBool(z.LockEnabled)
+	if err != nil {
+		err = msgp.WrapError(err, "LockEnabled")
+		return
+	}
+	// write "PolicyConfigJSON"
+	err = en.Append(0xb0, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.PolicyConfigJSON)
+	if err != nil {
+		err = msgp.WrapError(err, "PolicyConfigJSON")
+		return
+	}
+	// write "NotificationConfigXML"
+	err = en.Append(0xb5, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.NotificationConfigXML)
+	if err != nil {
+		err = msgp.WrapError(err, "NotificationConfigXML")
+		return
+	}
+	// write "LifecycleConfigXML"
+	err = en.Append(0xb2, 0x4c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.LifecycleConfigXML)
+	if err != nil {
+		err = msgp.WrapError(err, "LifecycleConfigXML")
+		return
+	}
+	// write "ObjectLockConfigXML"
+	err = en.Append(0xb3, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.ObjectLockConfigXML)
+	if err != nil {
+		err = msgp.WrapError(err, "ObjectLockConfigXML")
+		return
+	}
+	// write "EncryptionConfigXML"
+	err = en.Append(0xb3, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.EncryptionConfigXML)
+	if err != nil {
+		err = msgp.WrapError(err, "EncryptionConfigXML")
+		return
+	}
+	// write "TaggingConfigXML"
+	err = en.Append(0xb0, 0x54, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.TaggingConfigXML)
+	if err != nil {
+		err = msgp.WrapError(err, "TaggingConfigXML")
+		return
+	}
+	// write "QuotaConfigJSON"
+	err = en.Append(0xaf, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.QuotaConfigJSON)
+	if err != nil {
+		err = msgp.WrapError(err, "QuotaConfigJSON")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 10
+	// string "Name"
+	o = append(o, 0x8a, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = msgp.AppendString(o, z.Name)
+	// string "Created"
+	o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
+	o = msgp.AppendTime(o, z.Created)
+	// string "LockEnabled"
+	o = append(o, 0xab, 0x4c, 0x6f, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64)
+	o = msgp.AppendBool(o, z.LockEnabled)
+	// string "PolicyConfigJSON"
+	o = append(o, 0xb0, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
+	o = msgp.AppendBytes(o, z.PolicyConfigJSON)
+	// string "NotificationConfigXML"
+	o = append(o, 0xb5, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	o = msgp.AppendBytes(o, z.NotificationConfigXML)
+	// string "LifecycleConfigXML"
+	o = append(o, 0xb2, 0x4c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	o = msgp.AppendBytes(o, z.LifecycleConfigXML)
+	// string "ObjectLockConfigXML"
+	o = append(o, 0xb3, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	o = msgp.AppendBytes(o, z.ObjectLockConfigXML)
+	// string "EncryptionConfigXML"
+	o = append(o, 0xb3, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	o = msgp.AppendBytes(o, z.EncryptionConfigXML)
+	// string "TaggingConfigXML"
+	o = append(o, 0xb0, 0x54, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
+	o = msgp.AppendBytes(o, z.TaggingConfigXML)
+	// string "QuotaConfigJSON"
+	o = append(o, 0xaf, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
+	o = msgp.AppendBytes(o, z.QuotaConfigJSON)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Name":
+			z.Name, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "Created":
+			z.Created, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Created")
+				return
+			}
+		case "LockEnabled":
+			z.LockEnabled, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "LockEnabled")
+				return
+			}
+		case "PolicyConfigJSON":
+			z.PolicyConfigJSON, bts, err = msgp.ReadBytesBytes(bts, z.PolicyConfigJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "PolicyConfigJSON")
+				return
+			}
+		case "NotificationConfigXML":
+			z.NotificationConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.NotificationConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "NotificationConfigXML")
+				return
+			}
+		case "LifecycleConfigXML":
+			z.LifecycleConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.LifecycleConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "LifecycleConfigXML")
+				return
+			}
+		case "ObjectLockConfigXML":
+			z.ObjectLockConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.ObjectLockConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "ObjectLockConfigXML")
+				return
+			}
+		case "EncryptionConfigXML":
+			z.EncryptionConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.EncryptionConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "EncryptionConfigXML")
+				return
+			}
+		case "TaggingConfigXML":
+			z.TaggingConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.TaggingConfigXML)
+			if err != nil {
+				err = msgp.WrapError(err, "TaggingConfigXML")
+				return
+			}
+		case "QuotaConfigJSON":
+			z.QuotaConfigJSON, bts, err = msgp.ReadBytesBytes(bts, z.QuotaConfigJSON)
+			if err != nil {
+				err = msgp.WrapError(err, "QuotaConfigJSON")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *BucketMetadata) Msgsize() (s int) {
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON)
+	return
+}

cmd/bucket-metadata_gen_test.go

@@ -0,0 +1,123 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalBucketMetadata(t *testing.T) {
+	v := BucketMetadata{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBucketMetadata(b *testing.B) {
+	v := BucketMetadata{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBucketMetadata(b *testing.B) {
+	v := BucketMetadata{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBucketMetadata(b *testing.B) {
+	v := BucketMetadata{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBucketMetadata(t *testing.T) {
+	v := BucketMetadata{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBucketMetadata Msgsize() is inaccurate")
+	}
+
+	vn := BucketMetadata{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBucketMetadata(b *testing.B) {
+	v := BucketMetadata{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBucketMetadata(b *testing.B) {
+	v := BucketMetadata{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/bucket-notification-handlers.go

@@ -19,11 +19,8 @@ package cmd
 import (
 	"encoding/json"
 	"encoding/xml"
-	"errors"
 	"io"
 	"net/http"
-	"net/url"
-	"path"
 	"reflect"
 	"time"
 
@@ -39,8 +36,6 @@ const (
 	bucketNotificationConfig = "notification.xml"
 )
 
-var errNoSuchNotifications = errors.New("The specified bucket does not have bucket notifications")
-
 // GetBucketNotificationHandler - This HTTP handler returns event notification configuration
 // as per http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html.
 // It returns empty configuration if its not set.
@@ -74,71 +69,44 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	// Construct path to notification.xml for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucketName, bucketNotificationConfig)
-
-	var config = event.Config{}
-	configData, err := readConfig(ctx, objAPI, configFile)
+	config, err := globalBucketMetadataSys.GetNotificationConfig(bucketName)
 	if err != nil {
-		if err != errConfigNotFound {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-		config.SetRegion(globalServerRegion)
-		config.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
-		notificationBytes, err := xml.Marshal(config)
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-
-		writeSuccessResponseXML(w, notificationBytes)
-		return
-	}
-
-	config.SetRegion(globalServerRegion)
-
-	if err = xml.Unmarshal(configData, &config); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-
+	config.SetRegion(globalServerRegion)
 	if err = config.Validate(globalServerRegion, globalNotificationSys.targetList); err != nil {
 		arnErr, ok := err.(*event.ErrARNNotFound)
-		if !ok {
+		if ok {
+			for i, queue := range config.QueueList {
+				// Remove ARN not found queues, because we previously allowed
+				// adding unexpected entries into the config.
+				//
+				// With newer config disallowing changing / turning off
+				// notification targets without removing ARN in notification
+				// configuration we won't see this problem anymore.
+				if reflect.DeepEqual(queue.ARN, arnErr.ARN) && i < len(config.QueueList) {
+					config.QueueList = append(config.QueueList[:i],
+						config.QueueList[i+1:]...)
+				}
+				// This is a one time activity we shall do this
+				// here and allow stale ARN to be removed. We shall
+				// never reach a stage where we will have stale
+				// notification configs.
+			}
+		} else {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		for i, queue := range config.QueueList {
-			// Remove ARN not found queues, because we previously allowed
-			// adding unexpected entries into the config.
-			//
-			// With newer config disallowing changing / turning off
-			// notification targets without removing ARN in notification
-			// configuration we won't see this problem anymore.
-			if reflect.DeepEqual(queue.ARN, arnErr.ARN) && i < len(config.QueueList) {
-				config.QueueList = append(config.QueueList[:i],
-					config.QueueList[i+1:]...)
-			}
-			// This is a one time activity we shall do this
-			// here and allow stale ARN to be removed. We shall
-			// never reach a stage where we will have stale
-			// notification configs.
-		}
 	}
 
-	// If xml namespace is empty, set a default value before returning.
-	if config.XMLNS == "" {
-		config.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
-	}
-
-	notificationBytes, err := xml.Marshal(config)
+	configData, err := xml.Marshal(config)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	writeSuccessResponseXML(w, notificationBytes)
+	writeSuccessResponseXML(w, configData)
 }
 
 // PutBucketNotificationHandler - This HTTP handler stores given notification configuration as per
@@ -179,8 +147,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	lreader := io.LimitReader(r.Body, r.ContentLength)
-	config, err := event.ParseConfig(lreader, globalServerRegion, globalNotificationSys.targetList)
+	config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalServerRegion, globalNotificationSys.targetList)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		if event.IsEventError(err) {
@@ -190,14 +157,19 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
+	configData, err := xml.Marshal(config)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if err = globalBucketMetadataSys.Update(bucketName, bucketNotificationConfig, configData); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	rulesMap := config.ToRulesMap()
 	globalNotificationSys.AddRulesMap(bucketName, rulesMap)
-	globalNotificationSys.PutBucketNotification(ctx, bucketName, rulesMap)
 
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -282,16 +254,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	w.Header().Set(xhttp.ContentType, "text/event-stream")
 
-	doneCh := make(chan struct{})
-	defer close(doneCh)
-
 	// Listen Publisher and peer-listen-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
 	listenCh := make(chan interface{}, 4000)
 
-	peers := getRestClients(globalEndpoints)
+	peers := newPeerRestClients(globalEndpoints)
 
-	globalHTTPListen.Subscribe(listenCh, doneCh, func(evI interface{}) bool {
+	globalHTTPListen.Subscribe(listenCh, ctx.Done(), func(evI interface{}) bool {
 		ev, ok := evI.(event.Event)
 		if !ok {
 			return false
@@ -299,18 +268,14 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		if ev.S3.Bucket.Name != values.Get(peerRESTListenBucket) {
 			return false
 		}
-		objectName, uerr := url.QueryUnescape(ev.S3.Object.Key)
-		if uerr != nil {
-			objectName = ev.S3.Object.Key
-		}
-		return len(rulesMap.Match(ev.EventName, objectName).ToSlice()) != 0
+		return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key)
 	})
 
 	for _, peer := range peers {
 		if peer == nil {
 			continue
 		}
-		peer.Listen(listenCh, doneCh, values)
+		peer.Listen(listenCh, ctx.Done(), values)
 	}
 
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
@@ -336,7 +301,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 				return
 			}
 			w.(http.Flusher).Flush()
-		case <-GlobalServiceDoneCh:
+		case <-ctx.Done():
 			return
 		}
 	}

cmd/bucket-object-lock.go

@@ -17,12 +17,9 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
-	"errors"
 	"math"
 	"net/http"
-	"path"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
@@ -30,8 +27,33 @@ import (
 	"github.com/minio/minio/pkg/bucket/policy"
 )
 
+// BucketObjectLockSys - map of bucket and retention configuration.
+type BucketObjectLockSys struct{}
+
+// Get - Get retention configuration.
+func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention, err error) {
+	if globalIsGateway {
+		objAPI := newObjectLayerWithoutSafeModeFn()
+		if objAPI == nil {
+			return r, errServerNotInitialized
+		}
+
+		return r, nil
+	}
+
+	config, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName)
+	if err != nil {
+		if _, ok := err.(BucketObjectLockConfigNotFound); ok {
+			return r, nil
+		}
+		return r, err
+
+	}
+	return config.ToRetention(), nil
+}
+
 // Similar to enforceRetentionBypassForDelete but for WebUI
-func enforceRetentionBypassForDeleteWeb(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn) APIErrorCode {
+func enforceRetentionBypassForDeleteWeb(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, govBypassPerms bool) APIErrorCode {
 	opts, err := getOpts(ctx, r, bucket, object)
 	if err != nil {
 		return toAPIErrorCode(ctx, err)
@@ -80,7 +102,7 @@ func enforceRetentionBypassForDeleteWeb(ctx context.Context, r *http.Request, bu
 			// and must explicitly include x-amz-bypass-governance-retention:true
 			// as a request header with any request that requires overriding
 			// governance mode.
-			byPassSet := objectlock.IsObjectLockGovernanceBypassSet(r.Header)
+			byPassSet := govBypassPerms && objectlock.IsObjectLockGovernanceBypassSet(r.Header)
 			if !byPassSet {
 				t, err := objectlock.UTCNowNTP()
 				if err != nil {
@@ -91,6 +113,11 @@ func enforceRetentionBypassForDeleteWeb(ctx context.Context, r *http.Request, bu
 				if !ret.RetainUntilDate.Before(t) {
 					return ErrObjectLocked
 				}
+
+				if !govBypassPerms {
+					return ErrObjectLocked
+				}
+
 				return ErrNone
 			}
 		}
@@ -98,9 +125,9 @@ func enforceRetentionBypassForDeleteWeb(ctx context.Context, r *http.Request, bu
 	return ErrNone
 }
 
-// enforceRetentionForLifecycle checks if it is appropriate to remove an
-// object according to locking configuration when this is lifecycle asking.
-func enforceRetentionForLifecycle(ctx context.Context, objInfo ObjectInfo) (locked bool) {
+// enforceRetentionForDeletion checks if it is appropriate to remove an
+// object according to locking configuration when this is lifecycle/ bucket quota asking.
+func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locked bool) {
 	lhold := objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
 	if lhold.Status.Valid() && lhold.Status == objectlock.LegalHoldOn {
 		return true
@@ -290,11 +317,16 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 	retentionRequested := objectlock.IsObjectLockRetentionRequested(r.Header)
 	legalHoldRequested := objectlock.IsObjectLockLegalHoldRequested(r.Header)
 
-	retentionCfg, isWORMBucket := globalBucketObjectLockConfig.Get(bucket)
-	if !isWORMBucket {
+	retentionCfg, err := globalBucketObjectLockSys.Get(bucket)
+	if err != nil {
+		return mode, retainDate, legalHold, ErrInvalidBucketObjectLockConfiguration
+	}
+
+	if !retentionCfg.LockEnabled {
 		if legalHoldRequested || retentionRequested {
 			return mode, retainDate, legalHold, ErrInvalidBucketObjectLockConfiguration
 		}
+
 		// If this not a WORM enabled bucket, we should return right here.
 		return mode, retainDate, legalHold, ErrNone
 	}
@@ -314,7 +346,7 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 	if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil {
 		objExists = true
 		r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
-		if globalWORMEnabled || ((r.Mode == objectlock.RetCompliance) && r.RetainUntilDate.After(t)) {
+		if r.Mode == objectlock.RetCompliance && r.RetainUntilDate.After(t) {
 			return mode, retainDate, legalHold, ErrObjectLocked
 		}
 		mode = r.Mode
@@ -351,7 +383,7 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 		return rMode, rDate, legalHold, ErrNone
 	}
 
-	if !retentionRequested && isWORMBucket {
+	if !retentionRequested && retentionCfg.Validity > 0 {
 		if retentionPermErr != ErrNone {
 			return mode, retainDate, legalHold, retentionPermErr
 		}
@@ -364,7 +396,7 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 		if objExists && retainDate.After(t) {
 			return mode, retainDate, legalHold, ErrObjectLocked
 		}
-		if !legalHoldRequested && !retentionCfg.IsEmpty() {
+		if !legalHoldRequested {
 			// inherit retention from bucket configuration
 			return retentionCfg.Mode, objectlock.RetentionDate{Time: t.Add(retentionCfg.Validity)}, legalHold, ErrNone
 		}
@@ -373,43 +405,7 @@ func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, obj
 	return mode, retainDate, legalHold, ErrNone
 }
 
-func initBucketObjectLockConfig(buckets []BucketInfo, objAPI ObjectLayer) error {
-	for _, bucket := range buckets {
-		ctx := logger.SetReqInfo(GlobalContext, &logger.ReqInfo{BucketName: bucket.Name})
-		configFile := path.Join(bucketConfigPrefix, bucket.Name, bucketObjectLockEnabledConfigFile)
-		bucketObjLockData, err := readConfig(ctx, objAPI, configFile)
-		if err != nil {
-			if errors.Is(err, errConfigNotFound) {
-				continue
-			}
-			return err
-		}
-
-		if string(bucketObjLockData) != bucketObjectLockEnabledConfig {
-			// this should never happen
-			logger.LogIf(ctx, objectlock.ErrMalformedBucketObjectConfig)
-			continue
-		}
-
-		configFile = path.Join(bucketConfigPrefix, bucket.Name, objectLockConfig)
-		configData, err := readConfig(ctx, objAPI, configFile)
-		if err != nil {
-			if errors.Is(err, errConfigNotFound) {
-				globalBucketObjectLockConfig.Set(bucket.Name, objectlock.Retention{})
-				continue
-			}
-			return err
-		}
-
-		config, err := objectlock.ParseObjectLockConfig(bytes.NewReader(configData))
-		if err != nil {
-			return err
-		}
-		retention := objectlock.Retention{}
-		if config.Rule != nil {
-			retention = config.ToRetention()
-		}
-		globalBucketObjectLockConfig.Set(bucket.Name, retention)
-	}
-	return nil
+// NewBucketObjectLockSys returns initialized BucketObjectLockSys
+func NewBucketObjectLockSys() *BucketObjectLockSys {
+	return &BucketObjectLockSys{}
 }

cmd/bucket-policy-handlers.go

@@ -87,13 +87,16 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if err = objAPI.SetBucketPolicy(ctx, bucket, bucketPolicy); err != nil {
+	configData, err := json.Marshal(bucketPolicy)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	globalPolicySys.Set(bucket, *bucketPolicy)
-	globalNotificationSys.SetBucketPolicy(ctx, bucket, bucketPolicy)
+	if err = globalBucketMetadataSys.Update(bucket, bucketPolicyConfig, configData); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
 
 	// Success.
 	writeSuccessNoContent(w)
@@ -125,14 +128,11 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err := objAPI.DeleteBucketPolicy(ctx, bucket); err != nil {
+	if err := globalBucketMetadataSys.Update(bucket, bucketPolicyConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	globalPolicySys.Remove(bucket)
-	globalNotificationSys.RemoveBucketPolicy(ctx, bucket)
-
 	// Success.
 	writeSuccessNoContent(w)
 }
@@ -164,18 +164,18 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Read bucket access policy.
-	bucketPolicy, err := objAPI.GetBucketPolicy(ctx, bucket)
+	config, err := globalBucketMetadataSys.GetPolicyConfig(bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	policyData, err := json.Marshal(bucketPolicy)
+	configData, err := json.Marshal(config)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Write to client.
-	w.Write(policyData)
+	writeSuccessResponseJSON(w, configData)
 }

cmd/bucket-policy-handlers_test.go

@@ -102,7 +102,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	credentials auth.Credentials, t *testing.T) {
 
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, ""); err != nil {
+	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, "", false); err != nil {
 		t.Fatal(err)
 	}
 

cmd/bucket-policy.go

@@ -17,15 +17,11 @@
 package cmd
 
 import (
-	"bytes"
-	"context"
 	"encoding/json"
 	"net/http"
 	"net/url"
-	"path"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
@@ -37,56 +33,30 @@ import (
 )
 
 // PolicySys - policy subsystem.
-type PolicySys struct {
-	sync.RWMutex
-	bucketPolicyMap map[string]policy.Policy
-}
+type PolicySys struct{}
 
-// Set - sets policy to given bucket name.  If policy is empty, existing policy is removed.
-func (sys *PolicySys) Set(bucketName string, policy policy.Policy) {
+// Get returns stored bucket policy
+func (sys *PolicySys) Get(bucket string) (*policy.Policy, error) {
 	if globalIsGateway {
-		// Set policy is a non-op under gateway mode.
-		return
+		objAPI := newObjectLayerFn()
+		if objAPI == nil {
+			return nil, errServerNotInitialized
+		}
+		return objAPI.GetBucketPolicy(GlobalContext, bucket)
 	}
-
-	sys.Lock()
-	defer sys.Unlock()
-
-	if policy.IsEmpty() {
-		delete(sys.bucketPolicyMap, bucketName)
-	} else {
-		sys.bucketPolicyMap[bucketName] = policy
-	}
-}
-
-// Remove - removes policy for given bucket name.
-func (sys *PolicySys) Remove(bucketName string) {
-	sys.Lock()
-	defer sys.Unlock()
-
-	delete(sys.bucketPolicyMap, bucketName)
+	return globalBucketMetadataSys.GetPolicyConfig(bucket)
 }
 
 // IsAllowed - checks given policy args is allowed to continue the Rest API.
 func (sys *PolicySys) IsAllowed(args policy.Args) bool {
-	if globalIsGateway {
-		// When gateway is enabled, no cached value
-		// is used to validate bucket policies.
-		objAPI := newObjectLayerFn()
-		if objAPI != nil {
-			config, err := objAPI.GetBucketPolicy(GlobalContext, args.BucketName)
-			if err == nil {
-				return config.IsAllowed(args)
-			}
-		}
-	} else {
-		sys.RLock()
-		defer sys.RUnlock()
+	p, err := sys.Get(args.BucketName)
+	if err == nil {
+		return p.IsAllowed(args)
+	}
 
-		// If policy is available for given bucket, check the policy.
-		if p, found := sys.bucketPolicyMap[args.BucketName]; found {
-			return p.IsAllowed(args)
-		}
+	// Log unhandled errors.
+	if _, ok := err.(BucketPolicyNotFound); !ok {
+		logger.LogIf(GlobalContext, err)
 	}
 
 	// As policy is not available for given bucket name, returns IsOwner i.e.
@@ -94,56 +64,9 @@ func (sys *PolicySys) IsAllowed(args policy.Args) bool {
 	return args.IsOwner
 }
 
-// Loads policies for all buckets into PolicySys.
-func (sys *PolicySys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
-	for _, bucket := range buckets {
-		config, err := objAPI.GetBucketPolicy(GlobalContext, bucket.Name)
-		if err != nil {
-			if _, ok := err.(BucketPolicyNotFound); ok {
-				sys.Remove(bucket.Name)
-			}
-			continue
-		}
-		// This part is specifically written to handle migration
-		// when the Version string is empty, this was allowed
-		// in all previous minio releases but we need to migrate
-		// those policies by properly setting the Version string
-		// from now on.
-		if config.Version == "" {
-			logger.Info("Found in-consistent bucket policies, Migrating them for Bucket: (%s)", bucket.Name)
-			config.Version = policy.DefaultVersion
-
-			if err = savePolicyConfig(GlobalContext, objAPI, bucket.Name, config); err != nil {
-				logger.LogIf(GlobalContext, err)
-				return err
-			}
-		}
-		sys.Set(bucket.Name, *config)
-	}
-	return nil
-}
-
-// Init - initializes policy system from policy.json of all buckets.
-func (sys *PolicySys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
-	if objAPI == nil {
-		return errInvalidArgument
-	}
-
-	// In gateway mode, we don't need to load the policies
-	// from the backend.
-	if globalIsGateway {
-		return nil
-	}
-
-	// Load PolicySys once during boot.
-	return sys.load(buckets, objAPI)
-}
-
 // NewPolicySys - creates new policy system.
 func NewPolicySys() *PolicySys {
-	return &PolicySys{
-		bucketPolicyMap: make(map[string]policy.Policy),
-	}
+	return &PolicySys{}
 }
 
 func getConditionValues(r *http.Request, lc string, username string, claims map[string]interface{}) map[string][]string {
@@ -226,50 +149,6 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 	return args
 }
 
-// getPolicyConfig - get policy config for given bucket name.
-func getPolicyConfig(objAPI ObjectLayer, bucketName string) (*policy.Policy, error) {
-	// Construct path to policy.json for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
-
-	configData, err := readConfig(GlobalContext, objAPI, configFile)
-	if err != nil {
-		if err == errConfigNotFound {
-			err = BucketPolicyNotFound{Bucket: bucketName}
-		}
-
-		return nil, err
-	}
-
-	return policy.ParseConfig(bytes.NewReader(configData), bucketName)
-}
-
-func savePolicyConfig(ctx context.Context, objAPI ObjectLayer, bucketName string, bucketPolicy *policy.Policy) error {
-	data, err := json.Marshal(bucketPolicy)
-	if err != nil {
-		return err
-	}
-
-	// Construct path to policy.json for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
-
-	return saveConfig(ctx, objAPI, configFile, data)
-}
-
-func removePolicyConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) error {
-	// Construct path to policy.json for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
-
-	if err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile); err != nil {
-		if _, ok := err.(ObjectNotFound); ok {
-			return BucketPolicyNotFound{Bucket: bucketName}
-		}
-
-		return err
-	}
-
-	return nil
-}
-
 // PolicyToBucketAccessPolicy - converts policy.Policy to minio-go/policy.BucketAccessPolicy.
 func PolicyToBucketAccessPolicy(bucketPolicy *policy.Policy) (*miniogopolicy.BucketAccessPolicy, error) {
 	// Return empty BucketAccessPolicy for empty bucket policy.

cmd/bucket-quota.go

@@ -0,0 +1,257 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/madmin"
+)
+
+// BucketQuotaSys - map of bucket and quota configuration.
+type BucketQuotaSys struct {
+	bucketStorageCache timedValue
+}
+
+// Get - Get quota configuration.
+func (sys *BucketQuotaSys) Get(bucketName string) (*madmin.BucketQuota, error) {
+	if globalIsGateway {
+		objAPI := newObjectLayerFn()
+		if objAPI == nil {
+			return nil, errServerNotInitialized
+		}
+		return &madmin.BucketQuota{}, nil
+	}
+
+	return globalBucketMetadataSys.GetQuotaConfig(bucketName)
+}
+
+// NewBucketQuotaSys returns initialized BucketQuotaSys
+func NewBucketQuotaSys() *BucketQuotaSys {
+	return &BucketQuotaSys{}
+}
+
+// parseBucketQuota parses BucketQuota from json
+func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota, err error) {
+	quotaCfg = &madmin.BucketQuota{}
+	if err = json.Unmarshal(data, quotaCfg); err != nil {
+		return quotaCfg, err
+	}
+	if !quotaCfg.IsValid() {
+		return quotaCfg, fmt.Errorf("Invalid quota config %#v", quotaCfg)
+	}
+	return
+}
+
+func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64) error {
+	objAPI := newObjectLayerWithoutSafeModeFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	q, err := sys.Get(bucket)
+	if err != nil {
+		return nil
+	}
+
+	if q.Type == madmin.FIFOQuota {
+		return nil
+	}
+
+	if q.Quota == 0 {
+		// No quota set return quickly.
+		return nil
+	}
+
+	sys.bucketStorageCache.Once.Do(func() {
+		sys.bucketStorageCache.TTL = 10 * time.Second
+		sys.bucketStorageCache.Update = func() (interface{}, error) {
+			return loadDataUsageFromBackend(ctx, objAPI)
+		}
+	})
+
+	v, err := sys.bucketStorageCache.Get()
+	if err != nil {
+		return err
+	}
+
+	dui := v.(DataUsageInfo)
+
+	bui, ok := dui.BucketsUsage[bucket]
+	if !ok {
+		// bucket not found, cannot enforce quota
+		// call will fail anyways later.
+		return nil
+	}
+
+	if (bui.Size + uint64(size)) > q.Quota {
+		return BucketQuotaExceeded{Bucket: bucket}
+	}
+
+	return nil
+}
+
+func enforceBucketQuota(ctx context.Context, bucket string, size int64) error {
+	if size < 0 {
+		return nil
+	}
+
+	return globalBucketQuotaSys.check(ctx, bucket, size)
+}
+
+const (
+	bgQuotaInterval = 1 * time.Hour
+)
+
+// initQuotaEnforcement starts the routine that deletes objects in bucket
+// that exceeds the FIFO quota
+func initQuotaEnforcement(ctx context.Context, objAPI ObjectLayer) {
+	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
+		go startBucketQuotaEnforcement(ctx, objAPI)
+	}
+}
+
+func startBucketQuotaEnforcement(ctx context.Context, objAPI ObjectLayer) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.NewTimer(bgQuotaInterval).C:
+			logger.LogIf(ctx, enforceFIFOQuota(ctx, objAPI))
+		}
+
+	}
+}
+
+// enforceFIFOQuota deletes objects in FIFO order until sufficient objects
+// have been deleted so as to bring bucket usage within quota
+func enforceFIFOQuota(ctx context.Context, objectAPI ObjectLayer) error {
+	// Turn off quota enforcement if data usage info is unavailable.
+	if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
+		return nil
+	}
+
+	buckets, err := objectAPI.ListBuckets(ctx)
+	if err != nil {
+		return err
+	}
+
+	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+	if err != nil {
+		return err
+	}
+
+	for _, binfo := range buckets {
+		bucket := binfo.Name
+
+		bui, ok := dataUsageInfo.BucketsUsage[bucket]
+		if !ok {
+			// bucket doesn't exist anymore, or we
+			// do not have any information to proceed.
+			continue
+		}
+
+		// Check if the current bucket has quota restrictions, if not skip it
+		cfg, err := globalBucketQuotaSys.Get(bucket)
+		if err != nil {
+			continue
+		}
+
+		if cfg.Type != madmin.FIFOQuota {
+			continue
+		}
+
+		var toFree uint64
+		if bui.Size > cfg.Quota && cfg.Quota > 0 {
+			toFree = bui.Size - cfg.Quota
+		}
+
+		if toFree == 0 {
+			continue
+		}
+
+		// Allocate new results channel to receive ObjectInfo.
+		objInfoCh := make(chan ObjectInfo)
+
+		// Walk through all objects
+		if err := objectAPI.Walk(ctx, bucket, "", objInfoCh); err != nil {
+			return err
+		}
+
+		// reuse the fileScorer used by disk cache to score entries by
+		// ModTime to find the oldest objects in bucket to delete. In
+		// the context of bucket quota enforcement - number of hits are
+		// irrelevant.
+		scorer, err := newFileScorer(toFree, time.Now().Unix(), 1)
+		if err != nil {
+			return err
+		}
+
+		rcfg, _ := globalBucketObjectLockSys.Get(bucket)
+
+		for obj := range objInfoCh {
+			// skip objects currently under retention
+			if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
+				continue
+			}
+			scorer.addFile(obj.Name, obj.ModTime, obj.Size, 1)
+		}
+		var objects []string
+		numKeys := len(scorer.fileNames())
+		for i, key := range scorer.fileNames() {
+			objects = append(objects, key)
+			if len(objects) < maxDeleteList && (i < numKeys-1) {
+				// skip deletion until maxObjectList or end of slice
+				continue
+			}
+
+			if len(objects) == 0 {
+				break
+			}
+			// Deletes a list of objects.
+			deleteErrs, err := objectAPI.DeleteObjects(ctx, bucket, objects)
+			if err != nil {
+				logger.LogIf(ctx, err)
+			} else {
+				for i := range deleteErrs {
+					if deleteErrs[i] != nil {
+						logger.LogIf(ctx, deleteErrs[i])
+						continue
+					}
+					// Notify object deleted event.
+					sendEvent(eventArgs{
+						EventName:  event.ObjectRemovedDelete,
+						BucketName: bucket,
+						Object: ObjectInfo{
+							Name: objects[i],
+						},
+						Host: "Internal: [FIFO-QUOTA-EXPIRY]",
+					})
+				}
+				objects = nil
+			}
+		}
+	}
+	return nil
+}

cmd/common-main.go

@@ -145,7 +145,7 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 
 	// Fetch address option
 	globalCLIContext.Addr = ctx.GlobalString("address")
-	if globalCLIContext.Addr == "" || globalCLIContext.Addr == ":"+globalMinioDefaultPort {
+	if globalCLIContext.Addr == "" || globalCLIContext.Addr == ":"+GlobalMinioDefaultPort {
 		globalCLIContext.Addr = ctx.String("address")
 	}
 
@@ -173,12 +173,24 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 }
 
 func handleCommonEnvVars() {
-	var err error
+	wormEnabled, err := config.LookupWorm()
+	if err != nil {
+		logger.Fatal(config.ErrInvalidWormValue(err), "Invalid worm configuration")
+	}
+	if wormEnabled {
+		logger.Fatal(errors.New("WORM is deprecated"), "global MINIO_WORM support is removed, please downgrade your server or migrate to https://github.com/minio/minio/tree/master/docs/retention")
+	}
+
 	globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.EnableOn))
 	if err != nil {
 		logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
 	}
 
+	globalFSOSync, err = config.ParseBool(env.Get(config.EnvFSOSync, config.EnableOff))
+	if err != nil {
+		logger.Fatal(config.ErrInvalidFSOSyncValue(err), "Invalid MINIO_FS_OSYNC value in environment variable")
+	}
+
 	domains := env.Get(config.EnvDomain, "")
 	if len(domains) != 0 {
 		for _, domainName := range strings.Split(domains, config.ValueSeparator) {
@@ -239,12 +251,6 @@ func handleCommonEnvVars() {
 		os.Unsetenv(config.EnvAccessKeyOld)
 		os.Unsetenv(config.EnvSecretKeyOld)
 	}
-
-	globalWORMEnabled, err = config.LookupWorm()
-	if err != nil {
-		logger.Fatal(config.ErrInvalidWormValue(err), "Invalid worm configuration")
-	}
-
 }
 
 func logStartupMessage(msg string) {

cmd/config-common.go

@@ -50,7 +50,11 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
 }
 
 func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
-	return objAPI.DeleteObject(ctx, minioMetaBucket, configFile)
+	err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile)
+	if err != nil && isErrObjectNotFound(err) {
+		return errConfigNotFound
+	}
+	return err
 }
 
 func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {

cmd/config-current.go

@@ -307,7 +307,7 @@ func validateConfig(s config.Config) error {
 		return err
 	}
 
-	return notify.TestNotificationTargets(s, GlobalServiceDoneCh, NewGatewayHTTPTransport(),
+	return notify.TestNotificationTargets(s, GlobalContext.Done(), NewGatewayHTTPTransport(),
 		globalNotificationSys.ConfiguredTargetIDs())
 }
 
@@ -371,6 +371,7 @@ func lookupConfigs(s config.Config) {
 	}
 
 	globalAPIThrottling.init(apiRequestsMax, apiConfig.APIRequestsDeadline)
+	globalReadyDeadline = apiConfig.APIReadyDeadline
 
 	if globalIsXL {
 		globalStorageClass, err = storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default],
@@ -469,12 +470,12 @@ func lookupConfigs(s config.Config) {
 		}
 	}
 
-	globalConfigTargetList, err = notify.GetNotificationTargets(s, GlobalServiceDoneCh, NewGatewayHTTPTransport())
+	globalConfigTargetList, err = notify.GetNotificationTargets(s, GlobalContext.Done(), NewGatewayHTTPTransport())
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
 
-	globalEnvTargetList, err = notify.GetNotificationTargets(newServerConfig(), GlobalServiceDoneCh, NewGatewayHTTPTransport())
+	globalEnvTargetList, err = notify.GetNotificationTargets(newServerConfig(), GlobalContext.Done(), NewGatewayHTTPTransport())
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}

cmd/config-encrypted.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 	"unicode/utf8"
 
 	etcd "github.com/coreos/etcd/clientv3"
@@ -35,43 +36,10 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 		return nil
 	}
 
-	// If its server mode or nas gateway, migrate the backend.
-	doneCh := make(chan struct{})
-
-	var encrypted bool
-	var err error
-
-	// Migrating Config backend needs a retry mechanism for
-	// the following reasons:
-	//  - Read quorum is lost just after the initialization
-	//    of the object layer.
-	retryTimerCh := newRetryTimerSimple(doneCh)
-	var stop bool
-
-	rquorum := InsufficientReadQuorum{}
-	wquorum := InsufficientWriteQuorum{}
-	bucketNotFound := BucketNotFound{}
-
-	for !stop {
-		select {
-		case <-retryTimerCh:
-			if encrypted, err = checkBackendEncrypted(objAPI); err != nil {
-				if errors.Is(err, errDiskNotFound) ||
-					errors.As(err, &rquorum) ||
-					errors.As(err, &bucketNotFound) {
-					logger.Info("Waiting for config backend to be encrypted..")
-					continue
-				}
-				close(doneCh)
-				return err
-			}
-			stop = true
-		case <-globalOSSignalCh:
-			close(doneCh)
-			return fmt.Errorf("Config encryption process stopped gracefully")
-		}
+	encrypted, err := checkBackendEncrypted(objAPI)
+	if err != nil {
+		return fmt.Errorf("Unable to encrypt config %w", err)
 	}
-	close(doneCh)
 
 	if encrypted {
 		// backend is encrypted, but credentials are not specified
@@ -91,34 +59,12 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer, server bool) error {
 		}
 	}
 
-	doneCh = make(chan struct{})
-	defer close(doneCh)
-
-	retryTimerCh = newRetryTimerSimple(doneCh)
-
-	// Migrating Config backend needs a retry mechanism for
-	// the following reasons:
-	//  - Read quorum is lost just after the initialization
-	//    of the object layer.
-	for {
-		select {
-		case <-retryTimerCh:
-			// Migrate IAM configuration
-			if err = migrateConfigPrefixToEncrypted(objAPI, globalOldCred, encrypted); err != nil {
-				if errors.Is(err, errDiskNotFound) ||
-					errors.As(err, &rquorum) ||
-					errors.As(err, &wquorum) ||
-					errors.As(err, &bucketNotFound) {
-					logger.Info("Waiting for config backend to be encrypted..")
-					continue
-				}
-				return err
-			}
-			return nil
-		case <-globalOSSignalCh:
-			return fmt.Errorf("Config encryption process stopped gracefully")
-		}
+	// Migrate IAM configuration
+	if err = migrateConfigPrefixToEncrypted(objAPI, globalOldCred, encrypted); err != nil {
+		return fmt.Errorf("Unable to migrate all config at .minio.sys/config/: %w", err)
 	}
+
+	return nil
 }
 
 const (
@@ -164,9 +110,6 @@ func decryptData(edata []byte, creds ...auth.Credentials) ([]byte, error) {
 }
 
 func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client) error {
-	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
-	defer cancel()
-
 	encrypted, err := checkBackendEtcdEncrypted(ctx, client)
 	if err != nil {
 		return err
@@ -207,7 +150,10 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 		logger.Info("Attempting encryption of all IAM users and policies on etcd")
 	}
 
-	r, err := client.Get(ctx, minioConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
+	listCtx, cancel := context.WithTimeout(ctx, 1*time.Minute)
+	defer cancel()
+
+	r, err := client.Get(listCtx, minioConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
 	if err != nil {
 		return err
 	}

cmd/config.go

@@ -40,9 +40,6 @@ const (
 
 	// MinIO configuration file.
 	minioConfigFile = "config.json"
-
-	// MinIO configuration backup file
-	minioConfigBackupFile = minioConfigFile + ".backup"
 )
 
 func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData bool, count int) (
@@ -168,13 +165,14 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 		}
 	}
 
-	var config = config.New()
+	var srvCfg = config.New()
 	var json = jsoniter.ConfigCompatibleWithStandardLibrary
-	if err = json.Unmarshal(configData, &config); err != nil {
+	if err = json.Unmarshal(configData, &srvCfg); err != nil {
 		return nil, err
 	}
 
-	return config, nil
+	// Add any missing entries
+	return srvCfg.Merge(), nil
 }
 
 // ConfigSys - config system.
@@ -186,16 +184,14 @@ func (sys *ConfigSys) Load(objAPI ObjectLayer) error {
 }
 
 // WatchConfigNASDisk - watches nas disk on periodic basis.
-func (sys *ConfigSys) WatchConfigNASDisk(objAPI ObjectLayer) {
+func (sys *ConfigSys) WatchConfigNASDisk(ctx context.Context, objAPI ObjectLayer) {
 	configInterval := globalRefreshIAMInterval
 	watchDisk := func() {
-		ticker := time.NewTicker(configInterval)
-		defer ticker.Stop()
 		for {
 			select {
-			case <-GlobalServiceDoneCh:
+			case <-ctx.Done():
 				return
-			case <-ticker.C:
+			case <-time.After(configInterval):
 				loadConfig(objAPI)
 			}
 		}

cmd/config/api/api.go

@@ -29,6 +29,7 @@ import (
 const (
 	apiRequestsMax      = "requests_max"
 	apiRequestsDeadline = "requests_deadline"
+	apiReadyDeadline    = "ready_deadline"
 )
 
 // DefaultKVS - default storage class config
@@ -42,6 +43,10 @@ var (
 			Key:   apiRequestsDeadline,
 			Value: "10s",
 		},
+		config.KV{
+			Key:   apiReadyDeadline,
+			Value: "10s",
+		},
 	}
 )
 
@@ -49,6 +54,7 @@ var (
 type Config struct {
 	APIRequestsMax      int           `json:"requests_max"`
 	APIRequestsDeadline time.Duration `json:"requests_deadline"`
+	APIReadyDeadline    time.Duration `json:"ready_deadline"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -83,9 +89,15 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, err
 	}
 
+	readyDeadline, err := time.ParseDuration(env.Get(config.EnvAPIReadyDeadline, kvs.Get(apiReadyDeadline)))
+	if err != nil {
+		return cfg, err
+	}
+
 	cfg = Config{
 		APIRequestsMax:      requestsMax,
 		APIRequestsDeadline: requestsDeadline,
+		APIReadyDeadline:    readyDeadline,
 	}
 	return cfg, nil
 }

cmd/config/api/help.go

@@ -33,5 +33,11 @@ var (
 			Optional:    true,
 			Type:        "duration",
 		},
+		config.HelpKV{
+			Key:         apiReadyDeadline,
+			Description: `set the deadline for health check API /minio/health/ready e.g. "1m"`,
+			Optional:    true,
+			Type:        "duration",
+		},
 	}
 )

cmd/config/certsinfo.go

@@ -0,0 +1,92 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package config
+
+import (
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"fmt"
+	"net/http"
+	"strings"
+
+	color "github.com/minio/minio/pkg/color"
+)
+
+// Extra ASN1 OIDs that we may need to handle
+var (
+	oidEmailAddress = []int{1, 2, 840, 113549, 1, 9, 1}
+)
+
+// printName prints the fields of a distinguished name, which include such
+// things as its common name and locality.
+func printName(names []pkix.AttributeTypeAndValue, buf *strings.Builder) []string {
+	values := []string{}
+	for _, name := range names {
+		oid := name.Type
+		if len(oid) == 4 && oid[0] == 2 && oid[1] == 5 && oid[2] == 4 {
+			switch oid[3] {
+			case 3:
+				values = append(values, fmt.Sprintf("CN=%s", name.Value))
+			case 6:
+				values = append(values, fmt.Sprintf("C=%s", name.Value))
+			case 8:
+				values = append(values, fmt.Sprintf("ST=%s", name.Value))
+			case 10:
+				values = append(values, fmt.Sprintf("O=%s", name.Value))
+			case 11:
+				values = append(values, fmt.Sprintf("OU=%s", name.Value))
+			default:
+				values = append(values, fmt.Sprintf("UnknownOID=%s", name.Type.String()))
+			}
+		} else if oid.Equal(oidEmailAddress) {
+			values = append(values, fmt.Sprintf("emailAddress=%s", name.Value))
+		} else {
+			values = append(values, fmt.Sprintf("UnknownOID=%s", name.Type.String()))
+		}
+	}
+	if len(values) > 0 {
+		buf.WriteString(values[0])
+		for i := 1; i < len(values); i++ {
+			buf.WriteString(", " + values[i])
+		}
+		buf.WriteString("\n")
+	}
+	return values
+}
+
+// CertificateText returns a human-readable string representation
+// of the certificate cert. The format is similar to the OpenSSL
+// way of printing certificates (not identical).
+func CertificateText(cert *x509.Certificate) string {
+	var buf strings.Builder
+
+	buf.WriteString(color.Blue("\nCertificate:\n"))
+	if cert.SignatureAlgorithm != x509.UnknownSignatureAlgorithm {
+		buf.WriteString(color.Blue("%4sSignature Algorithm: ", "") + color.Bold(fmt.Sprintf("%s\n", cert.SignatureAlgorithm)))
+	}
+
+	// Issuer information
+	buf.WriteString(color.Blue("%4sIssuer: ", ""))
+	printName(cert.Issuer.Names, &buf)
+
+	// Validity information
+	buf.WriteString(color.Blue("%4sValidity\n", ""))
+	buf.WriteString(color.Bold(fmt.Sprintf("%8sNot Before: %s\n", "", cert.NotBefore.Format(http.TimeFormat))))
+	buf.WriteString(color.Bold(fmt.Sprintf("%8sNot After : %s\n", "", cert.NotAfter.Format(http.TimeFormat))))
+
+	return buf.String()
+}

cmd/config/config.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"io"
 	"regexp"
-	"sort"
 	"strings"
 
 	"github.com/minio/minio-go/v6/pkg/set"
@@ -435,6 +434,26 @@ func LookupWorm() (bool, error) {
 	return ParseBool(env.Get(EnvWorm, EnableOff))
 }
 
+// Merge - merges a new config with all the
+// missing values for default configs,
+// returns a config.
+func (c Config) Merge() Config {
+	cp := New()
+	for subSys, tgtKV := range c {
+		for tgt := range tgtKV {
+			ckvs := c[subSys][tgt]
+			for _, kv := range cp[subSys][Default] {
+				_, ok := c[subSys][tgt].Lookup(kv.Key)
+				if !ok {
+					ckvs.Set(kv.Key, kv.Value)
+				}
+			}
+			cp[subSys][tgt] = ckvs
+		}
+	}
+	return cp
+}
+
 // New - initialize a new server config.
 func New() Config {
 	srvCfg := make(Config)
@@ -580,33 +599,6 @@ func (c Config) Clone() Config {
 	return cp
 }
 
-// Converts an input string of form "k1=v1 k2=v2" into fields
-// of ["k1=v1", "k2=v2"], the tokenization of each `k=v`
-// happens with the right number of input keys, if keys
-// input is empty returned value is empty slice as well.
-func kvFields(input string, keys []string) []string {
-	var valueIndexes []int
-	for _, key := range keys {
-		i := strings.Index(input, key+KvSeparator)
-		if i == -1 {
-			continue
-		}
-		valueIndexes = append(valueIndexes, i)
-	}
-
-	sort.Ints(valueIndexes)
-	var fields = make([]string, len(valueIndexes))
-	for i := range valueIndexes {
-		j := i + 1
-		if j < len(valueIndexes) {
-			fields[i] = strings.TrimSpace(input[valueIndexes[i]:valueIndexes[j]])
-		} else {
-			fields[i] = strings.TrimSpace(input[valueIndexes[i]:])
-		}
-	}
-	return fields
-}
-
 // SetKVS - set specific key values per sub-system.
 func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 	if len(s) == 0 {
@@ -635,7 +627,7 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) error {
 		tgt = subSystemValue[1]
 	}
 
-	fields := kvFields(inputs[1], defaultKVS[subSys].Keys())
+	fields := madmin.KvFields(inputs[1], defaultKVS[subSys].Keys())
 	if len(fields) == 0 {
 		return Errorf("sub-system '%s' cannot have empty keys", subSys)
 	}

cmd/config/config_test.go

@@ -19,6 +19,8 @@ package config
 
 import (
 	"testing"
+
+	"github.com/minio/minio/pkg/madmin"
 )
 
 func TestKVFields(t *testing.T) {
@@ -90,7 +92,7 @@ func TestKVFields(t *testing.T) {
 	for _, test := range tests {
 		test := test
 		t.Run("", func(t *testing.T) {
-			gotFields := kvFields(test.input, test.keys)
+			gotFields := madmin.KvFields(test.input, test.keys)
 			if len(gotFields) != len(test.expectedFields) {
 				t.Errorf("Expected keys %d, found %d", len(test.expectedFields), len(gotFields))
 			}

cmd/config/constants.go

@@ -32,10 +32,12 @@ const (
 	EnvRegionName   = "MINIO_REGION_NAME"
 	EnvPublicIPs    = "MINIO_PUBLIC_IPS"
 	EnvEndpoints    = "MINIO_ENDPOINTS"
+	EnvFSOSync      = "MINIO_FS_OSYNC"
 
 	// API sub-system
 	EnvAPIRequestsMax      = "MINIO_API_REQUESTS_MAX"
 	EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE"
+	EnvAPIReadyDeadline    = "MINIO_API_READY_DEADLINE"
 
 	EnvUpdate = "MINIO_UPDATE"
 

cmd/config/errors.go

@@ -24,6 +24,12 @@ var (
 		"Browser can only accept `on` and `off` values. To disable web browser access, set this value to `off`",
 	)
 
+	ErrInvalidFSOSyncValue = newErrFn(
+		"Invalid O_SYNC value",
+		"Please check the passed value",
+		"Can only accept `on` and `off` values. To enable O_SYNC for fs backend, set this value to `on`",
+	)
+
 	ErrInvalidDomainValue = newErrFn(
 		"Invalid domain value",
 		"Please check the passed value",
@@ -169,6 +175,18 @@ Example 1:
    $ minio server /data/minio/`,
 	)
 
+	ErrUnsupportedBackend = newErrFn(
+		"Unable to write to the backend",
+		"Please ensure your disk supports O_DIRECT",
+		"",
+	)
+
+	ErrCorruptedBackend = newErrFn(
+		"Unable to use the specified backend, pre-existing content detected",
+		"Please ensure your disk mount does not have any pre-existing content",
+		"",
+	)
+
 	ErrUnableToWriteInBackend = newErrFn(
 		"Unable to write to the backend",
 		"Please ensure MinIO binary has write permissions for the backend",

cmd/config/etcd/dns/types.go

@@ -23,7 +23,6 @@ import (
 
 const (
 	defaultTTL            = 30
-	defaultPrefixPath     = "/skydns"
 	defaultContextTimeout = 5 * time.Minute
 )
 

cmd/config/identity/ldap/config.go

@@ -58,6 +58,7 @@ type Config struct {
 	stsExpiryDuration time.Duration // contains converted value
 	tlsSkipVerify     bool          // allows skipping TLS verification
 	serverInsecure    bool          // allows plain text connection to LDAP Server
+	serverStartTLS    bool          // allows plain text connection to LDAP Server
 	rootCAs           *x509.CertPool
 }
 
@@ -73,11 +74,13 @@ const (
 	GroupSearchBaseDN    = "group_search_base_dn"
 	TLSSkipVerify        = "tls_skip_verify"
 	ServerInsecure       = "server_insecure"
+	ServerStartTLS       = "server_starttls"
 
 	EnvServerAddr           = "MINIO_IDENTITY_LDAP_SERVER_ADDR"
 	EnvSTSExpiry            = "MINIO_IDENTITY_LDAP_STS_EXPIRY"
 	EnvTLSSkipVerify        = "MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY"
 	EnvServerInsecure       = "MINIO_IDENTITY_LDAP_SERVER_INSECURE"
+	EnvServerStartTLS       = "MINIO_IDENTITY_LDAP_SERVER_STARTTLS"
 	EnvUsernameFormat       = "MINIO_IDENTITY_LDAP_USERNAME_FORMAT"
 	EnvUsernameSearchFilter = "MINIO_IDENTITY_LDAP_USERNAME_SEARCH_FILTER"
 	EnvUsernameSearchBaseDN = "MINIO_IDENTITY_LDAP_USERNAME_SEARCH_BASE_DN"
@@ -129,6 +132,10 @@ var (
 			Key:   ServerInsecure,
 			Value: config.EnableOff,
 		},
+		config.KV{
+			Key:   ServerStartTLS,
+			Value: config.EnableOff,
+		},
 	}
 )
 
@@ -257,6 +264,18 @@ func (l *Config) Connect() (ldapConn *ldap.Conn, err error) {
 		return ldap.Dial("tcp", l.ServerAddr)
 	}
 
+	if l.serverStartTLS {
+		conn, err := ldap.Dial("tcp", l.ServerAddr)
+		if err != nil {
+			return nil, err
+		}
+		err = conn.StartTLS(&tls.Config{
+			InsecureSkipVerify: l.tlsSkipVerify,
+			RootCAs:            l.rootCAs,
+		})
+		return conn, err
+	}
+
 	return ldap.DialTLS("tcp", l.ServerAddr, &tls.Config{
 		InsecureSkipVerify: l.tlsSkipVerify,
 		RootCAs:            l.rootCAs,
@@ -303,6 +322,12 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
 			return l, err
 		}
 	}
+	if v := env.Get(EnvServerStartTLS, kvs.Get(ServerStartTLS)); v != "" {
+		l.serverStartTLS, err = config.ParseBool(v)
+		if err != nil {
+			return l, err
+		}
+	}
 	if v := env.Get(EnvTLSSkipVerify, kvs.Get(TLSSkipVerify)); v != "" {
 		l.tlsSkipVerify, err = config.ParseBool(v)
 		if err != nil {

cmd/config/identity/openid/ecdsa-sha3.go

@@ -18,7 +18,9 @@ package openid
 
 import (
 	"crypto"
+
 	"github.com/dgrijalva/jwt-go"
+
 	// Needed for SHA3 to work - See: https://golang.org/src/crypto/crypto.go?s=1034:1288
 	_ "golang.org/x/crypto/sha3"
 )

cmd/config/identity/openid/jwt.go

@@ -30,7 +30,7 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/env"
-	"github.com/minio/minio/pkg/iam/policy"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	xnet "github.com/minio/minio/pkg/net"
 )
 

cmd/config/identity/openid/rsa-sha3.go

@@ -18,7 +18,9 @@ package openid
 
 import (
 	"crypto"
+
 	"github.com/dgrijalva/jwt-go"
+
 	// Needed for SHA3 to work - See: https://golang.org/src/crypto/crypto.go?s=1034:1288
 	_ "golang.org/x/crypto/sha3"
 )

cmd/config/notify/help.go

@@ -24,7 +24,7 @@ import (
 const (
 	formatComment     = `'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'`
 	queueDirComment   = `staging dir for undelivered messages e.g. '/home/events'`
-	queueLimitComment = `maximum limit for undelivered messages, defaults to '10000'`
+	queueLimitComment = `maximum limit for undelivered messages, defaults to '100000'`
 )
 
 // Help template inputs for all notification targets

cmd/config/notify/parse.go

@@ -36,7 +36,6 @@ import (
 
 const (
 	formatNamespace = "namespace"
-	formatAccess    = "access"
 )
 
 // ErrTargetsOffline - Indicates single/multiple target failures.
@@ -102,8 +101,10 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 	var targetsOffline bool
 
 	defer func() {
-		// Automatically close all connections to targets when an error occur
-		if err != nil {
+		// Automatically close all connections to targets when an error occur.
+		// Close all the targets if returnOnTargetError is set
+		// Else, close only the failed targets
+		if err != nil && returnOnTargetError {
 			for _, t := range targetList.TargetMap() {
 				_ = t.Close()
 			}
@@ -174,6 +175,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 			if returnOnTargetError {
 				return nil, err
 			}
+			_ = newTarget.Close()
 		}
 
 		if err = targetList.Add(newTarget); err != nil {
@@ -194,6 +196,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 			if returnOnTargetError {
 				return nil, err
 			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			logger.LogIf(context.Background(), err)
@@ -214,6 +217,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 			if returnOnTargetError {
 				return nil, err
 			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			logger.LogIf(context.Background(), err)
@@ -234,6 +238,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 			if returnOnTargetError {
 				return nil, err
 			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			logger.LogIf(context.Background(), err)
@@ -253,6 +258,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 			if returnOnTargetError {
 				return nil, err
 			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			logger.LogIf(context.Background(), err)
@@ -272,6 +278,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 			if returnOnTargetError {
 				return nil, err
 			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			logger.LogIf(context.Background(), err)
@@ -291,6 +298,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 			if returnOnTargetError {
 				return nil, err
 			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			logger.LogIf(context.Background(), err)
@@ -310,6 +318,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 			if returnOnTargetError {
 				return nil, err
 			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			logger.LogIf(context.Background(), err)
@@ -329,6 +338,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 			if returnOnTargetError {
 				return nil, err
 			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			logger.LogIf(context.Background(), err)
@@ -348,6 +358,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 			if returnOnTargetError {
 				return nil, err
 			}
+			_ = newTarget.Close()
 		}
 		if err = targetList.Add(newTarget); err != nil {
 			logger.LogIf(context.Background(), err)

cmd/consolelogger.go

@@ -74,7 +74,7 @@ func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
 }
 
 // Subscribe starts console logging for this node.
-func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh chan struct{}, node string, last int, logKind string, filter func(entry interface{}) bool) {
+func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan struct{}, node string, last int, logKind string, filter func(entry interface{}) bool) {
 	// Enable console logging for remote client.
 	if !sys.HasLogListeners() {
 		logger.AddTarget(sys)

cmd/crypto/kes.go

@@ -1,4 +1,4 @@
-// MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+// MinIO Cloud Storage, (C) 2019-2020 MinIO, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -27,6 +28,10 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
+
+	xhttp "github.com/minio/minio/cmd/http"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // KesConfig contains the configuration required
@@ -112,6 +117,7 @@ func NewKes(cfg KesConfig) (KMS, error) {
 		Certificates: []tls.Certificate{cert},
 		RootCAs:      certPool,
 	}
+	cfg.Transport.ForceAttemptHTTP2 = true
 	return &kesService{
 		client: &kesClient{
 			addr: cfg.Endpoint,
@@ -208,6 +214,18 @@ type kesClient struct {
 	httpClient http.Client
 }
 
+// Response KES response struct
+type response struct {
+	Plaintext  []byte `json:"plaintext"`
+	Ciphertext []byte `json:"ciphertext,omitempty"`
+}
+
+// Request KES request struct
+type request struct {
+	Ciphertext []byte `json:"ciphertext,omitempty"`
+	Context    []byte `json:"context"`
+}
+
 // GenerateDataKey requests a new data key from the KES server.
 // On success, the KES server will respond with the plaintext key
 // and the ciphertext key as the plaintext key encrypted with
@@ -217,10 +235,7 @@ type kesClient struct {
 // such that you have to provide the same context when decrypting
 // the data key.
 func (c *kesClient) GenerateDataKey(name string, context []byte) ([]byte, []byte, error) {
-	type Request struct {
-		Context []byte `json:"context"`
-	}
-	body, err := json.Marshal(Request{
+	body, err := json.Marshal(request{
 		Context: context,
 	})
 	if err != nil {
@@ -228,25 +243,57 @@ func (c *kesClient) GenerateDataKey(name string, context []byte) ([]byte, []byte
 	}
 
 	url := fmt.Sprintf("%s/v1/key/generate/%s", c.addr, url.PathEscape(name))
-	resp, err := c.httpClient.Post(url, "application/json", bytes.NewReader(body))
+
+	const limit = 1 << 20 // A plaintext/ciphertext key pair will never be larger than 1 MB
+	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
 	if err != nil {
 		return nil, nil, err
 	}
-	if resp.StatusCode != http.StatusOK {
-		return nil, nil, c.parseErrorResponse(resp)
-	}
-	defer resp.Body.Close()
 
-	type Response struct {
-		Plaintext  []byte `json:"plaintext"`
-		Ciphertext []byte `json:"ciphertext"`
+	return resp.Plaintext, resp.Ciphertext, nil
+}
+
+func (c *kesClient) post(url string, body io.Reader, limit int64) (*response, error) {
+	resp, err := c.httpClient.Post(url, "application/json", body)
+	if err != nil {
+		return nil, err
 	}
-	const limit = 1 << 20 // A plaintext/ciphertext key pair will never be larger than 1 MB
-	var response Response
-	if err = json.NewDecoder(io.LimitReader(resp.Body, limit)).Decode(&response); err != nil {
-		return nil, nil, err
+
+	// Drain the entire body to make sure we have re-use connections
+	defer xhttp.DrainBody(resp.Body)
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, c.parseErrorResponse(resp)
+	}
+
+	response := &response{}
+	if err = json.NewDecoder(io.LimitReader(resp.Body, limit)).Decode(response); err != nil {
+		return nil, err
+	}
+	return response, nil
+}
+
+func (c *kesClient) postRetry(url string, body io.ReadSeeker, limit int64) (*response, error) {
+	for i := 0; ; i++ {
+		body.Seek(0, io.SeekStart) // seek to the beginning of the body.
+
+		response, err := c.post(url, body, limit)
+		if err == nil {
+			return response, nil
+		}
+
+		if !xnet.IsNetworkOrHostDown(err) && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
+			return nil, err
+		}
+
+		// retriable network errors.
+		remain := retryMax - i
+		if remain <= 0 {
+			return response, err
+		}
+
+		<-time.After(LinearJitterBackoff(retryWaitMin, retryWaitMax, i))
 	}
-	return response.Plaintext, response.Ciphertext, nil
 }
 
 // GenerateDataKey decrypts an encrypted data key with the key
@@ -256,11 +303,7 @@ func (c *kesClient) GenerateDataKey(name string, context []byte) ([]byte, []byte
 // The optional context must match the value you provided when
 // generating the data key.
 func (c *kesClient) DecryptDataKey(name string, ciphertext, context []byte) ([]byte, error) {
-	type Request struct {
-		Ciphertext []byte `json:"ciphertext"`
-		Context    []byte `json:"context"`
-	}
-	body, err := json.Marshal(Request{
+	body, err := json.Marshal(request{
 		Ciphertext: ciphertext,
 		Context:    context,
 	})
@@ -269,37 +312,26 @@ func (c *kesClient) DecryptDataKey(name string, ciphertext, context []byte) ([]b
 	}
 
 	url := fmt.Sprintf("%s/v1/key/decrypt/%s", c.addr, url.PathEscape(name))
-	resp, err := c.httpClient.Post(url, "application/json", bytes.NewReader(body))
+
+	const limit = 32 * 1024 // A data key will never be larger than 32 KB
+	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
 	if err != nil {
 		return nil, err
 	}
-	if resp.StatusCode != http.StatusOK {
-		return nil, c.parseErrorResponse(resp)
-	}
-	defer resp.Body.Close()
-
-	type Response struct {
-		Plaintext []byte `json:"plaintext"`
-	}
-	const limit = 32 * 1024 // A data key will never be larger than 32 KB
-	var response Response
-	if err = json.NewDecoder(io.LimitReader(resp.Body, limit)).Decode(&response); err != nil {
-		return nil, err
-	}
-	return response.Plaintext, nil
+	return resp.Plaintext, nil
 }
 
 func (c *kesClient) parseErrorResponse(resp *http.Response) error {
 	if resp.Body == nil {
-		return nil
+		return Errorf("%s: no body", http.StatusText(resp.StatusCode))
 	}
-	defer resp.Body.Close()
 
 	const limit = 32 * 1024 // A (valid) error response will not be greater than 32 KB
 	var errMsg strings.Builder
 	if _, err := io.Copy(&errMsg, io.LimitReader(resp.Body, limit)); err != nil {
-		return err
+		return Errorf("%s: %s", http.StatusText(resp.StatusCode), err)
 	}
+
 	return Errorf("%s: %s", http.StatusText(resp.StatusCode), errMsg.String())
 }
 

cmd/crypto/metadata.go

@@ -44,6 +44,8 @@ func RemoveSensitiveEntries(metadata map[string]string) { // The functions is te
 // header entries from the metadata map.
 func RemoveSSEHeaders(metadata map[string]string) {
 	delete(metadata, SSEHeader)
+	delete(metadata, SSEKmsID)
+	delete(metadata, SSEKmsContext)
 	delete(metadata, SSECKeyMD5)
 	delete(metadata, SSECAlgorithm)
 }

cmd/crypto/retry.go

@@ -0,0 +1,65 @@
+// MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"math/rand"
+	"time"
+)
+
+// default retry configuration
+const (
+	retryWaitMin = 500 * time.Millisecond // minimum retry limit.
+	retryWaitMax = 3 * time.Second        // 3 secs worth of max retry.
+	retryMax     = 2
+)
+
+// LinearJitterBackoff provides the time.Duration for a caller to
+// perform linear backoff based on the attempt number and with jitter to
+// prevent a thundering herd.
+//
+// min and max here are *not* absolute values. The number to be multiplied by
+// the attempt number will be chosen at random from between them, thus they are
+// bounding the jitter.
+//
+// For instance:
+// * To get strictly linear backoff of one second increasing each retry, set
+// both to one second (1s, 2s, 3s, 4s, ...)
+// * To get a small amount of jitter centered around one second increasing each
+// retry, set to around one second, such as a min of 800ms and max of 1200ms
+// (892ms, 2102ms, 2945ms, 4312ms, ...)
+// * To get extreme jitter, set to a very wide spread, such as a min of 100ms
+// and a max of 20s (15382ms, 292ms, 51321ms, 35234ms, ...)
+func LinearJitterBackoff(min, max time.Duration, attemptNum int) time.Duration {
+	// attemptNum always starts at zero but we want to start at 1 for multiplication
+	attemptNum++
+
+	if max <= min {
+		// Unclear what to do here, or they are the same, so return min *
+		// attemptNum
+		return min * time.Duration(attemptNum)
+	}
+
+	// Seed rand; doing this every time is fine
+	rand := rand.New(rand.NewSource(int64(time.Now().Nanosecond())))
+
+	// Pick a random number that lies somewhere between the min and max and
+	// multiply by the attemptNum. attemptNum starts at zero so we always
+	// increment here. We first get a random percentage, then apply that to the
+	// difference between min and max, and add to min.
+	jitter := rand.Float64() * float64(max-min)
+	jitterMin := int64(jitter) + int64(min)
+	return time.Duration(jitterMin * int64(attemptNum))
+}

cmd/daily-lifecycle-ops.go

@@ -27,7 +27,6 @@ import (
 
 const (
 	bgLifecycleInterval = 24 * time.Hour
-	bgLifecycleTick     = time.Hour
 )
 
 // initDailyLifecycle starts the routine that receives the daily
@@ -51,11 +50,6 @@ func startDailyLifecycle(ctx context.Context, objAPI ObjectLayer) {
 }
 
 func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
-	// No action is expected when WORM is enabled
-	if globalWORMEnabled {
-		return nil
-	}
-
 	buckets, err := objAPI.ListBuckets(ctx)
 	if err != nil {
 		return err
@@ -63,12 +57,15 @@ func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 
 	for _, bucket := range buckets {
 		// Check if the current bucket has a configured lifecycle policy, skip otherwise
-		l, ok := globalLifecycleSys.Get(bucket.Name)
-		if !ok {
+		l, err := globalLifecycleSys.Get(bucket.Name)
+		if err != nil {
+			if _, ok := err.(BucketLifecycleNotFound); !ok {
+				logger.LogIf(ctx, err)
+			}
 			continue
 		}
 
-		_, bucketHasLockConfig := globalBucketObjectLockConfig.Get(bucket.Name)
+		rcfg, _ := globalBucketObjectLockSys.Get(bucket.Name)
 
 		// Calculate the common prefix of all lifecycle rules
 		var prefixes []string
@@ -88,13 +85,13 @@ func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
 		for {
 			var objects []string
 			for obj := range objInfoCh {
-				if len(objects) == maxObjectList {
+				if len(objects) == maxDeleteList {
 					// Reached maximum delete requests, attempt a delete for now.
 					break
 				}
 				// Find the action that need to be executed
 				if l.ComputeAction(obj.Name, obj.UserTags, obj.ModTime) == lifecycle.DeleteAction {
-					if bucketHasLockConfig && enforceRetentionForLifecycle(ctx, obj) {
+					if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
 						continue
 					}
 					objects = append(objects, obj.Name)

cmd/data-update-tracker.go

@@ -0,0 +1,621 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/binary"
+	"errors"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/env"
+	"github.com/willf/bloom"
+)
+
+const (
+	// Estimate bloom filter size. With this many items
+	dataUpdateTrackerEstItems = 1000000
+	// ... we want this false positive rate:
+	dataUpdateTrackerFP        = 0.99
+	dataUpdateTrackerQueueSize = 10000
+
+	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
+	dataUpdateTrackerVersion      = 1
+	dataUpdateTrackerSaveInterval = 5 * time.Minute
+
+	// Reset bloom filters every n cycle
+	dataUpdateTrackerResetEvery = 1000
+)
+
+var (
+	objectUpdatedCh      chan<- string
+	intDataUpdateTracker *dataUpdateTracker
+)
+
+func init() {
+	intDataUpdateTracker = newDataUpdateTracker()
+	objectUpdatedCh = intDataUpdateTracker.input
+}
+
+type dataUpdateTracker struct {
+	mu         sync.Mutex
+	input      chan string
+	save       chan struct{}
+	debug      bool
+	saveExited chan struct{}
+	dirty      bool
+
+	Current dataUpdateFilter
+	History dataUpdateTrackerHistory
+	Saved   time.Time
+}
+
+// newDataUpdateTracker returns a dataUpdateTracker with default settings.
+func newDataUpdateTracker() *dataUpdateTracker {
+	d := &dataUpdateTracker{
+		Current: dataUpdateFilter{
+			idx: 1,
+		},
+		debug:      env.Get(envDataUsageCrawlDebug, config.EnableOff) == config.EnableOn,
+		input:      make(chan string, dataUpdateTrackerQueueSize),
+		save:       make(chan struct{}, 1),
+		saveExited: make(chan struct{}),
+	}
+	d.Current.bf = d.newBloomFilter()
+	d.dirty = true
+	return d
+}
+
+type dataUpdateTrackerHistory []dataUpdateFilter
+
+type dataUpdateFilter struct {
+	idx uint64
+	bf  bloomFilter
+}
+
+type bloomFilter struct {
+	*bloom.BloomFilter
+}
+
+// emptyBloomFilter returns an empty bloom filter.
+func emptyBloomFilter() bloomFilter {
+	return bloomFilter{BloomFilter: &bloom.BloomFilter{}}
+}
+
+// containsDir returns whether the bloom filter contains a directory.
+// Note that objects in XL mode are also considered directories.
+func (b bloomFilter) containsDir(in string) bool {
+	split := splitPathDeterministic(path.Clean(in))
+
+	if len(split) == 0 {
+		return false
+	}
+	var tmp [dataUsageHashLen]byte
+	hashPath(path.Join(split...)).bytes(tmp[:])
+	return b.Test(tmp[:])
+}
+
+// bytes returns the bloom filter serialized as a byte slice.
+func (b bloomFilter) bytes() []byte {
+	if b.BloomFilter == nil {
+		return nil
+	}
+	var buf bytes.Buffer
+	_, err := b.WriteTo(&buf)
+	if err != nil {
+		logger.LogIf(GlobalContext, err)
+		return nil
+	}
+	return buf.Bytes()
+}
+
+// sort the dataUpdateTrackerHistory, newest first.
+// Returns whether the history is complete.
+func (d dataUpdateTrackerHistory) sort() bool {
+	if len(d) == 0 {
+		return true
+	}
+	sort.Slice(d, func(i, j int) bool {
+		return d[i].idx > d[j].idx
+	})
+	return d[0].idx-d[len(d)-1].idx == uint64(len(d))
+}
+
+// removeOlderThan will remove entries older than index 'n'.
+func (d *dataUpdateTrackerHistory) removeOlderThan(n uint64) {
+	d.sort()
+	dd := *d
+	end := len(dd)
+	for i := end - 1; i >= 0; i-- {
+		if dd[i].idx < n {
+			end = i
+		}
+	}
+	dd = dd[:end]
+	*d = dd
+}
+
+// newBloomFilter returns a new bloom filter with default settings.
+func (d *dataUpdateTracker) newBloomFilter() bloomFilter {
+	return bloomFilter{bloom.NewWithEstimates(dataUpdateTrackerEstItems, dataUpdateTrackerFP)}
+}
+
+// current returns the current index.
+func (d *dataUpdateTracker) current() uint64 {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	return d.Current.idx
+}
+
+// start will load the current data from the drives start collecting information and
+// start a saver goroutine.
+// All of these will exit when the context is canceled.
+func (d *dataUpdateTracker) start(ctx context.Context, drives ...string) {
+	if len(drives) <= 0 {
+		logger.LogIf(ctx, errors.New("dataUpdateTracker.start: No drives specified"))
+		return
+	}
+	d.load(ctx, drives...)
+	go d.startCollector(ctx)
+	go d.startSaver(ctx, dataUpdateTrackerSaveInterval, drives)
+}
+
+// load will attempt to load data tracking information from the supplied drives.
+// The data will only be loaded if d.Saved is older than the one found on disk.
+// The newest working cache will be kept in d.
+// If no valid data usage tracker can be found d will remain unchanged.
+// If object is shared the caller should lock it.
+func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
+	if len(drives) <= 0 {
+		logger.LogIf(ctx, errors.New("dataUpdateTracker.load: No drives specified"))
+		return
+	}
+	for _, drive := range drives {
+
+		cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
+		f, err := os.Open(cacheFormatPath)
+		if err != nil {
+			if os.IsNotExist(err) {
+				continue
+			}
+			logger.LogIf(ctx, err)
+			continue
+		}
+		err = d.deserialize(f, d.Saved)
+		if err != nil && err != io.EOF {
+			logger.LogIf(ctx, err)
+		}
+		f.Close()
+	}
+}
+
+// startSaver will start a saver that will write d to all supplied drives at specific intervals.
+// The saver will save and exit when supplied context is closed.
+func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Duration, drives []string) {
+	t := time.NewTicker(interval)
+	defer t.Stop()
+	var buf bytes.Buffer
+	d.mu.Lock()
+	saveNow := d.save
+	exited := make(chan struct{})
+	d.saveExited = exited
+	d.mu.Unlock()
+	defer close(exited)
+	for {
+		var exit bool
+		select {
+		case <-ctx.Done():
+			exit = true
+		case <-t.C:
+		case <-saveNow:
+		}
+		buf.Reset()
+		d.mu.Lock()
+		if !d.dirty {
+			d.mu.Unlock()
+			return
+		}
+		d.Saved = UTCNow()
+		err := d.serialize(&buf)
+		if d.debug {
+			logger.Info(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v", buf.Len(), d.Current.idx)
+		}
+		d.dirty = false
+		d.mu.Unlock()
+		if err != nil {
+			logger.LogIf(ctx, err, "Error serializing usage tracker data")
+			if exit {
+				return
+			}
+			continue
+		}
+		if buf.Len() == 0 {
+			logger.LogIf(ctx, errors.New("zero sized output, skipping save"))
+			continue
+		}
+		for _, drive := range drives {
+			cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
+			err := ioutil.WriteFile(cacheFormatPath, buf.Bytes(), os.ModePerm)
+			if err != nil {
+				if os.IsNotExist(err) {
+					continue
+				}
+				logger.LogIf(ctx, err)
+				continue
+			}
+		}
+		if exit {
+			return
+		}
+	}
+}
+
+// serialize all data in d to dst.
+// Caller should hold lock if d is expected to be shared.
+// If an error is returned, there will likely be partial data written to dst.
+func (d *dataUpdateTracker) serialize(dst io.Writer) (err error) {
+	ctx := GlobalContext
+	var tmp [8]byte
+	o := bufio.NewWriter(dst)
+	defer func() {
+		if err == nil {
+			err = o.Flush()
+		}
+	}()
+
+	// Version
+	if err := o.WriteByte(dataUpdateTrackerVersion); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+	// Timestamp.
+	binary.LittleEndian.PutUint64(tmp[:], uint64(d.Saved.Unix()))
+	if _, err := o.Write(tmp[:]); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+
+	// Current
+	binary.LittleEndian.PutUint64(tmp[:], d.Current.idx)
+	if _, err := o.Write(tmp[:]); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+
+	if _, err := d.Current.bf.WriteTo(o); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+
+	// History
+	binary.LittleEndian.PutUint64(tmp[:], uint64(len(d.History)))
+	if _, err := o.Write(tmp[:]); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+
+	for _, bf := range d.History {
+		// Current
+		binary.LittleEndian.PutUint64(tmp[:], bf.idx)
+		if _, err := o.Write(tmp[:]); err != nil {
+			if d.debug {
+				logger.LogIf(ctx, err)
+			}
+			return err
+		}
+
+		if _, err := bf.bf.WriteTo(o); err != nil {
+			if d.debug {
+				logger.LogIf(ctx, err)
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+// deserialize will deserialize the supplied input if the input is newer than the supplied time.
+func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) error {
+	ctx := GlobalContext
+	var dst dataUpdateTracker
+	var tmp [8]byte
+
+	// Version
+	if _, err := io.ReadFull(src, tmp[:1]); err != nil {
+		if d.debug {
+			if err != io.EOF {
+				logger.LogIf(ctx, err)
+			}
+		}
+		return err
+	}
+	switch tmp[0] {
+	case dataUpdateTrackerVersion:
+	default:
+		return errors.New("dataUpdateTracker: Unknown data version")
+	}
+	// Timestamp.
+	if _, err := io.ReadFull(src, tmp[:8]); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+	t := time.Unix(int64(binary.LittleEndian.Uint64(tmp[:])), 0)
+	if !t.After(newerThan) {
+		return nil
+	}
+
+	// Current
+	if _, err := io.ReadFull(src, tmp[:8]); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+	dst.Current.idx = binary.LittleEndian.Uint64(tmp[:])
+	dst.Current.bf = emptyBloomFilter()
+	if _, err := dst.Current.bf.ReadFrom(src); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+
+	// History
+	if _, err := io.ReadFull(src, tmp[:8]); err != nil {
+		if d.debug {
+			logger.LogIf(ctx, err)
+		}
+		return err
+	}
+	n := binary.LittleEndian.Uint64(tmp[:])
+	dst.History = make(dataUpdateTrackerHistory, int(n))
+	for i, e := range dst.History {
+		if _, err := io.ReadFull(src, tmp[:8]); err != nil {
+			if d.debug {
+				logger.LogIf(ctx, err)
+			}
+			return err
+		}
+		e.idx = binary.LittleEndian.Uint64(tmp[:])
+		e.bf = emptyBloomFilter()
+		if _, err := e.bf.ReadFrom(src); err != nil {
+			if d.debug {
+				logger.LogIf(ctx, err)
+			}
+			return err
+		}
+		dst.History[i] = e
+	}
+	// Ignore what remains on the stream.
+	// Update d:
+	d.Current = dst.Current
+	d.History = dst.History
+	d.Saved = dst.Saved
+	return nil
+}
+
+// start a collector that picks up entries from objectUpdatedCh
+// and adds them  to the current bloom filter.
+func (d *dataUpdateTracker) startCollector(ctx context.Context) {
+	var tmp [dataUsageHashLen]byte
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case in := <-d.input:
+			bucket, _ := path2BucketObjectWithBasePath("", in)
+			if bucket == "" {
+				if d.debug && len(in) > 0 {
+					logger.Info(color.Green("data-usage:")+" no bucket (%s)", in)
+				}
+				continue
+			}
+
+			if isReservedOrInvalidBucket(bucket, false) {
+				if false && d.debug {
+					logger.Info(color.Green("data-usage:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
+				}
+				continue
+			}
+			split := splitPathDeterministic(in)
+
+			// Add all paths until level 3.
+			d.mu.Lock()
+			for i := range split {
+				if d.debug && false {
+					logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
+				}
+				hashPath(path.Join(split[:i+1]...)).bytes(tmp[:])
+				d.Current.bf.Add(tmp[:])
+			}
+			d.dirty = d.dirty || len(split) > 0
+			d.mu.Unlock()
+		}
+	}
+}
+
+// find entry with specified index.
+// Returns nil if not found.
+func (d dataUpdateTrackerHistory) find(idx uint64) *dataUpdateFilter {
+	for _, f := range d {
+		if f.idx == idx {
+			return &f
+		}
+	}
+	return nil
+}
+
+// filterFrom will return a combined bloom filter.
+func (d *dataUpdateTracker) filterFrom(ctx context.Context, oldest, newest uint64) *bloomFilterResponse {
+	bf := d.newBloomFilter()
+	bfr := bloomFilterResponse{
+		OldestIdx:  oldest,
+		CurrentIdx: d.Current.idx,
+		Complete:   true,
+	}
+	// Loop through each index requested.
+	for idx := oldest; idx <= newest; idx++ {
+		v := d.History.find(idx)
+		if v == nil {
+			if d.Current.idx == idx {
+				// Merge current.
+				err := bf.Merge(d.Current.bf.BloomFilter)
+				logger.LogIf(ctx, err)
+				if err != nil {
+					bfr.Complete = false
+				}
+				continue
+			}
+			bfr.Complete = false
+			bfr.OldestIdx = idx + 1
+			continue
+		}
+
+		err := bf.Merge(v.bf.BloomFilter)
+		if err != nil {
+			bfr.Complete = false
+			logger.LogIf(ctx, err)
+			continue
+		}
+		bfr.NewestIdx = idx
+	}
+	var dst bytes.Buffer
+	_, err := bf.WriteTo(&dst)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return nil
+	}
+	bfr.Filter = dst.Bytes()
+
+	return &bfr
+}
+
+// cycleFilter will cycle the bloom filter to start recording to index y if not already.
+// The response will contain a bloom filter starting at index x up to, but not including index y.
+// If y is 0, the response will not update y, but return the currently recorded information
+// from the up until and including current y.
+func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uint64) (*bloomFilterResponse, error) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if current == 0 {
+		if len(d.History) == 0 {
+			return d.filterFrom(ctx, d.Current.idx, d.Current.idx), nil
+		}
+		d.History.sort()
+		return d.filterFrom(ctx, d.History[len(d.History)-1].idx, d.Current.idx), nil
+	}
+
+	// Move current to history if new one requested
+	if d.Current.idx != current {
+		d.dirty = true
+		if d.debug {
+			logger.Info(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v", d.Current.idx, current)
+		}
+
+		d.History = append(d.History, d.Current)
+		d.Current.idx = current
+		d.Current.bf = d.newBloomFilter()
+		select {
+		case d.save <- struct{}{}:
+		default:
+		}
+	}
+	d.History.removeOlderThan(oldest)
+	return d.filterFrom(ctx, oldest, current), nil
+}
+
+// splitPathDeterministic will split the provided relative path
+// deterministically and return up to the first 3 elements of the path.
+// Slash and dot prefixes are removed.
+// Trailing slashes are removed.
+// Returns 0 length if no parts are found after trimming.
+func splitPathDeterministic(in string) []string {
+	split := strings.Split(in, SlashSeparator)
+
+	// Trim empty start/end
+	for len(split) > 0 {
+		if len(split[0]) > 0 && split[0] != "." {
+			break
+		}
+		split = split[1:]
+	}
+	for len(split) > 0 {
+		if len(split[len(split)-1]) > 0 {
+			break
+		}
+		split = split[:len(split)-1]
+	}
+
+	// Return up to 3 parts.
+	if len(split) > 3 {
+		split = split[:3]
+	}
+	return split
+}
+
+// bloomFilterRequest request bloom filters.
+// Current index will be updated to current and entries back to Oldest is returned.
+type bloomFilterRequest struct {
+	Oldest  uint64
+	Current uint64
+}
+
+type bloomFilterResponse struct {
+	// Current index being written to.
+	CurrentIdx uint64
+	// Oldest index in the returned bloom filter.
+	OldestIdx uint64
+	// Newest Index in the returned bloom filter.
+	NewestIdx uint64
+	// Are all indexes between oldest and newest filled?
+	Complete bool
+	// Binary data of the bloom filter.
+	Filter []byte
+}
+
+// ObjectPathUpdated indicates a path has been updated.
+// The function will never block.
+func ObjectPathUpdated(s string) {
+	select {
+	case objectUpdatedCh <- s:
+	default:
+	}
+}

cmd/data-update-tracker_test.go

@@ -0,0 +1,262 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path"
+	"path/filepath"
+	"sync"
+	"testing"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/logger/message/log"
+)
+
+type testLoggerI interface {
+	Helper()
+	Log(args ...interface{})
+}
+
+type testingLogger struct {
+	mu sync.Mutex
+	t  testLoggerI
+}
+
+func (t *testingLogger) Send(entry interface{}, errKind string) error {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	if t.t == nil {
+		return nil
+	}
+	e, ok := entry.(log.Entry)
+	if !ok {
+		return fmt.Errorf("unexpected log entry structure %#v", entry)
+	}
+
+	t.t.Helper()
+	t.t.Log(e.Level, ":", errKind, e.Message)
+	return nil
+}
+
+func addTestingLogging(t testLoggerI) func() {
+	tl := &testingLogger{t: t}
+	logger.AddTarget(tl)
+	return func() {
+		tl.mu.Lock()
+		defer tl.mu.Unlock()
+		tl.t = nil
+	}
+}
+
+func TestDataUpdateTracker(t *testing.T) {
+	dut := newDataUpdateTracker()
+	// Change some defaults.
+	dut.debug = testing.Verbose()
+	dut.input = make(chan string)
+	dut.save = make(chan struct{})
+
+	defer addTestingLogging(t)()
+
+	dut.Current.bf = dut.newBloomFilter()
+
+	tmpDir, err := ioutil.TempDir("", "TestDataUpdateTracker")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(tmpDir)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	dut.start(ctx, tmpDir)
+
+	var tests = []struct {
+		in    string
+		check []string // if not empty, check against these instead.
+		exist bool
+	}{
+		{
+			in:    "bucket/directory/file.txt",
+			check: []string{"bucket", "bucket/", "/bucket", "bucket/directory", "bucket/directory/", "bucket/directory/file.txt", "/bucket/directory/file.txt"},
+			exist: true,
+		},
+		{
+			// System bucket
+			in:    ".minio.sys/ignoreme/pls",
+			exist: false,
+		},
+		{
+			// Not a valid bucket
+			in:    "./bucket/okfile.txt",
+			check: []string{"./bucket/okfile.txt", "/bucket/okfile.txt", "bucket/okfile.txt"},
+			exist: false,
+		},
+		{
+			// Not a valid bucket
+			in:    "æ/okfile.txt",
+			check: []string{"æ/okfile.txt", "æ/okfile.txt", "æ"},
+			exist: false,
+		},
+		{
+			in:    "/bucket2/okfile2.txt",
+			check: []string{"./bucket2/okfile2.txt", "/bucket2/okfile2.txt", "bucket2/okfile2.txt", "bucket2"},
+			exist: true,
+		},
+		{
+			in:    "/bucket3/prefix/okfile2.txt",
+			check: []string{"./bucket3/prefix/okfile2.txt", "/bucket3/prefix/okfile2.txt", "bucket3/prefix/okfile2.txt", "bucket3/prefix", "bucket3"},
+			exist: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.in, func(t *testing.T) {
+			dut.input <- tt.in
+			dut.input <- "" // Sending empty string ensures the previous is added to filter.
+			dut.mu.Lock()
+			defer dut.mu.Unlock()
+			if len(tt.check) == 0 {
+				got := dut.Current.bf.containsDir(tt.in)
+				if got != tt.exist {
+					// For unlimited tests this could lead to false positives,
+					// but it should be deterministic.
+					t.Errorf("entry %q, got: %v, want %v", tt.in, got, tt.exist)
+				}
+				return
+			}
+			for _, check := range tt.check {
+				got := dut.Current.bf.containsDir(check)
+				if got != tt.exist {
+					// For unlimited tests this could lead to false positives,
+					// but it should be deterministic.
+					t.Errorf("entry %q, check: %q, got: %v, want %v", tt.in, check, got, tt.exist)
+				}
+				continue
+			}
+		})
+	}
+	// Cycle to history
+	_, err = dut.cycleFilter(ctx, 1, 2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dut.input <- "cycle2/file.txt"
+	dut.input <- "" // Sending empty string ensures the previous is added to filter.
+
+	tests = append(tests, struct {
+		in    string
+		check []string
+		exist bool
+	}{in: "cycle2/file.txt", exist: true})
+
+	// Shut down
+	cancel()
+	<-dut.saveExited
+
+	if dut.current() != 2 {
+		t.Fatal("wrong current idx after save. want 2, got:", dut.current())
+	}
+
+	ctx, cancel = context.WithCancel(context.Background())
+	defer cancel()
+
+	// Reload...
+	dut = newDataUpdateTracker()
+	dut.start(ctx, tmpDir)
+
+	if dut.current() != 2 {
+		t.Fatal("current idx after load not preserved. want 2, got:", dut.current())
+	}
+	bfr2, err := dut.cycleFilter(ctx, 1, 3)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !bfr2.Complete {
+		t.Fatal("Wanted complete, didn't get it")
+	}
+	if bfr2.CurrentIdx != 3 {
+		t.Fatal("wanted index 3, got", bfr2.CurrentIdx)
+	}
+	if bfr2.OldestIdx != 1 {
+		t.Fatal("wanted oldest index 3, got", bfr2.OldestIdx)
+	}
+
+	// Rerun test with returned bfr2
+	bf := dut.newBloomFilter()
+	_, err = bf.ReadFrom(bytes.NewBuffer(bfr2.Filter))
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, tt := range tests {
+		t.Run(tt.in+"-reloaded", func(t *testing.T) {
+			if len(tt.check) == 0 {
+				got := bf.containsDir(tt.in)
+				if got != tt.exist {
+					// For unlimited tests this could lead to false positives,
+					// but it should be deterministic.
+					t.Errorf("entry %q, got: %v, want %v", tt.in, got, tt.exist)
+				}
+				return
+			}
+			for _, check := range tt.check {
+				got := bf.containsDir(check)
+				if got != tt.exist {
+					// For unlimited tests this could lead to false positives,
+					// but it should be deterministic.
+					t.Errorf("entry %q, check: %q, got: %v, want %v", tt.in, check, got, tt.exist)
+				}
+				continue
+			}
+		})
+	}
+}
+
+func BenchmarkDataUpdateTracker(b *testing.B) {
+	dut := newDataUpdateTracker()
+	// Change some defaults.
+	dut.debug = false
+	dut.input = make(chan string)
+	dut.save = make(chan struct{})
+
+	defer addTestingLogging(b)()
+
+	dut.Current.bf = dut.newBloomFilter()
+	// We do this unbuffered. This will very significantly reduce throughput, so this is a worst case.
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go dut.startCollector(ctx)
+	input := make([]string, 1000)
+	rng := rand.New(rand.NewSource(0xabad1dea))
+	tmp := []string{"bucket", "aprefix", "nextprefixlevel", "maybeobjname", "evendeeper", "ok-one-morelevel", "final.object"}
+	for i := range input {
+		tmp := tmp[:1+rng.Intn(cap(tmp)-1)]
+		input[i] = path.Join(tmp...)
+	}
+	b.SetBytes(1)
+	b.ResetTimer()
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		dut.input <- input[rng.Intn(len(input))]
+	}
+}

cmd/data-usage-cache.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"io"
 	"path"
@@ -62,9 +63,10 @@ type dataUsageEntryInfo struct {
 
 type dataUsageCacheInfo struct {
 	// Name of the bucket. Also root element.
-	Name       string
-	LastUpdate time.Time
-	NextCycle  uint8
+	Name        string
+	LastUpdate  time.Time
+	NextCycle   uint32
+	BloomFilter []byte `msg:"BloomFilter,omitempty"`
 }
 
 // merge other data usage entry into this, excluding children.
@@ -77,8 +79,8 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 }
 
 // mod returns true if the hash mod cycles == cycle.
-func (h dataUsageHash) mod(cycle uint8, cycles uint8) bool {
-	return uint8(h)%cycles == cycle%cycles
+func (h dataUsageHash) mod(cycle uint32, cycles uint32) bool {
+	return uint32(h)%cycles == cycle%cycles
 }
 
 // addChildString will add a child based on its name.
@@ -110,19 +112,20 @@ func (d *dataUsageCache) find(path string) *dataUsageEntry {
 }
 
 // dui converts the flattened version of the path to DataUsageInfo.
+// As a side effect d will be flattened, use a clone if this is not ok.
 func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
 	e := d.find(path)
 	if e == nil {
-		return DataUsageInfo{LastUpdate: UTCNow()}
+		// No entry found, return empty.
+		return DataUsageInfo{}
 	}
 	flat := d.flatten(*e)
 	return DataUsageInfo{
-		LastUpdate:            d.Info.LastUpdate,
-		ObjectsCount:          flat.Objects,
-		ObjectsTotalSize:      uint64(flat.Size),
-		ObjectsSizesHistogram: flat.ObjSizes.asMap(),
-		BucketsCount:          uint64(len(e.Children)),
-		BucketsSizes:          d.pathSizes(buckets),
+		LastUpdate:        d.Info.LastUpdate,
+		ObjectsTotalCount: flat.Objects,
+		ObjectsTotalSize:  uint64(flat.Size),
+		BucketsCount:      uint64(len(e.Children)),
+		BucketsUsage:      d.bucketsUsageInfo(buckets),
 	}
 }
 
@@ -158,6 +161,32 @@ func (d *dataUsageCache) replaceHashed(hash dataUsageHash, parent *dataUsageHash
 	}
 }
 
+// copyWithChildren will copy entry with hash from src if it exists along with any children.
+// If a parent is specified it will be added to that if not already there.
+// If the parent does not exist, it will be added.
+func (d *dataUsageCache) copyWithChildren(src *dataUsageCache, hash dataUsageHash, parent *dataUsageHash) {
+	if d.Cache == nil {
+		d.Cache = make(map[dataUsageHash]dataUsageEntry, 100)
+	}
+	e, ok := src.Cache[hash]
+	if !ok {
+		return
+	}
+	d.Cache[hash] = e
+	for ch := range e.Children {
+		if ch == hash {
+			logger.LogIf(GlobalContext, errors.New("dataUsageCache.copyWithChildren: Circular reference"))
+			return
+		}
+		d.copyWithChildren(src, ch, &hash)
+	}
+	if parent != nil {
+		p := d.Cache[*parent]
+		p.addChild(hash)
+		d.Cache[*parent] = p
+	}
+}
+
 // StringAll returns a detailed string representation of all entries in the cache.
 func (d *dataUsageCache) StringAll() string {
 	s := fmt.Sprintf("info:%+v\n", d.Info)
@@ -167,6 +196,12 @@ func (d *dataUsageCache) StringAll() string {
 	return strings.TrimSpace(s)
 }
 
+// insert the hash into dst.
+// dst must be at least dataUsageHashLen bytes long.
+func (h dataUsageHash) bytes(dst []byte) {
+	binary.LittleEndian.PutUint64(dst, uint64(h))
+}
+
 // String returns a human readable representation of the string.
 func (h dataUsageHash) String() string {
 	return fmt.Sprintf("%x", uint64(h))
@@ -197,25 +232,30 @@ func (h *sizeHistogram) add(size int64) {
 	}
 }
 
-// asMap returns the map as a map[string]uint64.
-func (h *sizeHistogram) asMap() map[string]uint64 {
-	res := make(map[string]uint64, 7)
+// toMap returns the map to a map[string]uint64.
+func (h *sizeHistogram) toMap() map[string]uint64 {
+	res := make(map[string]uint64, dataUsageBucketLen)
 	for i, count := range h {
 		res[ObjectsHistogramIntervals[i].name] = count
 	}
 	return res
 }
 
-// pathSizes returns the path sizes as a map.
-func (d *dataUsageCache) pathSizes(buckets []BucketInfo) map[string]uint64 {
-	var dst = make(map[string]uint64, len(buckets))
+// bucketsUsageInfo returns the buckets usage info as a map, with
+// key as bucket name
+func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]BucketUsageInfo {
+	var dst = make(map[string]BucketUsageInfo, len(buckets))
 	for _, bucket := range buckets {
 		e := d.find(bucket.Name)
 		if e == nil {
 			continue
 		}
 		flat := d.flatten(*e)
-		dst[bucket.Name] = uint64(flat.Size)
+		dst[bucket.Name] = BucketUsageInfo{
+			Size:                 uint64(flat.Size),
+			ObjectsCount:         uint64(flat.Objects),
+			ObjectSizesHistogram: flat.ObjSizes.toMap(),
+		}
 	}
 	return dst
 }
@@ -297,7 +337,7 @@ func (d *dataUsageCache) load(ctx context.Context, store ObjectLayer, name strin
 	var buf bytes.Buffer
 	err := store.GetObject(ctx, dataUsageBucket, name, 0, -1, &buf, "", ObjectOptions{})
 	if err != nil {
-		if !isErrObjectNotFound(err) {
+		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) {
 			return toObjectErr(err, dataUsageBucket, name)
 		}
 		*d = dataUsageCache{}
@@ -325,6 +365,9 @@ func (d *dataUsageCache) save(ctx context.Context, store ObjectLayer, name strin
 		name,
 		NewPutObjReader(r, nil, nil),
 		ObjectOptions{})
+	if isErrBucketNotFound(err) {
+		return nil
+	}
 	return err
 }
 

cmd/data-usage-cache_gen.go

@@ -37,11 +37,17 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "NextCycle":
-			z.NextCycle, err = dc.ReadUint8()
+			z.NextCycle, err = dc.ReadUint32()
 			if err != nil {
 				err = msgp.WrapError(err, "NextCycle")
 				return
 			}
+		case "BloomFilter":
+			z.BloomFilter, err = dc.ReadBytes(z.BloomFilter)
+			if err != nil {
+				err = msgp.WrapError(err, "BloomFilter")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -54,10 +60,24 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 }
 
 // EncodeMsg implements msgp.Encodable
-func (z dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 3
+func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
+	// omitempty: check for empty values
+	zb0001Len := uint32(4)
+	var zb0001Mask uint8 /* 4 bits */
+	if z.BloomFilter == nil {
+		zb0001Len--
+		zb0001Mask |= 0x8
+	}
+	// variable map header, size zb0001Len
+	err = en.Append(0x80 | uint8(zb0001Len))
+	if err != nil {
+		return
+	}
+	if zb0001Len == 0 {
+		return
+	}
 	// write "Name"
-	err = en.Append(0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -81,27 +101,55 @@ func (z dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteUint8(z.NextCycle)
+	err = en.WriteUint32(z.NextCycle)
 	if err != nil {
 		err = msgp.WrapError(err, "NextCycle")
 		return
 	}
+	if (zb0001Mask & 0x8) == 0 { // if not empty
+		// write "BloomFilter"
+		err = en.Append(0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
+		if err != nil {
+			return
+		}
+		err = en.WriteBytes(z.BloomFilter)
+		if err != nil {
+			err = msgp.WrapError(err, "BloomFilter")
+			return
+		}
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
-func (z dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
+func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 3
+	// omitempty: check for empty values
+	zb0001Len := uint32(4)
+	var zb0001Mask uint8 /* 4 bits */
+	if z.BloomFilter == nil {
+		zb0001Len--
+		zb0001Mask |= 0x8
+	}
+	// variable map header, size zb0001Len
+	o = append(o, 0x80|uint8(zb0001Len))
+	if zb0001Len == 0 {
+		return
+	}
 	// string "Name"
-	o = append(o, 0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "LastUpdate"
 	o = append(o, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65)
 	o = msgp.AppendTime(o, z.LastUpdate)
 	// string "NextCycle"
 	o = append(o, 0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
-	o = msgp.AppendUint8(o, z.NextCycle)
+	o = msgp.AppendUint32(o, z.NextCycle)
+	if (zb0001Mask & 0x8) == 0 { // if not empty
+		// string "BloomFilter"
+		o = append(o, 0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
+		o = msgp.AppendBytes(o, z.BloomFilter)
+	}
 	return
 }
 
@@ -136,11 +184,17 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				return
 			}
 		case "NextCycle":
-			z.NextCycle, bts, err = msgp.ReadUint8Bytes(bts)
+			z.NextCycle, bts, err = msgp.ReadUint32Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "NextCycle")
 				return
 			}
+		case "BloomFilter":
+			z.BloomFilter, bts, err = msgp.ReadBytesBytes(bts, z.BloomFilter)
+			if err != nil {
+				err = msgp.WrapError(err, "BloomFilter")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -154,8 +208,8 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 }
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z dataUsageCacheInfo) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.TimeSize + 10 + msgp.Uint8Size
+func (z *dataUsageCacheInfo) Msgsize() (s int) {
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.TimeSize + 10 + msgp.Uint32Size + 12 + msgp.BytesPrefixSize + len(z.BloomFilter)
 	return
 }