count metrics properly for any failures during drive heal (#20193)

or via `mc admin heal --set 1 --pool 1`

.github/workflows/vulncheck.yml

@@ -21,8 +21,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.4
-          check-latest: true
+          go-version: 1.22.5
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest
         shell: bash

.typos.toml

@@ -2,6 +2,9 @@
 extend-exclude = [
     ".git/",
     "docs/",
+    "CREDITS",
+    "go.mod",
+    "go.sum",
 ]
 ignore-hidden = false
 
@@ -16,6 +19,7 @@ extend-ignore-re = [
     "MIIDBTCCAe2gAwIBAgIQWHw7h.*",
     'http\.Header\{"X-Amz-Server-Side-Encryptio":',
     "ZoEoZdLlzVbOlT9rbhD7ZN7TLyiYXSAlB79uGEge",
+    "ERRO:",
 ]
 
 [default.extend-words]

CREDITS

@@ -6334,6 +6334,203 @@ SOFTWARE.
 
 ================================================================
 
+github.com/go-ini/ini
+https://github.com/go-ini/ini
+----------------------------------------------------------------
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and
+distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright
+owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities
+that control, are controlled by, or are under common control with that entity.
+For the purposes of this definition, "control" means (i) the power, direct or
+indirect, to cause the direction or management of such entity, whether by
+contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising
+permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including
+but not limited to software source code, documentation source, and configuration
+files.
+
+"Object" form shall mean any form resulting from mechanical transformation or
+translation of a Source form, including but not limited to compiled object code,
+generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form, made
+available under the License, as indicated by a copyright notice that is included
+in or attached to the work (an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that
+is based on (or derived from) the Work and for which the editorial revisions,
+annotations, elaborations, or other modifications represent, as a whole, an
+original work of authorship. For the purposes of this License, Derivative Works
+shall not include works that remain separable from, or merely link (or bind by
+name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original version
+of the Work and any modifications or additions to that Work or Derivative Works
+thereof, that is intentionally submitted to Licensor for inclusion in the Work
+by the copyright owner or by an individual or Legal Entity authorized to submit
+on behalf of the copyright owner. For the purposes of this definition,
+"submitted" means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems, and
+issue tracking systems that are managed by, or on behalf of, the Licensor for
+the purpose of discussing and improving the Work, but excluding communication
+that is conspicuously marked or otherwise designated in writing by the copyright
+owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
+of whom a Contribution has been received by Licensor and subsequently
+incorporated within the Work.
+
+2. Grant of Copyright License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the Work and such
+Derivative Works in Source or Object form.
+
+3. Grant of Patent License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable (except as stated in this section) patent license to make, have
+made, use, offer to sell, sell, import, and otherwise transfer the Work, where
+such license applies only to those patent claims licensable by such Contributor
+that are necessarily infringed by their Contribution(s) alone or by combination
+of their Contribution(s) with the Work to which such Contribution(s) was
+submitted. If You institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work or a
+Contribution incorporated within the Work constitutes direct or contributory
+patent infringement, then any patent licenses granted to You under this License
+for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution.
+
+You may reproduce and distribute copies of the Work or Derivative Works thereof
+in any medium, with or without modifications, and in Source or Object form,
+provided that You meet the following conditions:
+
+You must give any other recipients of the Work or Derivative Works a copy of
+this License; and
+You must cause any modified files to carry prominent notices stating that You
+changed the files; and
+You must retain, in the Source form of any Derivative Works that You distribute,
+all copyright, patent, trademark, and attribution notices from the Source form
+of the Work, excluding those notices that do not pertain to any part of the
+Derivative Works; and
+If the Work includes a "NOTICE" text file as part of its distribution, then any
+Derivative Works that You distribute must include a readable copy of the
+attribution notices contained within such NOTICE file, excluding those notices
+that do not pertain to any part of the Derivative Works, in at least one of the
+following places: within a NOTICE text file distributed as part of the
+Derivative Works; within the Source form or documentation, if provided along
+with the Derivative Works; or, within a display generated by the Derivative
+Works, if and wherever such third-party notices normally appear. The contents of
+the NOTICE file are for informational purposes only and do not modify the
+License. You may add Your own attribution notices within Derivative Works that
+You distribute, alongside or as an addendum to the NOTICE text from the Work,
+provided that such additional attribution notices cannot be construed as
+modifying the License.
+You may add Your own copyright statement to Your modifications and may provide
+additional or different license terms and conditions for use, reproduction, or
+distribution of Your modifications, or for any such Derivative Works as a whole,
+provided Your use, reproduction, and distribution of the Work otherwise complies
+with the conditions stated in this License.
+
+5. Submission of Contributions.
+
+Unless You explicitly state otherwise, any Contribution intentionally submitted
+for inclusion in the Work by You to the Licensor shall be under the terms and
+conditions of this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify the terms of
+any separate license agreement you may have executed with Licensor regarding
+such Contributions.
+
+6. Trademarks.
+
+This License does not grant permission to use the trade names, trademarks,
+service marks, or product names of the Licensor, except as required for
+reasonable and customary use in describing the origin of the Work and
+reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty.
+
+Unless required by applicable law or agreed to in writing, Licensor provides the
+Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
+including, without limitation, any warranties or conditions of TITLE,
+NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
+solely responsible for determining the appropriateness of using or
+redistributing the Work and assume any risks associated with Your exercise of
+permissions under this License.
+
+8. Limitation of Liability.
+
+In no event and under no legal theory, whether in tort (including negligence),
+contract, or otherwise, unless required by applicable law (such as deliberate
+and grossly negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special, incidental,
+or consequential damages of any character arising as a result of this License or
+out of the use or inability to use the Work (including but not limited to
+damages for loss of goodwill, work stoppage, computer failure or malfunction, or
+any and all other commercial damages or losses), even if such Contributor has
+been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability.
+
+While redistributing the Work or Derivative Works thereof, You may choose to
+offer, and charge a fee for, acceptance of support, warranty, indemnity, or
+other liability obligations and/or rights consistent with this License. However,
+in accepting such obligations, You may act only on Your own behalf and on Your
+sole responsibility, not on behalf of any other Contributor, and only if You
+agree to indemnify, defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason of your
+accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work
+
+To apply the Apache License to your work, attach the following boilerplate
+notice, with the fields enclosed by brackets "[]" replaced with your own
+identifying information. (Don't include the brackets!) The text should be
+enclosed in the appropriate comment syntax for the file format. We also
+recommend that a file or class name and description of purpose be included on
+the same "printed page" as the copyright notice for easier identification within
+third-party archives.
+
+   Copyright 2014 Unknwon
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+================================================================
+
 github.com/go-jose/go-jose/v4
 https://github.com/go-jose/go-jose/v4
 ----------------------------------------------------------------
@@ -11223,33 +11420,29 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 github.com/gorilla/websocket
 https://github.com/gorilla/websocket
 ----------------------------------------------------------------
-Copyright (c) 2023 The Gorilla Authors. All rights reserved.
+Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
+modification, are permitted provided that the following conditions are met:
 
-	 * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-	 * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-	 * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
+  Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
 
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 ================================================================
 
 github.com/hashicorp/errwrap
@@ -13082,376 +13275,6 @@ Mozilla Public License, version 2.0
    be used to construe this License against a Contributor.
 
 
-10. Versions of the License
-
-10.1. New Versions
-
-      Mozilla Foundation is the license steward. Except as provided in Section
-      10.3, no one other than the license steward has the right to modify or
-      publish new versions of this License. Each version will be given a
-      distinguishing version number.
-
-10.2. Effect of New Versions
-
-      You may distribute the Covered Software under the terms of the version
-      of the License under which You originally received the Covered Software,
-      or under the terms of any subsequent version published by the license
-      steward.
-
-10.3. Modified Versions
-
-      If you create software not governed by this License, and you want to
-      create a new license for such software, you may create and use a
-      modified version of this License if you rename the license and remove
-      any references to the name of the license steward (except to note that
-      such modified license differs from this License).
-
-10.4. Distributing Source Code Form that is Incompatible With Secondary
-      Licenses If You choose to distribute Source Code Form that is
-      Incompatible With Secondary Licenses under the terms of this version of
-      the License, the notice described in Exhibit B of this License must be
-      attached.
-
-Exhibit A - Source Code Form License Notice
-
-      This Source Code Form is subject to the
-      terms of the Mozilla Public License, v.
-      2.0. If a copy of the MPL was not
-      distributed with this file, You can
-      obtain one at
-      http://mozilla.org/MPL/2.0/.
-
-If it is not possible or desirable to put the notice in a particular file,
-then You may include the notice in a location (such as a LICENSE file in a
-relevant directory) where a recipient would be likely to look for such a
-notice.
-
-You may add additional accurate notices of copyright ownership.
-
-Exhibit B - "Incompatible With Secondary Licenses" Notice
-
-      This Source Code Form is "Incompatible
-      With Secondary Licenses", as defined by
-      the Mozilla Public License, v. 2.0.
-
-================================================================
-
-github.com/hashicorp/golang-lru/v2
-https://github.com/hashicorp/golang-lru/v2
-----------------------------------------------------------------
-Copyright (c) 2014 HashiCorp, Inc.
-
-Mozilla Public License, version 2.0
-
-1. Definitions
-
-1.1. "Contributor"
-
-     means each individual or legal entity that creates, contributes to the
-     creation of, or owns Covered Software.
-
-1.2. "Contributor Version"
-
-     means the combination of the Contributions of others (if any) used by a
-     Contributor and that particular Contributor's Contribution.
-
-1.3. "Contribution"
-
-     means Covered Software of a particular Contributor.
-
-1.4. "Covered Software"
-
-     means Source Code Form to which the initial Contributor has attached the
-     notice in Exhibit A, the Executable Form of such Source Code Form, and
-     Modifications of such Source Code Form, in each case including portions
-     thereof.
-
-1.5. "Incompatible With Secondary Licenses"
-     means
-
-     a. that the initial Contributor has attached the notice described in
-        Exhibit B to the Covered Software; or
-
-     b. that the Covered Software was made available under the terms of
-        version 1.1 or earlier of the License, but not also under the terms of
-        a Secondary License.
-
-1.6. "Executable Form"
-
-     means any form of the work other than Source Code Form.
-
-1.7. "Larger Work"
-
-     means a work that combines Covered Software with other material, in a
-     separate file or files, that is not Covered Software.
-
-1.8. "License"
-
-     means this document.
-
-1.9. "Licensable"
-
-     means having the right to grant, to the maximum extent possible, whether
-     at the time of the initial grant or subsequently, any and all of the
-     rights conveyed by this License.
-
-1.10. "Modifications"
-
-     means any of the following:
-
-     a. any file in Source Code Form that results from an addition to,
-        deletion from, or modification of the contents of Covered Software; or
-
-     b. any new file in Source Code Form that contains any Covered Software.
-
-1.11. "Patent Claims" of a Contributor
-
-      means any patent claim(s), including without limitation, method,
-      process, and apparatus claims, in any patent Licensable by such
-      Contributor that would be infringed, but for the grant of the License,
-      by the making, using, selling, offering for sale, having made, import,
-      or transfer of either its Contributions or its Contributor Version.
-
-1.12. "Secondary License"
-
-      means either the GNU General Public License, Version 2.0, the GNU Lesser
-      General Public License, Version 2.1, the GNU Affero General Public
-      License, Version 3.0, or any later versions of those licenses.
-
-1.13. "Source Code Form"
-
-      means the form of the work preferred for making modifications.
-
-1.14. "You" (or "Your")
-
-      means an individual or a legal entity exercising rights under this
-      License. For legal entities, "You" includes any entity that controls, is
-      controlled by, or is under common control with You. For purposes of this
-      definition, "control" means (a) the power, direct or indirect, to cause
-      the direction or management of such entity, whether by contract or
-      otherwise, or (b) ownership of more than fifty percent (50%) of the
-      outstanding shares or beneficial ownership of such entity.
-
-
-2. License Grants and Conditions
-
-2.1. Grants
-
-     Each Contributor hereby grants You a world-wide, royalty-free,
-     non-exclusive license:
-
-     a. under intellectual property rights (other than patent or trademark)
-        Licensable by such Contributor to use, reproduce, make available,
-        modify, display, perform, distribute, and otherwise exploit its
-        Contributions, either on an unmodified basis, with Modifications, or
-        as part of a Larger Work; and
-
-     b. under Patent Claims of such Contributor to make, use, sell, offer for
-        sale, have made, import, and otherwise transfer either its
-        Contributions or its Contributor Version.
-
-2.2. Effective Date
-
-     The licenses granted in Section 2.1 with respect to any Contribution
-     become effective for each Contribution on the date the Contributor first
-     distributes such Contribution.
-
-2.3. Limitations on Grant Scope
-
-     The licenses granted in this Section 2 are the only rights granted under
-     this License. No additional rights or licenses will be implied from the
-     distribution or licensing of Covered Software under this License.
-     Notwithstanding Section 2.1(b) above, no patent license is granted by a
-     Contributor:
-
-     a. for any code that a Contributor has removed from Covered Software; or
-
-     b. for infringements caused by: (i) Your and any other third party's
-        modifications of Covered Software, or (ii) the combination of its
-        Contributions with other software (except as part of its Contributor
-        Version); or
-
-     c. under Patent Claims infringed by Covered Software in the absence of
-        its Contributions.
-
-     This License does not grant any rights in the trademarks, service marks,
-     or logos of any Contributor (except as may be necessary to comply with
-     the notice requirements in Section 3.4).
-
-2.4. Subsequent Licenses
-
-     No Contributor makes additional grants as a result of Your choice to
-     distribute the Covered Software under a subsequent version of this
-     License (see Section 10.2) or under the terms of a Secondary License (if
-     permitted under the terms of Section 3.3).
-
-2.5. Representation
-
-     Each Contributor represents that the Contributor believes its
-     Contributions are its original creation(s) or it has sufficient rights to
-     grant the rights to its Contributions conveyed by this License.
-
-2.6. Fair Use
-
-     This License is not intended to limit any rights You have under
-     applicable copyright doctrines of fair use, fair dealing, or other
-     equivalents.
-
-2.7. Conditions
-
-     Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
-     Section 2.1.
-
-
-3. Responsibilities
-
-3.1. Distribution of Source Form
-
-     All distribution of Covered Software in Source Code Form, including any
-     Modifications that You create or to which You contribute, must be under
-     the terms of this License. You must inform recipients that the Source
-     Code Form of the Covered Software is governed by the terms of this
-     License, and how they can obtain a copy of this License. You may not
-     attempt to alter or restrict the recipients' rights in the Source Code
-     Form.
-
-3.2. Distribution of Executable Form
-
-     If You distribute Covered Software in Executable Form then:
-
-     a. such Covered Software must also be made available in Source Code Form,
-        as described in Section 3.1, and You must inform recipients of the
-        Executable Form how they can obtain a copy of such Source Code Form by
-        reasonable means in a timely manner, at a charge no more than the cost
-        of distribution to the recipient; and
-
-     b. You may distribute such Executable Form under the terms of this
-        License, or sublicense it under different terms, provided that the
-        license for the Executable Form does not attempt to limit or alter the
-        recipients' rights in the Source Code Form under this License.
-
-3.3. Distribution of a Larger Work
-
-     You may create and distribute a Larger Work under terms of Your choice,
-     provided that You also comply with the requirements of this License for
-     the Covered Software. If the Larger Work is a combination of Covered
-     Software with a work governed by one or more Secondary Licenses, and the
-     Covered Software is not Incompatible With Secondary Licenses, this
-     License permits You to additionally distribute such Covered Software
-     under the terms of such Secondary License(s), so that the recipient of
-     the Larger Work may, at their option, further distribute the Covered
-     Software under the terms of either this License or such Secondary
-     License(s).
-
-3.4. Notices
-
-     You may not remove or alter the substance of any license notices
-     (including copyright notices, patent notices, disclaimers of warranty, or
-     limitations of liability) contained within the Source Code Form of the
-     Covered Software, except that You may alter any license notices to the
-     extent required to remedy known factual inaccuracies.
-
-3.5. Application of Additional Terms
-
-     You may choose to offer, and to charge a fee for, warranty, support,
-     indemnity or liability obligations to one or more recipients of Covered
-     Software. However, You may do so only on Your own behalf, and not on
-     behalf of any Contributor. You must make it absolutely clear that any
-     such warranty, support, indemnity, or liability obligation is offered by
-     You alone, and You hereby agree to indemnify every Contributor for any
-     liability incurred by such Contributor as a result of warranty, support,
-     indemnity or liability terms You offer. You may include additional
-     disclaimers of warranty and limitations of liability specific to any
-     jurisdiction.
-
-4. Inability to Comply Due to Statute or Regulation
-
-   If it is impossible for You to comply with any of the terms of this License
-   with respect to some or all of the Covered Software due to statute,
-   judicial order, or regulation then You must: (a) comply with the terms of
-   this License to the maximum extent possible; and (b) describe the
-   limitations and the code they affect. Such description must be placed in a
-   text file included with all distributions of the Covered Software under
-   this License. Except to the extent prohibited by statute or regulation,
-   such description must be sufficiently detailed for a recipient of ordinary
-   skill to be able to understand it.
-
-5. Termination
-
-5.1. The rights granted under this License will terminate automatically if You
-     fail to comply with any of its terms. However, if You become compliant,
-     then the rights granted under this License from a particular Contributor
-     are reinstated (a) provisionally, unless and until such Contributor
-     explicitly and finally terminates Your grants, and (b) on an ongoing
-     basis, if such Contributor fails to notify You of the non-compliance by
-     some reasonable means prior to 60 days after You have come back into
-     compliance. Moreover, Your grants from a particular Contributor are
-     reinstated on an ongoing basis if such Contributor notifies You of the
-     non-compliance by some reasonable means, this is the first time You have
-     received notice of non-compliance with this License from such
-     Contributor, and You become compliant prior to 30 days after Your receipt
-     of the notice.
-
-5.2. If You initiate litigation against any entity by asserting a patent
-     infringement claim (excluding declaratory judgment actions,
-     counter-claims, and cross-claims) alleging that a Contributor Version
-     directly or indirectly infringes any patent, then the rights granted to
-     You by any and all Contributors for the Covered Software under Section
-     2.1 of this License shall terminate.
-
-5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
-     license agreements (excluding distributors and resellers) which have been
-     validly granted by You or Your distributors under this License prior to
-     termination shall survive termination.
-
-6. Disclaimer of Warranty
-
-   Covered Software is provided under this License on an "as is" basis,
-   without warranty of any kind, either expressed, implied, or statutory,
-   including, without limitation, warranties that the Covered Software is free
-   of defects, merchantable, fit for a particular purpose or non-infringing.
-   The entire risk as to the quality and performance of the Covered Software
-   is with You. Should any Covered Software prove defective in any respect,
-   You (not any Contributor) assume the cost of any necessary servicing,
-   repair, or correction. This disclaimer of warranty constitutes an essential
-   part of this License. No use of  any Covered Software is authorized under
-   this License except under this disclaimer.
-
-7. Limitation of Liability
-
-   Under no circumstances and under no legal theory, whether tort (including
-   negligence), contract, or otherwise, shall any Contributor, or anyone who
-   distributes Covered Software as permitted above, be liable to You for any
-   direct, indirect, special, incidental, or consequential damages of any
-   character including, without limitation, damages for lost profits, loss of
-   goodwill, work stoppage, computer failure or malfunction, or any and all
-   other commercial damages or losses, even if such party shall have been
-   informed of the possibility of such damages. This limitation of liability
-   shall not apply to liability for death or personal injury resulting from
-   such party's negligence to the extent applicable law prohibits such
-   limitation. Some jurisdictions do not allow the exclusion or limitation of
-   incidental or consequential damages, so this exclusion and limitation may
-   not apply to You.
-
-8. Litigation
-
-   Any litigation relating to this License may be brought only in the courts
-   of a jurisdiction where the defendant maintains its principal place of
-   business and such litigation shall be governed by laws of that
-   jurisdiction, without reference to its conflict-of-law provisions. Nothing
-   in this Section shall prevent a party's ability to bring cross-claims or
-   counter-claims.
-
-9. Miscellaneous
-
-   This License represents the complete agreement concerning the subject
-   matter hereof. If any provision of this License is held to be
-   unenforceable, such provision shall be reformed only to the extent
-   necessary to make it enforceable. Any law or regulation which provides that
-   the language of a contract shall be construed against the drafter shall not
-   be used to construe this License against a Contributor.
-
-
 10. Versions of the License
 
 10.1. New Versions
@@ -24119,6 +23942,43 @@ SOFTWARE.
 
 ================================================================
 
+github.com/munnerz/goautoneg
+https://github.com/munnerz/goautoneg
+----------------------------------------------------------------
+Copyright (c) 2011, Open Knowledge Foundation Ltd.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+
+    Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+
+    Neither the name of the Open Knowledge Foundation Ltd. nor the
+    names of its contributors may be used to endorse or promote
+    products derived from this software without specific prior written
+    permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
 github.com/nats-io/jwt/v2
 https://github.com/nats-io/jwt/v2
 ----------------------------------------------------------------
@@ -33171,203 +33031,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ================================================================
 
-gopkg.in/ini.v1
-https://gopkg.in/ini.v1
-----------------------------------------------------------------
-Apache License
-Version 2.0, January 2004
-http://www.apache.org/licenses/
-
-TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-1. Definitions.
-
-"License" shall mean the terms and conditions for use, reproduction, and
-distribution as defined by Sections 1 through 9 of this document.
-
-"Licensor" shall mean the copyright owner or entity authorized by the copyright
-owner that is granting the License.
-
-"Legal Entity" shall mean the union of the acting entity and all other entities
-that control, are controlled by, or are under common control with that entity.
-For the purposes of this definition, "control" means (i) the power, direct or
-indirect, to cause the direction or management of such entity, whether by
-contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
-outstanding shares, or (iii) beneficial ownership of such entity.
-
-"You" (or "Your") shall mean an individual or Legal Entity exercising
-permissions granted by this License.
-
-"Source" form shall mean the preferred form for making modifications, including
-but not limited to software source code, documentation source, and configuration
-files.
-
-"Object" form shall mean any form resulting from mechanical transformation or
-translation of a Source form, including but not limited to compiled object code,
-generated documentation, and conversions to other media types.
-
-"Work" shall mean the work of authorship, whether in Source or Object form, made
-available under the License, as indicated by a copyright notice that is included
-in or attached to the work (an example is provided in the Appendix below).
-
-"Derivative Works" shall mean any work, whether in Source or Object form, that
-is based on (or derived from) the Work and for which the editorial revisions,
-annotations, elaborations, or other modifications represent, as a whole, an
-original work of authorship. For the purposes of this License, Derivative Works
-shall not include works that remain separable from, or merely link (or bind by
-name) to the interfaces of, the Work and Derivative Works thereof.
-
-"Contribution" shall mean any work of authorship, including the original version
-of the Work and any modifications or additions to that Work or Derivative Works
-thereof, that is intentionally submitted to Licensor for inclusion in the Work
-by the copyright owner or by an individual or Legal Entity authorized to submit
-on behalf of the copyright owner. For the purposes of this definition,
-"submitted" means any form of electronic, verbal, or written communication sent
-to the Licensor or its representatives, including but not limited to
-communication on electronic mailing lists, source code control systems, and
-issue tracking systems that are managed by, or on behalf of, the Licensor for
-the purpose of discussing and improving the Work, but excluding communication
-that is conspicuously marked or otherwise designated in writing by the copyright
-owner as "Not a Contribution."
-
-"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
-of whom a Contribution has been received by Licensor and subsequently
-incorporated within the Work.
-
-2. Grant of Copyright License.
-
-Subject to the terms and conditions of this License, each Contributor hereby
-grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
-irrevocable copyright license to reproduce, prepare Derivative Works of,
-publicly display, publicly perform, sublicense, and distribute the Work and such
-Derivative Works in Source or Object form.
-
-3. Grant of Patent License.
-
-Subject to the terms and conditions of this License, each Contributor hereby
-grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
-irrevocable (except as stated in this section) patent license to make, have
-made, use, offer to sell, sell, import, and otherwise transfer the Work, where
-such license applies only to those patent claims licensable by such Contributor
-that are necessarily infringed by their Contribution(s) alone or by combination
-of their Contribution(s) with the Work to which such Contribution(s) was
-submitted. If You institute patent litigation against any entity (including a
-cross-claim or counterclaim in a lawsuit) alleging that the Work or a
-Contribution incorporated within the Work constitutes direct or contributory
-patent infringement, then any patent licenses granted to You under this License
-for that Work shall terminate as of the date such litigation is filed.
-
-4. Redistribution.
-
-You may reproduce and distribute copies of the Work or Derivative Works thereof
-in any medium, with or without modifications, and in Source or Object form,
-provided that You meet the following conditions:
-
-You must give any other recipients of the Work or Derivative Works a copy of
-this License; and
-You must cause any modified files to carry prominent notices stating that You
-changed the files; and
-You must retain, in the Source form of any Derivative Works that You distribute,
-all copyright, patent, trademark, and attribution notices from the Source form
-of the Work, excluding those notices that do not pertain to any part of the
-Derivative Works; and
-If the Work includes a "NOTICE" text file as part of its distribution, then any
-Derivative Works that You distribute must include a readable copy of the
-attribution notices contained within such NOTICE file, excluding those notices
-that do not pertain to any part of the Derivative Works, in at least one of the
-following places: within a NOTICE text file distributed as part of the
-Derivative Works; within the Source form or documentation, if provided along
-with the Derivative Works; or, within a display generated by the Derivative
-Works, if and wherever such third-party notices normally appear. The contents of
-the NOTICE file are for informational purposes only and do not modify the
-License. You may add Your own attribution notices within Derivative Works that
-You distribute, alongside or as an addendum to the NOTICE text from the Work,
-provided that such additional attribution notices cannot be construed as
-modifying the License.
-You may add Your own copyright statement to Your modifications and may provide
-additional or different license terms and conditions for use, reproduction, or
-distribution of Your modifications, or for any such Derivative Works as a whole,
-provided Your use, reproduction, and distribution of the Work otherwise complies
-with the conditions stated in this License.
-
-5. Submission of Contributions.
-
-Unless You explicitly state otherwise, any Contribution intentionally submitted
-for inclusion in the Work by You to the Licensor shall be under the terms and
-conditions of this License, without any additional terms or conditions.
-Notwithstanding the above, nothing herein shall supersede or modify the terms of
-any separate license agreement you may have executed with Licensor regarding
-such Contributions.
-
-6. Trademarks.
-
-This License does not grant permission to use the trade names, trademarks,
-service marks, or product names of the Licensor, except as required for
-reasonable and customary use in describing the origin of the Work and
-reproducing the content of the NOTICE file.
-
-7. Disclaimer of Warranty.
-
-Unless required by applicable law or agreed to in writing, Licensor provides the
-Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
-including, without limitation, any warranties or conditions of TITLE,
-NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
-solely responsible for determining the appropriateness of using or
-redistributing the Work and assume any risks associated with Your exercise of
-permissions under this License.
-
-8. Limitation of Liability.
-
-In no event and under no legal theory, whether in tort (including negligence),
-contract, or otherwise, unless required by applicable law (such as deliberate
-and grossly negligent acts) or agreed to in writing, shall any Contributor be
-liable to You for damages, including any direct, indirect, special, incidental,
-or consequential damages of any character arising as a result of this License or
-out of the use or inability to use the Work (including but not limited to
-damages for loss of goodwill, work stoppage, computer failure or malfunction, or
-any and all other commercial damages or losses), even if such Contributor has
-been advised of the possibility of such damages.
-
-9. Accepting Warranty or Additional Liability.
-
-While redistributing the Work or Derivative Works thereof, You may choose to
-offer, and charge a fee for, acceptance of support, warranty, indemnity, or
-other liability obligations and/or rights consistent with this License. However,
-in accepting such obligations, You may act only on Your own behalf and on Your
-sole responsibility, not on behalf of any other Contributor, and only if You
-agree to indemnify, defend, and hold each Contributor harmless for any liability
-incurred by, or claims asserted against, such Contributor by reason of your
-accepting any such warranty or additional liability.
-
-END OF TERMS AND CONDITIONS
-
-APPENDIX: How to apply the Apache License to your work
-
-To apply the Apache License to your work, attach the following boilerplate
-notice, with the fields enclosed by brackets "[]" replaced with your own
-identifying information. (Don't include the brackets!) The text should be
-enclosed in the appropriate comment syntax for the file format. We also
-recommend that a file or class name and description of purpose be included on
-the same "printed page" as the copyright notice for easier identification within
-third-party archives.
-
-   Copyright 2014 Unknwon
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-================================================================
-
 gopkg.in/yaml.v2
 https://gopkg.in/yaml.v2
 ----------------------------------------------------------------

Makefile

@@ -34,11 +34,14 @@ verifiers: lint check-gen
 
 check-gen: ## check for updated autogenerated files
 	@go generate ./... >/dev/null
+	@go mod tidy -compat=1.21
 	@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
+	@(! git diff --name-only | grep 'go.sum') || (echo "Non-committed changes in auto-generated go.sum is detected, please commit them to proceed." && false)
 
 lint: getdeps ## runs golangci-lint suite of linters
 	@echo "Running $@ check"
 	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml
+	@command typos && typos ./ || echo "typos binary is not found.. skipping.."
 
 lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
 	@echo "Running $@ check"
@@ -86,9 +89,9 @@ test-race: verifiers build ## builds minio, runs linters, tests (race)
 
 test-iam: install-race ## verify IAM (external IDP, etcd backends)
 	@echo "Running tests for IAM (external IDP, etcd backends)"
-	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue,dev -v -run TestIAM* ./cmd
+	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -timeout 15m -tags kqueue,dev -v -run TestIAM* ./cmd
 	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
-	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue,dev -v -run TestIAM* ./cmd
+	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 15m -race -tags kqueue,dev -v -run TestIAM* ./cmd
 
 test-iam-ldap-upgrade-import: install-race ## verify IAM (external LDAP IDP)
 	@echo "Running upgrade tests for IAM (LDAP backend)"
@@ -165,9 +168,9 @@ hotfix-vars:
 	$(eval VERSION := $(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD))
 
 hotfix: hotfix-vars clean install ## builds minio binary with hotfix tags
-	@wget -q -c https://github.com/minio/pkger/releases/download/v2.2.9/pkger_2.2.9_linux_amd64.deb
+	@wget -q -c https://github.com/minio/pkger/releases/download/v2.3.1/pkger_2.3.1_linux_amd64.deb
 	@wget -q -c https://raw.githubusercontent.com/minio/minio-service/v1.0.1/linux-systemd/distributed/minio.service
-	@sudo apt install ./pkger_2.2.9_linux_amd64.deb --yes
+	@sudo apt install ./pkger_2.3.1_linux_amd64.deb --yes
 	@mkdir -p minio-release/$(GOOS)-$(GOARCH)/archive
 	@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio
 	@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION)

buildscripts/verify-healing-empty-erasure-set.sh

@@ -39,7 +39,7 @@ function start_minio_3_node() {
 
 	export MC_HOST_myminio="http://minio:minio123@127.0.0.1:$((start_port + 1))"
 
-	/tmp/mc ready myminio
+	timeout 15m /tmp/mc ready myminio || fail
 
 	# Wait for all drives to be online and formatted
 	while [ $(/tmp/mc admin info --json myminio | jq '.info.servers[].drives[].state | select(. != "ok")' | wc -l) -gt 0 ]; do sleep 1; done
@@ -62,48 +62,23 @@ function start_minio_3_node() {
 		fi
 
 		# Failure
-		for i in $(seq 1 3); do
-			echo "server$i log:"
-			cat "${WORK_DIR}/dist-minio-server$i.log"
-		done
-		pkill -9 minio
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
+		fail
 	done
 
 	if ! ps -p $pid1 1>&2 >/dev/null; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/dist-minio-server1.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
+		echo "minio-server-1 is not running." && fail
 	fi
 
 	if ! ps -p $pid2 1>&2 >/dev/null; then
-		echo "server2 log:"
-		cat "${WORK_DIR}/dist-minio-server2.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
+		echo "minio-server-2 is not running." && fail
 	fi
 
 	if ! ps -p $pid3 1>&2 >/dev/null; then
-		echo "server3 log:"
-		cat "${WORK_DIR}/dist-minio-server3.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
+		echo "minio-server-3 is not running." && fail
 	fi
 
 	if ! pkill minio; then
-		for i in $(seq 1 3); do
-			echo "server$i log:"
-			cat "${WORK_DIR}/dist-minio-server$i.log"
-		done
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
+		fail
 	fi
 
 	sleep 1
@@ -115,8 +90,18 @@ function start_minio_3_node() {
 	fi
 }
 
+function fail() {
+	for i in $(seq 1 3); do
+		echo "server$i log:"
+		cat "${WORK_DIR}/dist-minio-server$i.log"
+	done
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+}
+
 function check_online() {
-	if ! grep -q 'Status:' ${WORK_DIR}/dist-minio-*.log; then
+	if ! grep -q 'API:' ${WORK_DIR}/dist-minio-*.log; then
 		echo "1"
 	fi
 }

buildscripts/verify-healing.sh

@@ -47,43 +47,25 @@ function start_minio_3_node() {
 	disown $pid3
 
 	export MC_HOST_myminio="http://minio:minio123@127.0.0.1:$((start_port + 1))"
-	/tmp/mc ready myminio
+	timeout 15m /tmp/mc ready myminio || fail
 
 	[ ${first_time} -eq 0 ] && upload_objects
 	[ ${first_time} -ne 0 ] && sleep 120
 
 	if ! ps -p $pid1 1>&2 >/dev/null; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/dist-minio-server1.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
+		echo "minio server 1 is not running" && fail
 	fi
 
 	if ! ps -p $pid2 1>&2 >/dev/null; then
-		echo "server2 log:"
-		cat "${WORK_DIR}/dist-minio-server2.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
+		echo "minio server 2 is not running" && fail
 	fi
 
 	if ! ps -p $pid3 1>&2 >/dev/null; then
-		echo "server3 log:"
-		cat "${WORK_DIR}/dist-minio-server3.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
+		echo "minio server 3 is not running" && fail
 	fi
 
 	if ! pkill minio; then
-		for i in $(seq 1 3); do
-			echo "server$i log:"
-			cat "${WORK_DIR}/dist-minio-server$i.log"
-		done
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
+		fail
 	fi
 
 	sleep 1
@@ -96,7 +78,7 @@ function start_minio_3_node() {
 }
 
 function check_heal() {
-	if ! grep -q 'Status:' ${WORK_DIR}/dist-minio-*.log; then
+	if ! grep -q 'API:' ${WORK_DIR}/dist-minio-*.log; then
 		return 1
 	fi
 
@@ -118,6 +100,17 @@ function purge() {
 	rm -rf "$1"
 }
 
+function fail() {
+	for i in $(seq 1 3); do
+		echo "server$i log:"
+		cat "${WORK_DIR}/dist-minio-server$i.log"
+	done
+	pkill -9 minio
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+}
+
 function __init__() {
 	echo "Initializing environment"
 	mkdir -p "$WORK_DIR"
@@ -155,14 +148,7 @@ function perform_test() {
 	check_heal ${1}
 	rv=$?
 	if [ "$rv" == "1" ]; then
-		for i in $(seq 1 3); do
-			echo "server$i log:"
-			cat "${WORK_DIR}/dist-minio-server$i.log"
-		done
-		pkill -9 minio
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
+		fail
 	fi
 }
 

cmd/admin-handler-utils.go

@@ -216,6 +216,12 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
+		case errors.Is(err, errTierInvalidConfig):
+			apiErr = APIError{
+				Code:           "XMinioAdminTierInvalidConfig",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		default:
 			apiErr = errorCodes.ToAPIErrWithErr(toAdminAPIErrCode(ctx, err), err)
 		}

cmd/admin-handlers-idp-ldap.go

@@ -479,3 +479,180 @@ func (a adminAPIHandlers) ListAccessKeysLDAP(w http.ResponseWriter, r *http.Requ
 
 	writeSuccessResponseJSON(w, encryptedData)
 }
+
+// ListAccessKeysLDAPBulk - GET /minio/admin/v3/idp/ldap/list-access-keys-bulk
+func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	dnList := r.Form["userDNs"]
+	isAll := r.Form.Get("all") == "true"
+	onlySelf := !isAll && len(dnList) == 0
+
+	if isAll && len(dnList) > 0 {
+		// This should be checked on client side, so return generic error
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	// Empty DN list and not self, list access keys for all users
+	if isAll {
+		if !globalIAMSys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          policy.ListUsersAdminAction,
+			ConditionValues: getConditionValues(r, "", cred),
+			IsOwner:         owner,
+			Claims:          cred.Claims,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+	} else if len(dnList) == 1 {
+		var dn string
+		foundResult, err := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(dnList[0])
+		if err == nil {
+			dn = foundResult.NormDN
+		}
+		if dn == cred.ParentUser || dnList[0] == cred.ParentUser {
+			onlySelf = true
+		}
+	}
+
+	if !globalIAMSys.IsAllowed(policy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          policy.ListServiceAccountsAdminAction,
+		ConditionValues: getConditionValues(r, "", cred),
+		IsOwner:         owner,
+		Claims:          cred.Claims,
+		DenyOnly:        onlySelf,
+	}) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
+	if onlySelf && len(dnList) == 0 {
+		selfDN := cred.AccessKey
+		if cred.ParentUser != "" {
+			selfDN = cred.ParentUser
+		}
+		dnList = append(dnList, selfDN)
+	}
+
+	var ldapUserList []string
+	if isAll {
+		ldapUsers, err := globalIAMSys.ListLDAPUsers(ctx)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		for user := range ldapUsers {
+			ldapUserList = append(ldapUserList, user)
+		}
+	} else {
+		for _, userDN := range dnList {
+			// Validate the userDN
+			foundResult, err := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(userDN)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+			if foundResult == nil {
+				continue
+			}
+			ldapUserList = append(ldapUserList, foundResult.NormDN)
+		}
+	}
+
+	listType := r.Form.Get("listType")
+	var listSTSKeys, listServiceAccounts bool
+	switch listType {
+	case madmin.AccessKeyListUsersOnly:
+		listSTSKeys = false
+		listServiceAccounts = false
+	case madmin.AccessKeyListSTSOnly:
+		listSTSKeys = true
+		listServiceAccounts = false
+	case madmin.AccessKeyListSvcaccOnly:
+		listSTSKeys = false
+		listServiceAccounts = true
+	case madmin.AccessKeyListAll:
+		listSTSKeys = true
+		listServiceAccounts = true
+	default:
+		err := errors.New("invalid list type")
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, err), r.URL)
+		return
+	}
+
+	accessKeyMap := make(map[string]madmin.ListAccessKeysLDAPResp)
+	for _, internalDN := range ldapUserList {
+		externalDN := globalIAMSys.LDAPConfig.DecodeDN(internalDN)
+		accessKeys := madmin.ListAccessKeysLDAPResp{}
+		if listSTSKeys {
+			stsKeys, err := globalIAMSys.ListSTSAccounts(ctx, internalDN)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+			for _, sts := range stsKeys {
+				expiryTime := sts.Expiration
+				accessKeys.STSKeys = append(accessKeys.STSKeys, madmin.ServiceAccountInfo{
+					AccessKey:  sts.AccessKey,
+					Expiration: &expiryTime,
+				})
+			}
+			// if only STS keys, skip if user has no STS keys
+			if !listServiceAccounts && len(stsKeys) == 0 {
+				continue
+			}
+		}
+
+		if listServiceAccounts {
+			serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, internalDN)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+			for _, svc := range serviceAccounts {
+				expiryTime := svc.Expiration
+				accessKeys.ServiceAccounts = append(accessKeys.ServiceAccounts, madmin.ServiceAccountInfo{
+					AccessKey:  svc.AccessKey,
+					Expiration: &expiryTime,
+				})
+			}
+			// if only service accounts, skip if user has no service accounts
+			if !listSTSKeys && len(serviceAccounts) == 0 {
+				continue
+			}
+		}
+		accessKeyMap[externalDN] = accessKeys
+	}
+
+	data, err := json.Marshal(accessKeyMap)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, encryptedData)
+}

cmd/admin-handlers-pools.go

@@ -374,6 +374,7 @@ func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request)
 	globalNotificationSys.StopRebalance(r.Context())
 	writeSuccessResponseHeadersOnly(w)
 	adminLogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt))
+	globalNotificationSys.LoadRebalanceMeta(ctx, false)
 }
 
 func proxyDecommissionRequest(ctx context.Context, defaultEndPoint Endpoint, w http.ResponseWriter, r *http.Request) (proxy bool) {

cmd/admin-handlers-users-race_test.go

@@ -120,9 +120,12 @@ func (s *TestSuiteIAM) TestDeleteUserRace(c *check) {
 			c.Fatalf("Unable to set user: %v", err)
 		}
 
-		err = s.adm.SetPolicy(ctx, policy, accessKey, false)
-		if err != nil {
-			c.Fatalf("Unable to set policy: %v", err)
+		userReq := madmin.PolicyAssociationReq{
+			Policies: []string{policy},
+			User:     accessKey,
+		}
+		if _, err := s.adm.AttachPolicy(ctx, userReq); err != nil {
+			c.Fatalf("Unable to attach policy: %v", err)
 		}
 
 		accessKeys[i] = accessKey

cmd/admin-handlers-users.go

@@ -2308,7 +2308,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					// clean import.
 					err := globalIAMSys.DeleteServiceAccount(ctx, svcAcctReq.AccessKey, true)
 					if err != nil {
-						delErr := fmt.Errorf("failed to delete existing service account(%s) before importing it: %w", svcAcctReq.AccessKey, err)
+						delErr := fmt.Errorf("failed to delete existing service account (%s) before importing it: %w", svcAcctReq.AccessKey, err)
 						writeErrorResponseJSON(ctx, w, importError(ctx, delErr, allSvcAcctsFile, user), r.URL)
 						return
 					}

cmd/admin-handlers-users_test.go

@@ -239,9 +239,12 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
 	c.Assert(v.Status, madmin.AccountEnabled)
 
 	// 3. Associate policy and check that user can access
-	err = s.adm.SetPolicy(ctx, "readwrite", accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{"readwrite"},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	client := s.getUserClient(c, accessKey, secretKey, "")
@@ -348,9 +351,12 @@ func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
 	if err != nil {
 		c.Fatalf("policy add error: %v", err)
 	}
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 	// 2.3 check user has access to bucket
 	c.mustListObjects(ctx, uClient, bucket)
@@ -470,9 +476,12 @@ func (s *TestSuiteIAM) TestAddServiceAccountPerms(c *check) {
 	c.mustNotListObjects(ctx, uClient, "testbucket")
 
 	// 3.2 associate policy to user
-	err = s.adm.SetPolicy(ctx, policy1, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy1},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	admClnt := s.getAdminClient(c, accessKey, secretKey, "")
@@ -490,10 +499,22 @@ func (s *TestSuiteIAM) TestAddServiceAccountPerms(c *check) {
 		c.Fatalf("policy was missing!")
 	}
 
-	// 3.2 associate policy to user
-	err = s.adm.SetPolicy(ctx, policy2, accessKey, false)
+	// Detach policy1 to set up for policy2
+	_, err = s.adm.DetachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy1},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to detach policy: %v", err)
+	}
+
+	// 3.2 associate policy to user
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy2},
+		User:     accessKey,
+	})
+	if err != nil {
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	// 3.3 check user can create service account implicitly.
@@ -571,9 +592,12 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
 	c.mustNotListObjects(ctx, uClient, bucket)
 
 	// 3.2 associate policy to user
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 	// 3.3 check user has access to bucket
 	c.mustListObjects(ctx, uClient, bucket)
@@ -726,9 +750,12 @@ func (s *TestSuiteIAM) TestGroupAddRemove(c *check) {
 	c.mustNotListObjects(ctx, uClient, bucket)
 
 	// 3. Associate policy to group and check user got access.
-	err = s.adm.SetPolicy(ctx, policy, group, true)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		Group:    group,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 	// 3.1 check user has access to bucket
 	c.mustListObjects(ctx, uClient, bucket)
@@ -871,9 +898,12 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
 		c.Fatalf("Unable to set user: %v", err)
 	}
 
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	// Create an madmin client with user creds
@@ -952,9 +982,12 @@ func (s *TestSuiteIAM) TestServiceAccountDurationSecondsCondition(c *check) {
 		c.Fatalf("Unable to set user: %v", err)
 	}
 
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	// Create an madmin client with user creds
@@ -1031,9 +1064,12 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
 		c.Fatalf("Unable to set user: %v", err)
 	}
 
-	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	_, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
+		Policies: []string{policy},
+		User:     accessKey,
+	})
 	if err != nil {
-		c.Fatalf("Unable to set policy: %v", err)
+		c.Fatalf("unable to attach policy: %v", err)
 	}
 
 	// 1. Create a service account for the user

cmd/admin-handlers.go

@@ -237,6 +237,7 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 
 	if globalIsDistErasure {
 		// Notify all other MinIO peers signal service.
+		startTime := time.Now().Add(restartUpdateDelay)
 		ng := WithNPeers(len(globalNotificationSys.peerClients))
 		for idx, client := range globalNotificationSys.peerClients {
 			_, ok := failedClients[idx]
@@ -247,7 +248,7 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 			ng.Go(ctx, func() error {
 				prs, ok := peerResults[client.String()]
 				if ok && prs.CurrentVersion != prs.UpdatedVersion && prs.UpdatedVersion != "" {
-					return client.SignalService(serviceRestart, "", dryRun)
+					return client.SignalService(serviceRestart, "", dryRun, &startTime)
 				}
 				return nil
 			}, idx, *client.host)
@@ -542,9 +543,12 @@ func (a adminAPIHandlers) ServiceV2Handler(w http.ResponseWriter, r *http.Reques
 	}
 
 	var objectAPI ObjectLayer
+	var execAt *time.Time
 	switch serviceSig {
 	case serviceRestart:
 		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceRestartAdminAction)
+		t := time.Now().Add(restartUpdateDelay)
+		execAt = &t
 	case serviceStop:
 		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceStopAdminAction)
 	case serviceFreeze, serviceUnFreeze:
@@ -571,7 +575,7 @@ func (a adminAPIHandlers) ServiceV2Handler(w http.ResponseWriter, r *http.Reques
 	}
 
 	if globalIsDistErasure {
-		for _, nerr := range globalNotificationSys.SignalServiceV2(serviceSig, dryRun) {
+		for _, nerr := range globalNotificationSys.SignalServiceV2(serviceSig, dryRun, execAt) {
 			if nerr.Err != nil && process {
 				waitingDrives := map[string]madmin.DiskMetrics{}
 				jerr := json.Unmarshal([]byte(nerr.Err.Error()), &waitingDrives)
@@ -844,9 +848,10 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
 }
 
 func lriToLockEntry(l lockRequesterInfo, now time.Time, resource, server string) *madmin.LockEntry {
+	t := time.Unix(0, l.Timestamp)
 	entry := &madmin.LockEntry{
-		Timestamp:  l.Timestamp,
-		Elapsed:    now.Sub(l.Timestamp),
+		Timestamp:  t,
+		Elapsed:    now.Sub(t),
 		Resource:   resource,
 		ServerList: []string{server},
 		Source:     l.Source,
@@ -2182,7 +2187,7 @@ func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseHeadersOnly(w)
 }
 
-// KMSKeyStatusHandler - GET /minio/admin/v3/kms/status
+// KMSStatusHandler - GET /minio/admin/v3/kms/status
 func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
@@ -2417,7 +2422,7 @@ func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) ma
 		Mode:          string(mode),
 		Domain:        domain,
 		Region:        globalSite.Region(),
-		SQSARN:        globalEventNotifier.GetARNList(false),
+		SQSARN:        globalEventNotifier.GetARNList(),
 		DeploymentID:  globalDeploymentID(),
 		Buckets:       buckets,
 		Objects:       objects,

cmd/admin-handlers_test.go

@@ -463,6 +463,7 @@ func TestTopLockEntries(t *testing.T) {
 			Owner:      lri.Owner,
 			ID:         lri.UID,
 			Quorum:     lri.Quorum,
+			Timestamp:  time.Unix(0, lri.Timestamp),
 		})
 	}
 

cmd/admin-heal-ops.go

@@ -761,6 +761,15 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		return nil
 	}
 
+	countOKDrives := func(drives []madmin.HealDriveInfo) (count int) {
+		for _, drive := range drives {
+			if drive.State == madmin.DriveStateOk {
+				count++
+			}
+		}
+		return count
+	}
+
 	// task queued, now wait for the response.
 	select {
 	case res := <-task.respCh:
@@ -781,6 +790,11 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		if res.err != nil {
 			res.result.Detail = res.err.Error()
 		}
+		if res.result.ParityBlocks > 0 && res.result.DataBlocks > 0 && res.result.DataBlocks > res.result.ParityBlocks {
+			if got := countOKDrives(res.result.After.Drives); got < res.result.ParityBlocks {
+				res.result.Detail = fmt.Sprintf("quorum loss - expected %d minimum, got drive states in OK %d", res.result.ParityBlocks, got)
+			}
+		}
 		return h.pushHealResultItem(res.result)
 	case <-h.ctx.Done():
 		return nil

cmd/admin-router.go

@@ -301,8 +301,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		// LDAP specific service accounts ops
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp/ldap/add-service-account").HandlerFunc(adminMiddleware(adminAPI.AddServiceAccountLDAP))
 		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp/ldap/list-access-keys").
-			HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysLDAP)).
-			Queries("userDN", "{userDN:.*}", "listType", "{listType:.*}")
+			HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysLDAP)).Queries("userDN", "{userDN:.*}", "listType", "{listType:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp/ldap/list-access-keys-bulk").
+			HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysLDAPBulk)).Queries("listType", "{listType:.*}")
 
 		// LDAP IAM operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(adminMiddleware(adminAPI.ListLDAPPolicyMappingEntities))
@@ -340,6 +341,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-jobs").HandlerFunc(
 			adminMiddleware(adminAPI.ListBatchJobs))
 
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/status-job").HandlerFunc(
+			adminMiddleware(adminAPI.BatchJobStatus))
+
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/describe-job").HandlerFunc(
 			adminMiddleware(adminAPI.DescribeBatchJob))
 		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/cancel-job").HandlerFunc(

cmd/api-response.go

@@ -946,10 +946,20 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
 
 // writeErrorResponse writes error headers
 func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
-	if err.HTTPStatusCode == http.StatusServiceUnavailable {
-		// Set retry-after header to indicate user-agents to retry request after 120secs.
+	switch err.HTTPStatusCode {
+	case http.StatusServiceUnavailable:
+		// Set retry-after header to indicate user-agents to retry request after 60 seconds.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
-		w.Header().Set(xhttp.RetryAfter, "120")
+		w.Header().Set(xhttp.RetryAfter, "60")
+	case http.StatusTooManyRequests:
+		_, deadline := globalAPIConfig.getRequestsPool()
+		if deadline <= 0 {
+			// Set retry-after header to indicate user-agents to retry request after 10 seconds.
+			// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+			w.Header().Set(xhttp.RetryAfter, "10")
+		} else {
+			w.Header().Set(xhttp.RetryAfter, strconv.Itoa(int(deadline.Seconds())))
+		}
 	}
 
 	switch err.Code {

cmd/api-router.go

@@ -436,7 +436,7 @@ func registerAPIRouter(router *mux.Router) {
 			Queries("notification", "")
 		// ListenNotification
 		router.Methods(http.MethodGet).
-			HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag)).
+			HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag, traceHdrsS3HFlag)).
 			Queries("events", "{events:.*}")
 		// ResetBucketReplicationStatus - MinIO extension API
 		router.Methods(http.MethodGet).
@@ -615,7 +615,7 @@ func registerAPIRouter(router *mux.Router) {
 
 	// ListenNotification
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).
-		HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag)).
+		HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag, traceHdrsS3HFlag)).
 		Queries("events", "{events:.*}")
 
 	// ListBuckets

cmd/background-newdisks-heal-ops.go

@@ -88,6 +88,8 @@ type healingTracker struct {
 
 	ItemsSkipped uint64
 	BytesSkipped uint64
+
+	RetryAttempts uint64
 	// Add future tracking capabilities
 	// Be sure that they are included in toHealingDisk
 }
@@ -360,10 +362,10 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 
 func getLocalDisksToHeal() (disksToHeal Endpoints) {
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 	for _, disk := range localDrives {
-		_, err := disk.GetDiskID()
+		_, err := disk.DiskInfo(context.Background(), DiskInfoOptions{})
 		if errors.Is(err, errUnformattedDisk) {
 			disksToHeal = append(disksToHeal, disk.Endpoint())
 			continue
@@ -382,6 +384,8 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 
 var newDiskHealingTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 
+var errRetryHealing = errors.New("some items failed to heal, we will retry healing this drive again")
+
 func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint) error {
 	poolIdx, setIdx := endpoint.PoolIdx, endpoint.SetIdx
 	disk := getStorageViaEndpoint(endpoint)
@@ -389,6 +393,17 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		return fmt.Errorf("Unexpected error disk must be initialized by now after formatting: %s", endpoint)
 	}
 
+	_, err := disk.DiskInfo(ctx, DiskInfoOptions{})
+	if err != nil {
+		if errors.Is(err, errDriveIsRoot) {
+			// This is a root drive, ignore and move on
+			return nil
+		}
+		if !errors.Is(err, errUnformattedDisk) {
+			return err
+		}
+	}
+
 	// Prevent parallel erasure set healing
 	locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%d/%d", poolIdx, setIdx))
 	lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
@@ -451,8 +466,27 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		return err
 	}
 
-	healingLogEvent(ctx, "Healing of drive '%s' is finished (healed: %d, skipped: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
+	// if objects have failed healing, we attempt a retry to heal the drive upto 3 times before giving up.
+	if tracker.ItemsFailed > 0 && tracker.RetryAttempts < 4 {
+		tracker.RetryAttempts++
+		bugLogIf(ctx, tracker.update(ctx))
 
+		healingLogEvent(ctx, "Healing of drive '%s' is incomplete, retrying %s time (healed: %d, skipped: %d, failed: %d).", disk,
+			humanize.Ordinal(int(tracker.RetryAttempts)), tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
+		return errRetryHealing
+	}
+
+	if tracker.ItemsFailed > 0 {
+		healingLogEvent(ctx, "Healing of drive '%s' is incomplete, retried %d times (healed: %d, skipped: %d, failed: %d).", disk,
+			tracker.RetryAttempts-1, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
+	} else {
+		if tracker.RetryAttempts > 0 {
+			healingLogEvent(ctx, "Healing of drive '%s' is complete, retried %d times (healed: %d, skipped: %d).", disk,
+				tracker.RetryAttempts-1, tracker.ItemsHealed, tracker.ItemsSkipped)
+		} else {
+			healingLogEvent(ctx, "Healing of drive '%s' is finished (healed: %d, skipped: %d).", disk, tracker.ItemsHealed, tracker.ItemsSkipped)
+		}
+	}
 	if serverDebugLog {
 		tracker.printTo(os.Stdout)
 		fmt.Printf("\n")
@@ -524,7 +558,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
 					if err := healFreshDisk(ctx, z, disk); err != nil {
 						globalBackgroundHealState.setDiskHealingStatus(disk, false)
 						timedout := OperationTimedOut{}
-						if !errors.Is(err, context.Canceled) && !errors.As(err, &timedout) {
+						if !errors.Is(err, context.Canceled) && !errors.As(err, &timedout) && !errors.Is(err, errRetryHealing) {
 							printEndpointError(disk, err, false)
 						}
 						return

cmd/background-newdisks-heal-ops_gen.go

@@ -200,6 +200,12 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "BytesSkipped")
 				return
 			}
+		case "RetryAttempts":
+			z.RetryAttempts, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "RetryAttempts")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -213,9 +219,9 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 25
+	// map header, size 26
 	// write "ID"
-	err = en.Append(0xde, 0x0, 0x19, 0xa2, 0x49, 0x44)
+	err = en.Append(0xde, 0x0, 0x1a, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -478,15 +484,25 @@ func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "BytesSkipped")
 		return
 	}
+	// write "RetryAttempts"
+	err = en.Append(0xad, 0x52, 0x65, 0x74, 0x72, 0x79, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.RetryAttempts)
+	if err != nil {
+		err = msgp.WrapError(err, "RetryAttempts")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 25
+	// map header, size 26
 	// string "ID"
-	o = append(o, 0xde, 0x0, 0x19, 0xa2, 0x49, 0x44)
+	o = append(o, 0xde, 0x0, 0x1a, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "PoolIndex"
 	o = append(o, 0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78)
@@ -566,6 +582,9 @@ func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "BytesSkipped"
 	o = append(o, 0xac, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
 	o = msgp.AppendUint64(o, z.BytesSkipped)
+	// string "RetryAttempts"
+	o = append(o, 0xad, 0x52, 0x65, 0x74, 0x72, 0x79, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
+	o = msgp.AppendUint64(o, z.RetryAttempts)
 	return
 }
 
@@ -763,6 +782,12 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "BytesSkipped")
 				return
 			}
+		case "RetryAttempts":
+			z.RetryAttempts, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "RetryAttempts")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -785,6 +810,6 @@ func (z *healingTracker) Msgsize() (s int) {
 	for za0002 := range z.HealedBuckets {
 		s += msgp.StringPrefixSize + len(z.HealedBuckets[za0002])
 	}
-	s += 7 + msgp.StringPrefixSize + len(z.HealID) + 13 + msgp.Uint64Size + 13 + msgp.Uint64Size
+	s += 7 + msgp.StringPrefixSize + len(z.HealID) + 13 + msgp.Uint64Size + 13 + msgp.Uint64Size + 14 + msgp.Uint64Size
 	return
 }

cmd/batch-expire.go

@@ -36,6 +36,7 @@ import (
 	"github.com/minio/pkg/v3/env"
 	"github.com/minio/pkg/v3/wildcard"
 	"github.com/minio/pkg/v3/workers"
+	"github.com/minio/pkg/v3/xtime"
 	"gopkg.in/yaml.v3"
 )
 
@@ -116,7 +117,7 @@ func (p BatchJobExpirePurge) Validate() error {
 // BatchJobExpireFilter holds all the filters currently supported for batch replication
 type BatchJobExpireFilter struct {
 	line, col     int
-	OlderThan     time.Duration       `yaml:"olderThan,omitempty" json:"olderThan"`
+	OlderThan     xtime.Duration      `yaml:"olderThan,omitempty" json:"olderThan"`
 	CreatedBefore *time.Time          `yaml:"createdBefore,omitempty" json:"createdBefore"`
 	Tags          []BatchJobKV        `yaml:"tags,omitempty" json:"tags"`
 	Metadata      []BatchJobKV        `yaml:"metadata,omitempty" json:"metadata"`
@@ -162,7 +163,7 @@ func (ef BatchJobExpireFilter) Matches(obj ObjectInfo, now time.Time) bool {
 	if len(ef.Name) > 0 && !wildcard.Match(ef.Name, obj.Name) {
 		return false
 	}
-	if ef.OlderThan > 0 && now.Sub(obj.ModTime) <= ef.OlderThan {
+	if ef.OlderThan > 0 && now.Sub(obj.ModTime) <= ef.OlderThan.D() {
 		return false
 	}
 
@@ -514,7 +515,7 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 		JobType:   string(job.Type()),
 		StartTime: job.Started,
 	}
-	if err := ri.load(ctx, api, job); err != nil {
+	if err := ri.loadOrInit(ctx, api, job); err != nil {
 		return err
 	}
 
@@ -552,22 +553,25 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 	go func() {
 		saveTicker := time.NewTicker(10 * time.Second)
 		defer saveTicker.Stop()
-		for {
+		quit := false
+		after := time.Minute
+		for !quit {
 			select {
 			case <-saveTicker.C:
-				// persist in-memory state to disk after every 10secs.
-				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
-
 			case <-ctx.Done():
-				// persist in-memory state immediately before exiting due to context cancellation.
-				batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
-				return
-
+				quit = true
 			case <-saverQuitCh:
-				// persist in-memory state immediately to disk.
-				batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
-				return
+				quit = true
 			}
+
+			if quit {
+				// save immediately if we are quitting
+				after = 0
+			}
+
+			ctx, cancel := context.WithTimeout(GlobalContext, 30*time.Second) // independent context
+			batchLogIf(ctx, ri.updateAfter(ctx, api, after, job))
+			cancel()
 		}
 	}()
 
@@ -584,7 +588,7 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 		versionsCount int
 		toDel         []expireObjInfo
 	)
-	failed := true
+	failed := false
 	for result := range results {
 		if result.Err != nil {
 			failed = true

cmd/batch-expire_gen.go

@@ -306,7 +306,7 @@ func (z *BatchJobExpireFilter) DecodeMsg(dc *msgp.Reader) (err error) {
 		}
 		switch msgp.UnsafeString(field) {
 		case "OlderThan":
-			z.OlderThan, err = dc.ReadDuration()
+			err = z.OlderThan.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "OlderThan")
 				return
@@ -433,7 +433,7 @@ func (z *BatchJobExpireFilter) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteDuration(z.OlderThan)
+	err = z.OlderThan.EncodeMsg(en)
 	if err != nil {
 		err = msgp.WrapError(err, "OlderThan")
 		return
@@ -544,7 +544,11 @@ func (z *BatchJobExpireFilter) MarshalMsg(b []byte) (o []byte, err error) {
 	// map header, size 8
 	// string "OlderThan"
 	o = append(o, 0x88, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
-	o = msgp.AppendDuration(o, z.OlderThan)
+	o, err = z.OlderThan.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "OlderThan")
+		return
+	}
 	// string "CreatedBefore"
 	o = append(o, 0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65)
 	if z.CreatedBefore == nil {
@@ -613,7 +617,7 @@ func (z *BatchJobExpireFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		}
 		switch msgp.UnsafeString(field) {
 		case "OlderThan":
-			z.OlderThan, bts, err = msgp.ReadDurationBytes(bts)
+			bts, err = z.OlderThan.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "OlderThan")
 				return
@@ -734,7 +738,7 @@ func (z *BatchJobExpireFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobExpireFilter) Msgsize() (s int) {
-	s = 1 + 10 + msgp.DurationSize + 14
+	s = 1 + 10 + z.OlderThan.Msgsize() + 14
 	if z.CreatedBefore == nil {
 		s += msgp.NilSize
 	} else {

cmd/batch-expire_test.go

@@ -20,7 +20,7 @@ package cmd
 import (
 	"testing"
 
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 func TestParseBatchJobExpire(t *testing.T) {
@@ -32,7 +32,7 @@ expire: # Expire objects that match a condition
   rules:
     - type: object  # regular objects with zero or more older versions
       name: NAME # match object names that satisfy the wildcard expression.
-      olderThan: 70h # match objects older than this value
+      olderThan: 7d10h # match objects older than this value
       createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
       tags:
         - key: name
@@ -64,7 +64,7 @@ expire: # Expire objects that match a condition
     delay: 500ms # least amount of delay between each retry
 `
 	var job BatchJobRequest
-	err := yaml.UnmarshalStrict([]byte(expireYaml), &job)
+	err := yaml.Unmarshal([]byte(expireYaml), &job)
 	if err != nil {
 		t.Fatal("Failed to parse batch-job-expire yaml", err)
 	}

cmd/batch-handlers.go

@@ -28,6 +28,7 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
+	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
@@ -57,6 +58,11 @@ import (
 
 var globalBatchConfig batch.Config
 
+const (
+	// Keep the completed/failed job stats 3 days before removing it
+	oldJobsExpiration = 3 * 24 * time.Hour
+)
+
 // BatchJobRequest this is an internal data structure not for external consumption.
 type BatchJobRequest struct {
 	ID        string               `yaml:"-" json:"name"`
@@ -262,7 +268,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 		JobType:   string(job.Type()),
 		StartTime: job.Started,
 	}
-	if err := ri.load(ctx, api, job); err != nil {
+	if err := ri.loadOrInit(ctx, api, job); err != nil {
 		return err
 	}
 	if ri.Complete {
@@ -281,12 +287,12 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	isStorageClassOnly := len(r.Flags.Filter.Metadata) == 1 && strings.EqualFold(r.Flags.Filter.Metadata[0].Key, xhttp.AmzStorageClass)
 
 	skip := func(oi ObjectInfo) (ok bool) {
-		if r.Flags.Filter.OlderThan > 0 && time.Since(oi.ModTime) < r.Flags.Filter.OlderThan {
+		if r.Flags.Filter.OlderThan > 0 && time.Since(oi.ModTime) < r.Flags.Filter.OlderThan.D() {
 			// skip all objects that are newer than specified older duration
 			return true
 		}
 
-		if r.Flags.Filter.NewerThan > 0 && time.Since(oi.ModTime) >= r.Flags.Filter.NewerThan {
+		if r.Flags.Filter.NewerThan > 0 && time.Since(oi.ModTime) >= r.Flags.Filter.NewerThan.D() {
 			// skip all objects that are older than specified newer duration
 			return true
 		}
@@ -534,7 +540,7 @@ func toObjectInfo(bucket, object string, objInfo miniogo.ObjectInfo) ObjectInfo
 	return oi
 }
 
-func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo) error {
+func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo, prefix string) error {
 	input := make(chan minio.SnowballObject, 1)
 	opts := minio.SnowballOptions{
 		Opts:     minio.PutObjectOptions{},
@@ -556,6 +562,10 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
 				continue
 			}
 
+			if prefix != "" {
+				entry.Name = pathJoin(prefix, entry.Name)
+			}
+
 			snowballObj := minio.SnowballObject{
 				// Create path to store objects within the bucket.
 				Key:       entry.Name,
@@ -719,63 +729,85 @@ const (
 
 	batchReplJobAPIVersion        = "v1"
 	batchReplJobDefaultRetries    = 3
-	batchReplJobDefaultRetryDelay = 250 * time.Millisecond
+	batchReplJobDefaultRetryDelay = time.Second
 )
 
-func getJobReportPath(job BatchJobRequest) string {
-	var fileName string
-	switch {
-	case job.Replicate != nil:
-		fileName = batchReplName
-	case job.KeyRotate != nil:
-		fileName = batchKeyRotationName
-	case job.Expire != nil:
-		fileName = batchExpireName
-	}
-	return pathJoin(batchJobReportsPrefix, job.ID, fileName)
-}
-
 func getJobPath(job BatchJobRequest) string {
 	return pathJoin(batchJobPrefix, job.ID)
 }
 
+func (ri *batchJobInfo) getJobReportPath() (string, error) {
+	var fileName string
+	switch madmin.BatchJobType(ri.JobType) {
+	case madmin.BatchJobReplicate:
+		fileName = batchReplName
+	case madmin.BatchJobKeyRotate:
+		fileName = batchKeyRotationName
+	case madmin.BatchJobExpire:
+		fileName = batchExpireName
+	default:
+		return "", fmt.Errorf("unknown job type: %v", ri.JobType)
+	}
+	return pathJoin(batchJobReportsPrefix, ri.JobID, fileName), nil
+}
+
+func (ri *batchJobInfo) loadOrInit(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+	err := ri.load(ctx, api, job)
+	if errors.Is(err, errNoSuchJob) {
+		switch {
+		case job.Replicate != nil:
+			ri.Version = batchReplVersionV1
+			ri.RetryAttempts = batchReplJobDefaultRetries
+			if job.Replicate.Flags.Retry.Attempts > 0 {
+				ri.RetryAttempts = job.Replicate.Flags.Retry.Attempts
+			}
+		case job.KeyRotate != nil:
+			ri.Version = batchKeyRotateVersionV1
+			ri.RetryAttempts = batchKeyRotateJobDefaultRetries
+			if job.KeyRotate.Flags.Retry.Attempts > 0 {
+				ri.RetryAttempts = job.KeyRotate.Flags.Retry.Attempts
+			}
+		case job.Expire != nil:
+			ri.Version = batchExpireVersionV1
+			ri.RetryAttempts = batchExpireJobDefaultRetries
+			if job.Expire.Retry.Attempts > 0 {
+				ri.RetryAttempts = job.Expire.Retry.Attempts
+			}
+		}
+		return nil
+	}
+	return err
+}
+
 func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+	path, err := job.getJobReportPath()
+	if err != nil {
+		batchLogIf(ctx, err)
+		return err
+	}
+	return ri.loadByPath(ctx, api, path)
+}
+
+func (ri *batchJobInfo) loadByPath(ctx context.Context, api ObjectLayer, path string) error {
 	var format, version uint16
-	switch {
-	case job.Replicate != nil:
+	switch filepath.Base(path) {
+	case batchReplName:
 		version = batchReplVersionV1
 		format = batchReplFormat
-	case job.KeyRotate != nil:
+	case batchKeyRotationName:
 		version = batchKeyRotateVersionV1
 		format = batchKeyRotationFormat
-	case job.Expire != nil:
+	case batchExpireName:
 		version = batchExpireVersionV1
 		format = batchExpireFormat
 	default:
 		return errors.New("no supported batch job request specified")
 	}
-	data, err := readConfig(ctx, api, getJobReportPath(job))
+
+	data, err := readConfig(ctx, api, path)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) || isErrObjectNotFound(err) {
-			ri.Version = int(version)
-			switch {
-			case job.Replicate != nil:
-				ri.RetryAttempts = batchReplJobDefaultRetries
-				if job.Replicate.Flags.Retry.Attempts > 0 {
-					ri.RetryAttempts = job.Replicate.Flags.Retry.Attempts
-				}
-			case job.KeyRotate != nil:
-				ri.RetryAttempts = batchKeyRotateJobDefaultRetries
-				if job.KeyRotate.Flags.Retry.Attempts > 0 {
-					ri.RetryAttempts = job.KeyRotate.Flags.Retry.Attempts
-				}
-			case job.Expire != nil:
-				ri.RetryAttempts = batchExpireJobDefaultRetries
-				if job.Expire.Retry.Attempts > 0 {
-					ri.RetryAttempts = job.Expire.Retry.Attempts
-				}
-			}
-			return nil
+			return errNoSuchJob
 		}
 		return err
 	}
@@ -919,7 +951,12 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 		if err != nil {
 			return err
 		}
-		return saveConfig(ctx, api, getJobReportPath(job), buf)
+		path, err := ri.getJobReportPath()
+		if err != nil {
+			batchLogIf(ctx, err)
+			return err
+		}
+		return saveConfig(ctx, api, path, buf)
 	}
 	ri.mu.Unlock()
 	return nil
@@ -944,8 +981,10 @@ func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo,
 	ri.mu.Lock()
 	defer ri.mu.Unlock()
 
-	ri.Bucket = bucket
-	ri.Object = info.Name
+	if success {
+		ri.Bucket = bucket
+		ri.Object = info.Name
+	}
 	ri.countItem(info.Size, info.DeleteMarker, success, attempt)
 }
 
@@ -971,7 +1010,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		JobType:   string(job.Type()),
 		StartTime: job.Started,
 	}
-	if err := ri.load(ctx, api, job); err != nil {
+	if err := ri.loadOrInit(ctx, api, job); err != nil {
 		return err
 	}
 	if ri.Complete {
@@ -981,18 +1020,18 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	lastObject := ri.Object
 
 	delay := job.Replicate.Flags.Retry.Delay
-	if delay == 0 {
+	if delay < time.Second {
 		delay = batchReplJobDefaultRetryDelay
 	}
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	selectObj := func(info FileInfo) (ok bool) {
-		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
+		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan.D() {
 			// skip all objects that are newer than specified older duration
 			return false
 		}
 
-		if r.Flags.Filter.NewerThan > 0 && time.Since(info.ModTime) >= r.Flags.Filter.NewerThan {
+		if r.Flags.Filter.NewerThan > 0 && time.Since(info.ModTime) >= r.Flags.Filter.NewerThan.D() {
 			// skip all objects that are older than specified newer duration
 			return false
 		}
@@ -1071,86 +1110,84 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 	c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
 
-	var (
-		walkCh = make(chan itemOrErr[ObjectInfo], 100)
-		slowCh = make(chan itemOrErr[ObjectInfo], 100)
-	)
-
-	if !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
-		go func() {
-			// Snowball currently needs the high level minio-go Client, not the Core one
-			cl, err := miniogo.New(u.Host, &miniogo.Options{
-				Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
-				Secure:       u.Scheme == "https",
-				Transport:    getRemoteInstanceTransport(),
-				BucketLookup: lookupStyle(r.Target.Path),
-			})
-			if err != nil {
-				batchLogIf(ctx, err)
-				return
-			}
-
-			// Already validated before arriving here
-			smallerThan, _ := humanize.ParseBytes(*r.Source.Snowball.SmallerThan)
-
-			batch := make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
-			writeFn := func(batch []ObjectInfo) {
-				if len(batch) > 0 {
-					if err := r.writeAsArchive(ctx, api, cl, batch); err != nil {
-						batchLogIf(ctx, err)
-						for _, b := range batch {
-							slowCh <- itemOrErr[ObjectInfo]{Item: b}
-						}
-					} else {
-						ri.trackCurrentBucketBatch(r.Source.Bucket, batch)
-						globalBatchJobsMetrics.save(job.ID, ri)
-						// persist in-memory state to disk after every 10secs.
-						batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
-					}
-				}
-			}
-			for obj := range walkCh {
-				if obj.Item.DeleteMarker || !obj.Item.VersionPurgeStatus.Empty() || obj.Item.Size >= int64(smallerThan) {
-					slowCh <- obj
-					continue
-				}
-
-				batch = append(batch, obj.Item)
-
-				if len(batch) < *r.Source.Snowball.Batch {
-					continue
-				}
-				writeFn(batch)
-				batch = batch[:0]
-			}
-			writeFn(batch)
-			xioutil.SafeClose(slowCh)
-		}()
-	} else {
-		slowCh = walkCh
-	}
-
-	workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_REPLICATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
-	if err != nil {
-		return err
-	}
-
-	wk, err := workers.New(workerSize)
-	if err != nil {
-		// invalid worker size.
-		return err
-	}
-
-	walkQuorum := env.Get("_MINIO_BATCH_REPLICATION_WALK_QUORUM", "strict")
-	if walkQuorum == "" {
-		walkQuorum = "strict"
-	}
-
 	retryAttempts := ri.RetryAttempts
 	retry := false
 	for attempts := 1; attempts <= retryAttempts; attempts++ {
 		attempts := attempts
+		var (
+			walkCh = make(chan itemOrErr[ObjectInfo], 100)
+			slowCh = make(chan itemOrErr[ObjectInfo], 100)
+		)
 
+		if r.Source.Snowball.Disable != nil && !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
+			go func() {
+				// Snowball currently needs the high level minio-go Client, not the Core one
+				cl, err := miniogo.New(u.Host, &miniogo.Options{
+					Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
+					Secure:       u.Scheme == "https",
+					Transport:    getRemoteInstanceTransport(),
+					BucketLookup: lookupStyle(r.Target.Path),
+				})
+				if err != nil {
+					batchLogOnceIf(ctx, err, job.ID+"miniogo.New")
+					return
+				}
+
+				// Already validated before arriving here
+				smallerThan, _ := humanize.ParseBytes(*r.Source.Snowball.SmallerThan)
+
+				batch := make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
+				writeFn := func(batch []ObjectInfo) {
+					if len(batch) > 0 {
+						if err := r.writeAsArchive(ctx, api, cl, batch, r.Target.Prefix); err != nil {
+							batchLogOnceIf(ctx, err, job.ID+"writeAsArchive")
+							for _, b := range batch {
+								slowCh <- itemOrErr[ObjectInfo]{Item: b}
+							}
+						} else {
+							ri.trackCurrentBucketBatch(r.Source.Bucket, batch)
+							globalBatchJobsMetrics.save(job.ID, ri)
+							// persist in-memory state to disk after every 10secs.
+							batchLogOnceIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job), job.ID+"updateAfter")
+						}
+					}
+				}
+				for obj := range walkCh {
+					if obj.Item.DeleteMarker || !obj.Item.VersionPurgeStatus.Empty() || obj.Item.Size >= int64(smallerThan) {
+						slowCh <- obj
+						continue
+					}
+
+					batch = append(batch, obj.Item)
+
+					if len(batch) < *r.Source.Snowball.Batch {
+						continue
+					}
+					writeFn(batch)
+					batch = batch[:0]
+				}
+				writeFn(batch)
+				xioutil.SafeClose(slowCh)
+			}()
+		} else {
+			slowCh = walkCh
+		}
+
+		workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_REPLICATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
+		if err != nil {
+			return err
+		}
+
+		wk, err := workers.New(workerSize)
+		if err != nil {
+			// invalid worker size.
+			return err
+		}
+
+		walkQuorum := env.Get("_MINIO_BATCH_REPLICATION_WALK_QUORUM", "strict")
+		if walkQuorum == "" {
+			walkQuorum = "strict"
+		}
 		ctx, cancel := context.WithCancel(ctx)
 		// one of source/target is s3, skip delete marker and all versions under the same object name.
 		s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
@@ -1171,7 +1208,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		for res := range slowCh {
 			if res.Err != nil {
 				ri.Failed = true
-				batchLogIf(ctx, res.Err)
+				batchLogOnceIf(ctx, res.Err, job.ID+"res.Err")
 				continue
 			}
 			result := res.Item
@@ -1198,7 +1235,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 						return
 					}
 					stopFn(result, err)
-					batchLogIf(ctx, err)
+					batchLogOnceIf(ctx, err, job.ID+"ReplicateToTarget")
 					success = false
 				} else {
 					stopFn(result, nil)
@@ -1206,7 +1243,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				ri.trackCurrentBucketObject(r.Source.Bucket, result, success, attempts)
 				globalBatchJobsMetrics.save(job.ID, ri)
 				// persist in-memory state to disk after every 10secs.
-				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+				batchLogOnceIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job), job.ID+"updateAfter2")
 
 				if wait := globalBatchConfig.ReplicationWait(); wait > 0 {
 					time.Sleep(wait)
@@ -1221,10 +1258,10 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 		globalBatchJobsMetrics.save(job.ID, ri)
 		// persist in-memory state to disk.
-		batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+		batchLogOnceIf(ctx, ri.updateAfter(ctx, api, 0, job), job.ID+"updateAfter3")
 
 		if err := r.Notify(ctx, ri); err != nil {
-			batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
+			batchLogOnceIf(ctx, fmt.Errorf("unable to notify %v", err), job.ID+"notify")
 		}
 
 		cancel()
@@ -1436,10 +1473,24 @@ func (j BatchJobRequest) Validate(ctx context.Context, o ObjectLayer) error {
 }
 
 func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) {
-	deleteConfig(ctx, api, getJobReportPath(j))
 	deleteConfig(ctx, api, getJobPath(j))
 }
 
+func (j BatchJobRequest) getJobReportPath() (string, error) {
+	var fileName string
+	switch {
+	case j.Replicate != nil:
+		fileName = batchReplName
+	case j.KeyRotate != nil:
+		fileName = batchKeyRotationName
+	case j.Expire != nil:
+		fileName = batchExpireName
+	default:
+		return "", errors.New("unknown job type")
+	}
+	return pathJoin(batchJobReportsPrefix, j.ID, fileName), nil
+}
+
 func (j *BatchJobRequest) save(ctx context.Context, api ObjectLayer) error {
 	if j.Replicate == nil && j.KeyRotate == nil && j.Expire == nil {
 		return errInvalidArgument
@@ -1520,6 +1571,9 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, result.Err), r.URL)
 			return
 		}
+		if strings.HasPrefix(result.Item.Name, batchJobReportsPrefix+slashSeparator) {
+			continue
+		}
 		req := &BatchJobRequest{}
 		if err := req.load(ctx, objectAPI, result.Item.Name); err != nil {
 			if !errors.Is(err, errNoSuchJob) {
@@ -1542,6 +1596,55 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
 	batchLogIf(ctx, json.NewEncoder(w).Encode(&listResult))
 }
 
+// BatchJobStatus - returns the status of a batch job saved in the disk
+func (a adminAPIHandlers) BatchJobStatus(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListBatchJobsAction)
+	if objectAPI == nil {
+		return
+	}
+
+	jobID := r.Form.Get("jobId")
+	if jobID == "" {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL)
+		return
+	}
+
+	req := BatchJobRequest{ID: jobID}
+	if i := strings.Index(jobID, "-"); i > 0 {
+		switch madmin.BatchJobType(jobID[:i]) {
+		case madmin.BatchJobReplicate:
+			req.Replicate = &BatchJobReplicateV1{}
+		case madmin.BatchJobKeyRotate:
+			req.KeyRotate = &BatchJobKeyRotateV1{}
+		case madmin.BatchJobExpire:
+			req.Expire = &BatchJobExpire{}
+		default:
+			writeErrorResponseJSON(ctx, w, toAPIError(ctx, errors.New("job ID format unrecognized")), r.URL)
+			return
+		}
+	}
+
+	ri := &batchJobInfo{}
+	if err := ri.load(ctx, objectAPI, req); err != nil {
+		if !errors.Is(err, errNoSuchJob) {
+			batchLogIf(ctx, err)
+		}
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	buf, err := json.Marshal(madmin.BatchJobStatus{LastMetric: ri.metric()})
+	if err != nil {
+		batchLogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	w.Write(buf)
+}
+
 var errNoSuchJob = errors.New("no such job")
 
 // DescribeBatchJob returns the currently active batch job definition
@@ -1633,7 +1736,7 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	job.ID = fmt.Sprintf("%s%s%d", shortuuid.New(), getKeySeparator(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
+	job.ID = fmt.Sprintf("%s-%s%s%d", job.Type(), shortuuid.New(), getKeySeparator(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
 	job.User = user
 	job.Started = time.Now()
 
@@ -1721,11 +1824,60 @@ func newBatchJobPool(ctx context.Context, o ObjectLayer, workers int) *BatchJobP
 		jobCancelers: make(map[string]context.CancelFunc),
 	}
 	jpool.ResizeWorkers(workers)
-	jpool.resume()
+
+	randomWait := func() time.Duration {
+		// randomWait depends on the number of nodes to avoid triggering resume and cleanups at the same time.
+		return time.Duration(rand.Float64() * float64(time.Duration(globalEndpoints.NEndpoints())*time.Hour))
+	}
+
+	go func() {
+		jpool.resume(randomWait)
+		jpool.cleanupReports(randomWait)
+	}()
+
 	return jpool
 }
 
-func (j *BatchJobPool) resume() {
+func (j *BatchJobPool) cleanupReports(randomWait func() time.Duration) {
+	t := time.NewTimer(randomWait())
+	defer t.Stop()
+
+	for {
+		select {
+		case <-GlobalContext.Done():
+			return
+		case <-t.C:
+			results := make(chan itemOrErr[ObjectInfo], 100)
+			ctx, cancel := context.WithCancel(j.ctx)
+			defer cancel()
+			if err := j.objLayer.Walk(ctx, minioMetaBucket, batchJobReportsPrefix, results, WalkOptions{}); err != nil {
+				batchLogIf(j.ctx, err)
+				t.Reset(randomWait())
+				continue
+			}
+			for result := range results {
+				if result.Err != nil {
+					batchLogIf(j.ctx, result.Err)
+					continue
+				}
+				ri := &batchJobInfo{}
+				if err := ri.loadByPath(ctx, j.objLayer, result.Item.Name); err != nil {
+					batchLogIf(ctx, err)
+					continue
+				}
+				if (ri.Complete || ri.Failed) && time.Since(ri.LastUpdate) > oldJobsExpiration {
+					deleteConfig(ctx, j.objLayer, result.Item.Name)
+				}
+			}
+
+			t.Reset(randomWait())
+		}
+	}
+}
+
+func (j *BatchJobPool) resume(randomWait func() time.Duration) {
+	time.Sleep(randomWait())
+
 	results := make(chan itemOrErr[ObjectInfo], 100)
 	ctx, cancel := context.WithCancel(j.ctx)
 	defer cancel()
@@ -1738,6 +1890,9 @@ func (j *BatchJobPool) resume() {
 			batchLogIf(j.ctx, result.Err)
 			continue
 		}
+		if strings.HasPrefix(result.Item.Name, batchJobReportsPrefix+slashSeparator) {
+			continue
+		}
 		// ignore batch-replicate.bin and batch-rotate.bin entries
 		if strings.HasSuffix(result.Item.Name, slashSeparator) {
 			continue
@@ -1988,7 +2143,7 @@ func (m *batchJobMetrics) purgeJobMetrics() {
 			var toDeleteJobMetrics []string
 			m.RLock()
 			for id, metrics := range m.metrics {
-				if time.Since(metrics.LastUpdate) > 24*time.Hour && (metrics.Complete || metrics.Failed) {
+				if time.Since(metrics.LastUpdate) > oldJobsExpiration && (metrics.Complete || metrics.Failed) {
 					toDeleteJobMetrics = append(toDeleteJobMetrics, id)
 				}
 			}

cmd/batch-replicate.go

@@ -21,8 +21,8 @@ import (
 	"time"
 
 	miniogo "github.com/minio/minio-go/v7"
-
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/pkg/v3/xtime"
 )
 
 //go:generate msgp -file $GOFILE
@@ -65,12 +65,12 @@ import (
 
 // BatchReplicateFilter holds all the filters currently supported for batch replication
 type BatchReplicateFilter struct {
-	NewerThan     time.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
-	OlderThan     time.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
-	CreatedAfter  time.Time     `yaml:"createdAfter,omitempty" json:"createdAfter"`
-	CreatedBefore time.Time     `yaml:"createdBefore,omitempty" json:"createdBefore"`
-	Tags          []BatchJobKV  `yaml:"tags,omitempty" json:"tags"`
-	Metadata      []BatchJobKV  `yaml:"metadata,omitempty" json:"metadata"`
+	NewerThan     xtime.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
+	OlderThan     xtime.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
+	CreatedAfter  time.Time      `yaml:"createdAfter,omitempty" json:"createdAfter"`
+	CreatedBefore time.Time      `yaml:"createdBefore,omitempty" json:"createdBefore"`
+	Tags          []BatchJobKV   `yaml:"tags,omitempty" json:"tags"`
+	Metadata      []BatchJobKV   `yaml:"metadata,omitempty" json:"metadata"`
 }
 
 // BatchJobReplicateFlags various configurations for replication job definition currently includes

cmd/batch-replicate_gen.go

@@ -1409,13 +1409,13 @@ func (z *BatchReplicateFilter) DecodeMsg(dc *msgp.Reader) (err error) {
 		}
 		switch msgp.UnsafeString(field) {
 		case "NewerThan":
-			z.NewerThan, err = dc.ReadDuration()
+			err = z.NewerThan.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "NewerThan")
 				return
 			}
 		case "OlderThan":
-			z.OlderThan, err = dc.ReadDuration()
+			err = z.OlderThan.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "OlderThan")
 				return
@@ -1489,7 +1489,7 @@ func (z *BatchReplicateFilter) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteDuration(z.NewerThan)
+	err = z.NewerThan.EncodeMsg(en)
 	if err != nil {
 		err = msgp.WrapError(err, "NewerThan")
 		return
@@ -1499,7 +1499,7 @@ func (z *BatchReplicateFilter) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteDuration(z.OlderThan)
+	err = z.OlderThan.EncodeMsg(en)
 	if err != nil {
 		err = msgp.WrapError(err, "OlderThan")
 		return
@@ -1567,10 +1567,18 @@ func (z *BatchReplicateFilter) MarshalMsg(b []byte) (o []byte, err error) {
 	// map header, size 6
 	// string "NewerThan"
 	o = append(o, 0x86, 0xa9, 0x4e, 0x65, 0x77, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
-	o = msgp.AppendDuration(o, z.NewerThan)
+	o, err = z.NewerThan.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "NewerThan")
+		return
+	}
 	// string "OlderThan"
 	o = append(o, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
-	o = msgp.AppendDuration(o, z.OlderThan)
+	o, err = z.OlderThan.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "OlderThan")
+		return
+	}
 	// string "CreatedAfter"
 	o = append(o, 0xac, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x66, 0x74, 0x65, 0x72)
 	o = msgp.AppendTime(o, z.CreatedAfter)
@@ -1619,13 +1627,13 @@ func (z *BatchReplicateFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		}
 		switch msgp.UnsafeString(field) {
 		case "NewerThan":
-			z.NewerThan, bts, err = msgp.ReadDurationBytes(bts)
+			bts, err = z.NewerThan.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "NewerThan")
 				return
 			}
 		case "OlderThan":
-			z.OlderThan, bts, err = msgp.ReadDurationBytes(bts)
+			bts, err = z.OlderThan.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "OlderThan")
 				return
@@ -1694,7 +1702,7 @@ func (z *BatchReplicateFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchReplicateFilter) Msgsize() (s int) {
-	s = 1 + 10 + msgp.DurationSize + 10 + msgp.DurationSize + 13 + msgp.TimeSize + 14 + msgp.TimeSize + 5 + msgp.ArrayHeaderSize
+	s = 1 + 10 + z.NewerThan.Msgsize() + 10 + z.OlderThan.Msgsize() + 13 + msgp.TimeSize + 14 + msgp.TimeSize + 5 + msgp.ArrayHeaderSize
 	for za0001 := range z.Tags {
 		s += z.Tags[za0001].Msgsize()
 	}

cmd/batch-replicate_test.go

@@ -0,0 +1,100 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+func TestParseBatchJobReplicate(t *testing.T) {
+	replicateYaml := `
+replicate:
+  apiVersion: v1
+  # source of the objects to be replicated
+  source:
+    type: minio # valid values are "s3" or "minio"
+    bucket: mytest
+    prefix: object-prefix1 # 'PREFIX' is optional
+    # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted
+    # Either the 'source' or 'remote' *must* be the "local" deployment
+#    endpoint: "http://127.0.0.1:9000"
+#    # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto"
+#    credentials:
+#      accessKey: minioadmin # Required
+#      secretKey: minioadmin # Required
+#    # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used
+    snowball: # automatically activated if the source is local
+      disable: true # optionally turn-off snowball archive transfer
+#      batch: 100 # upto this many objects per archive
+#      inmemory: true # indicates if the archive must be staged locally or in-memory
+#      compress: false # S2/Snappy compressed archive
+#      smallerThan: 5MiB # create archive for all objects smaller than 5MiB
+#      skipErrs: false # skips any source side read() errors
+
+  # target where the objects must be replicated
+  target:
+    type: minio # valid values are "s3" or "minio"
+    bucket: mytest
+    prefix: stage # 'PREFIX' is optional
+    # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted
+
+    # Either the 'source' or 'remote' *must* be the "local" deployment
+    endpoint: "http://127.0.0.1:9001"
+    # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto"
+    credentials:
+      accessKey: minioadmin
+      secretKey: minioadmin
+    # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used
+
+  # NOTE: All flags are optional
+  # - filtering criteria only applies for all source objects match the criteria
+  # - configurable notification endpoints
+  # - configurable retries for the job (each retry skips successfully previously replaced objects)
+  flags:
+    filter:
+      newerThan: "7d10h31s" # match objects newer than this value (e.g. 7d10h31s)
+      olderThan: "7d" # match objects older than this value (e.g. 7d10h31s)
+#      createdAfter: "date" # match objects created after "date"
+#      createdBefore: "date" # match objects created before "date"
+
+      ## NOTE: tags are not supported when "source" is remote.
+      tags:
+         - key: "name"
+           value: "pick*" # match objects with tag 'name', with all values starting with 'pick'
+
+      metadata:
+         - key: "content-type"
+           value: "image/*" # match objects with 'content-type', with all values starting with 'image/'
+
+#    notify:
+#      endpoint: "https://notify.endpoint" # notification endpoint to receive job status events
+#      token: "Bearer xxxxx" # optional authentication token for the notification endpoint
+#
+#    retry:
+#      attempts: 10 # number of retries for the job before giving up
+#      delay: "500ms" # least amount of delay between each retry
+
+`
+	var job BatchJobRequest
+	err := yaml.Unmarshal([]byte(replicateYaml), &job)
+	if err != nil {
+		t.Fatal("Failed to parse batch-job-replicate yaml", err)
+	}
+}

cmd/batch-rotate.go

@@ -257,7 +257,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		JobType:   string(job.Type()),
 		StartTime: job.Started,
 	}
-	if err := ri.load(ctx, api, job); err != nil {
+	if err := ri.loadOrInit(ctx, api, job); err != nil {
 		return err
 	}
 	if ri.Complete {
@@ -389,6 +389,17 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 					stopFn(result, err)
 					batchLogIf(ctx, err)
 					success = false
+					if attempts >= retryAttempts {
+						auditOptions := AuditLogOptions{
+							Event:     "KeyRotate",
+							APIName:   "StartBatchJob",
+							Bucket:    result.Bucket,
+							Object:    result.Name,
+							VersionID: result.VersionID,
+							Error:     err.Error(),
+						}
+						auditLogInternal(ctx, auditOptions)
+					}
 				} else {
 					stopFn(result, nil)
 				}

cmd/bootstrap-peer-server.go

@@ -19,9 +19,13 @@ package cmd
 
 import (
 	"context"
+	"crypto/md5"
+	"encoding/hex"
 	"errors"
 	"fmt"
+	"io"
 	"math/rand"
+	"os"
 	"reflect"
 	"strings"
 	"sync"
@@ -43,10 +47,15 @@ type ServerSystemConfig struct {
 	NEndpoints int
 	CmdLines   []string
 	MinioEnv   map[string]string
+	Checksum   string
 }
 
 // Diff - returns error on first difference found in two configs.
 func (s1 *ServerSystemConfig) Diff(s2 *ServerSystemConfig) error {
+	if s1.Checksum != s2.Checksum {
+		return fmt.Errorf("Expected MinIO binary checksum: %s, seen: %s", s1.Checksum, s2.Checksum)
+	}
+
 	ns1 := s1.NEndpoints
 	ns2 := s2.NEndpoints
 	if ns1 != ns2 {
@@ -82,7 +91,7 @@ func (s1 *ServerSystemConfig) Diff(s2 *ServerSystemConfig) error {
 			extra = append(extra, k)
 		}
 	}
-	msg := "Expected same MINIO_ environment variables and values across all servers: "
+	msg := "Expected MINIO_* environment name and values across all servers to be same: "
 	if len(missing) > 0 {
 		msg += fmt.Sprintf(`Missing environment values: %v. `, missing)
 	}
@@ -97,14 +106,17 @@ func (s1 *ServerSystemConfig) Diff(s2 *ServerSystemConfig) error {
 }
 
 var skipEnvs = map[string]struct{}{
-	"MINIO_OPTS":          {},
-	"MINIO_CERT_PASSWD":   {},
-	"MINIO_SERVER_DEBUG":  {},
-	"MINIO_DSYNC_TRACE":   {},
-	"MINIO_ROOT_USER":     {},
-	"MINIO_ROOT_PASSWORD": {},
-	"MINIO_ACCESS_KEY":    {},
-	"MINIO_SECRET_KEY":    {},
+	"MINIO_OPTS":                   {},
+	"MINIO_CERT_PASSWD":            {},
+	"MINIO_SERVER_DEBUG":           {},
+	"MINIO_DSYNC_TRACE":            {},
+	"MINIO_ROOT_USER":              {},
+	"MINIO_ROOT_PASSWORD":          {},
+	"MINIO_ACCESS_KEY":             {},
+	"MINIO_SECRET_KEY":             {},
+	"MINIO_OPERATOR_VERSION":       {},
+	"MINIO_VSPHERE_PLUGIN_VERSION": {},
+	"MINIO_CI_CD":                  {},
 }
 
 func getServerSystemCfg() *ServerSystemConfig {
@@ -120,7 +132,7 @@ func getServerSystemCfg() *ServerSystemConfig {
 		}
 		envValues[envK] = logger.HashString(env.Get(envK, ""))
 	}
-	scfg := &ServerSystemConfig{NEndpoints: globalEndpoints.NEndpoints(), MinioEnv: envValues}
+	scfg := &ServerSystemConfig{NEndpoints: globalEndpoints.NEndpoints(), MinioEnv: envValues, Checksum: binaryChecksum}
 	var cmdLines []string
 	for _, ep := range globalEndpoints {
 		cmdLines = append(cmdLines, ep.CmdLine)
@@ -167,6 +179,18 @@ func (client *bootstrapRESTClient) String() string {
 	return client.gridConn.String()
 }
 
+var binaryChecksum = getBinaryChecksum()
+
+func getBinaryChecksum() string {
+	mw := md5.New()
+	b, err := os.Open(os.Args[0])
+	if err == nil {
+		defer b.Close()
+		io.Copy(mw, b)
+	}
+	return hex.EncodeToString(mw.Sum(nil))
+}
+
 func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointServerPools, gm *grid.Manager) error {
 	srcCfg := getServerSystemCfg()
 	clnts := newBootstrapRESTClients(endpointServerPools, gm)
@@ -196,7 +220,7 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 				err := clnt.Verify(ctx, srcCfg)
 				mu.Lock()
 				if err != nil {
-					bootstrapTraceMsg(fmt.Sprintf("clnt.Verify: %v, endpoint: %s", err, clnt))
+					bootstrapTraceMsg(fmt.Sprintf("bootstrapVerify: %v, endpoint: %s", err, clnt))
 					if !isNetworkError(err) {
 						bootLogOnceIf(context.Background(), fmt.Errorf("%s has incorrect configuration: %w", clnt, err), "incorrect_"+clnt.String())
 						incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt, err))

cmd/bootstrap-peer-server_gen.go

@@ -79,6 +79,12 @@ func (z *ServerSystemConfig) DecodeMsg(dc *msgp.Reader) (err error) {
 				}
 				z.MinioEnv[za0002] = za0003
 			}
+		case "Checksum":
+			z.Checksum, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Checksum")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -92,9 +98,9 @@ func (z *ServerSystemConfig) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *ServerSystemConfig) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 3
+	// map header, size 4
 	// write "NEndpoints"
-	err = en.Append(0x83, 0xaa, 0x4e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73)
+	err = en.Append(0x84, 0xaa, 0x4e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73)
 	if err != nil {
 		return
 	}
@@ -142,15 +148,25 @@ func (z *ServerSystemConfig) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
+	// write "Checksum"
+	err = en.Append(0xa8, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Checksum)
+	if err != nil {
+		err = msgp.WrapError(err, "Checksum")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *ServerSystemConfig) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 3
+	// map header, size 4
 	// string "NEndpoints"
-	o = append(o, 0x83, 0xaa, 0x4e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73)
+	o = append(o, 0x84, 0xaa, 0x4e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73)
 	o = msgp.AppendInt(o, z.NEndpoints)
 	// string "CmdLines"
 	o = append(o, 0xa8, 0x43, 0x6d, 0x64, 0x4c, 0x69, 0x6e, 0x65, 0x73)
@@ -165,6 +181,9 @@ func (z *ServerSystemConfig) MarshalMsg(b []byte) (o []byte, err error) {
 		o = msgp.AppendString(o, za0002)
 		o = msgp.AppendString(o, za0003)
 	}
+	// string "Checksum"
+	o = append(o, 0xa8, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d)
+	o = msgp.AppendString(o, z.Checksum)
 	return
 }
 
@@ -241,6 +260,12 @@ func (z *ServerSystemConfig) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				}
 				z.MinioEnv[za0002] = za0003
 			}
+		case "Checksum":
+			z.Checksum, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Checksum")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -266,5 +291,6 @@ func (z *ServerSystemConfig) Msgsize() (s int) {
 			s += msgp.StringPrefixSize + len(za0002) + msgp.StringPrefixSize + len(za0003)
 		}
 	}
+	s += 9 + msgp.StringPrefixSize + len(z.Checksum)
 	return
 }

cmd/bucket-handlers.go

@@ -51,7 +51,6 @@ import (
 	sse "github.com/minio/minio/internal/bucket/encryption"
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
-	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
@@ -92,7 +91,7 @@ const (
 // -- If IP of the entry doesn't match, this means entry is
 //
 //	for another instance. Log an error to console.
-func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
+func initFederatorBackend(buckets []string, objLayer ObjectLayer) {
 	if len(buckets) == 0 {
 		return
 	}
@@ -113,10 +112,10 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	domainMissing := err == dns.ErrDomainMissing
 	if dnsBuckets != nil {
 		for _, bucket := range buckets {
-			bucketsSet.Add(bucket.Name)
-			r, ok := dnsBuckets[bucket.Name]
+			bucketsSet.Add(bucket)
+			r, ok := dnsBuckets[bucket]
 			if !ok {
-				bucketsToBeUpdated.Add(bucket.Name)
+				bucketsToBeUpdated.Add(bucket)
 				continue
 			}
 			if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() {
@@ -135,7 +134,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 				// but if we do see a difference with local domain IPs with
 				// hostSlice from etcd then we should update with newer
 				// domainIPs, we proceed to do that here.
-				bucketsToBeUpdated.Add(bucket.Name)
+				bucketsToBeUpdated.Add(bucket)
 				continue
 			}
 
@@ -144,7 +143,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 			// bucket names are globally unique in federation at a given
 			// path prefix, name collision is not allowed. We simply log
 			// an error and continue.
-			bucketsInConflict.Add(bucket.Name)
+			bucketsInConflict.Add(bucket)
 		}
 	}
 
@@ -1342,22 +1341,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 					continue
 				}
 
-				asize, err := objInfo.GetActualSize()
-				if err != nil {
-					asize = objInfo.Size
-				}
-
-				globalCacheConfig.Set(&cache.ObjectInfo{
-					Key:          objInfo.Name,
-					Bucket:       objInfo.Bucket,
-					ETag:         getDecryptedETag(formValues, objInfo, false),
-					ModTime:      objInfo.ModTime,
-					Expires:      objInfo.Expires.UTC().Format(http.TimeFormat),
-					CacheControl: objInfo.CacheControl,
-					Metadata:     cleanReservedKeys(objInfo.UserDefined),
-					Size:         asize,
-				})
-
 				fanOutResp = append(fanOutResp, minio.PutObjectFanOutResponse{
 					Key:          objInfo.Name,
 					ETag:         getDecryptedETag(formValues, objInfo, false),
@@ -1452,22 +1435,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		w.Header().Set(xhttp.Location, obj)
 	}
 
-	asize, err := objInfo.GetActualSize()
-	if err != nil {
-		asize = objInfo.Size
-	}
-
-	defer globalCacheConfig.Set(&cache.ObjectInfo{
-		Key:          objInfo.Name,
-		Bucket:       objInfo.Bucket,
-		ETag:         etag,
-		ModTime:      objInfo.ModTime,
-		Expires:      objInfo.ExpiresStr(),
-		CacheControl: objInfo.CacheControl,
-		Metadata:     cleanReservedKeys(objInfo.UserDefined),
-		Size:         asize,
-	})
-
 	// Notify object created event.
 	defer sendEvent(eventArgs{
 		EventName:    event.ObjectCreatedPost,

cmd/bucket-metadata-sys.go

@@ -392,9 +392,7 @@ func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket s
 		return nil, time.Time{}, BucketReplicationConfigNotFound{Bucket: bucket}
 	}
 	if reloaded {
-		globalBucketTargetSys.set(BucketInfo{
-			Name: bucket,
-		}, meta)
+		globalBucketTargetSys.set(bucket, meta)
 	}
 	return meta.replicationConfig, meta.ReplicationConfigUpdatedAt, nil
 }
@@ -413,9 +411,7 @@ func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.Buc
 		return nil, BucketRemoteTargetNotFound{Bucket: bucket}
 	}
 	if reloaded {
-		globalBucketTargetSys.set(BucketInfo{
-			Name: bucket,
-		}, meta)
+		globalBucketTargetSys.set(bucket, meta)
 	}
 	return meta.bucketTargetConfig, nil
 }
@@ -471,7 +467,7 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (met
 }
 
 // Init - initializes bucket metadata system for all buckets.
-func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []string, objAPI ObjectLayer) error {
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
@@ -484,7 +480,7 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
 }
 
 // concurrently load bucket metadata to speed up loading bucket metadata.
-func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, failedBuckets map[string]struct{}) {
+func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []string, failedBuckets map[string]struct{}) {
 	g := errgroup.WithNErrs(len(buckets))
 	bucketMetas := make([]BucketMetadata, len(buckets))
 	for index := range buckets {
@@ -494,8 +490,8 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 			// herd upon start up sequence.
 			time.Sleep(25*time.Millisecond + time.Duration(rand.Int63n(int64(100*time.Millisecond))))
 
-			_, _ = sys.objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{Recreate: true})
-			meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[index].Name)
+			_, _ = sys.objAPI.HealBucket(ctx, buckets[index], madmin.HealOpts{Recreate: true})
+			meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[index])
 			if err != nil {
 				return err
 			}
@@ -508,7 +504,7 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 	for index, err := range errs {
 		if err != nil {
 			internalLogOnceIf(ctx, fmt.Errorf("Unable to load bucket metadata, will be retried: %w", err),
-				"load-bucket-metadata-"+buckets[index].Name, logger.WarningKind)
+				"load-bucket-metadata-"+buckets[index], logger.WarningKind)
 		}
 	}
 
@@ -519,7 +515,7 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 		if errs[i] != nil {
 			continue
 		}
-		sys.metadataMap[buckets[i].Name] = meta
+		sys.metadataMap[buckets[i]] = meta
 	}
 	sys.Unlock()
 
@@ -528,7 +524,7 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 			if failedBuckets == nil {
 				failedBuckets = make(map[string]struct{})
 			}
-			failedBuckets[buckets[i].Name] = struct{}{}
+			failedBuckets[buckets[i]] = struct{}{}
 			continue
 		}
 		globalEventNotifier.set(buckets[i], meta)   // set notification targets
@@ -548,7 +544,7 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, fa
 		case <-ctx.Done():
 			return
 		case <-t.C:
-			buckets, err := sys.objAPI.ListBuckets(ctx, BucketOptions{})
+			buckets, err := sys.objAPI.ListBuckets(ctx, BucketOptions{NoMetadata: true})
 			if err != nil {
 				internalLogIf(ctx, err, logger.WarningKind)
 				break
@@ -579,8 +575,8 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, fa
 
 				// Initialize the failed buckets
 				if _, ok := failedBuckets[buckets[i].Name]; ok {
-					globalEventNotifier.set(buckets[i], meta)
-					globalBucketTargetSys.set(buckets[i], meta)
+					globalEventNotifier.set(buckets[i].Name, meta)
+					globalBucketTargetSys.set(buckets[i].Name, meta)
 					delete(failedBuckets, buckets[i].Name)
 				}
 
@@ -600,8 +596,8 @@ func (sys *BucketMetadataSys) Initialized() bool {
 }
 
 // Loads bucket metadata for all buckets into BucketMetadataSys.
-func (sys *BucketMetadataSys) init(ctx context.Context, buckets []BucketInfo) {
-	count := 100 // load 100 bucket metadata at a time.
+func (sys *BucketMetadataSys) init(ctx context.Context, buckets []string) {
+	count := globalEndpoints.ESCount() * 10
 	failedBuckets := make(map[string]struct{})
 	for {
 		if len(buckets) < count {

cmd/bucket-replication-handlers.go

@@ -75,7 +75,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
-	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, true)
+	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, &validateReplicationDestinationOptions{CheckRemoteBucket: true})
 	if apiErr != noError {
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
@@ -559,7 +559,7 @@ func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.Respon
 		lockEnabled = lcfg.Enabled()
 	}
 
-	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, true)
+	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, &validateReplicationDestinationOptions{CheckRemoteBucket: true})
 	if apiErr != noError {
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return

cmd/bucket-replication.go

@@ -91,7 +91,10 @@ func getReplicationConfig(ctx context.Context, bucketName string) (rc *replicati
 
 // validateReplicationDestination returns error if replication destination bucket missing or not configured
 // It also returns true if replication destination is same as this server.
-func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config, checkRemote bool) (bool, APIError) {
+func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config, opts *validateReplicationDestinationOptions) (bool, APIError) {
+	if opts == nil {
+		opts = &validateReplicationDestinationOptions{}
+	}
 	var arns []string
 	if rCfg.RoleArn != "" {
 		arns = append(arns, rCfg.RoleArn)
@@ -113,7 +116,7 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 		if clnt == nil {
 			return sameTarget, toAPIError(ctx, BucketRemoteTargetNotFound{Bucket: bucket})
 		}
-		if checkRemote { // validate remote bucket
+		if opts.CheckRemoteBucket { // validate remote bucket
 			found, err := clnt.BucketExists(ctx, arn.Bucket)
 			if err != nil {
 				return sameTarget, errorCodes.ToAPIErrWithErr(ErrRemoteDestinationNotFoundError, err)
@@ -133,23 +136,29 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 				}
 			}
 		}
+		// if checked bucket, then check the ready is unnecessary
+		if !opts.CheckRemoteBucket && opts.CheckReady {
+			endpoint := clnt.EndpointURL().String()
+			if errInt, ok := opts.checkReadyErr.Load(endpoint); !ok {
+				err = checkRemoteEndpoint(ctx, clnt.EndpointURL())
+				opts.checkReadyErr.Store(endpoint, err)
+			} else {
+				if errInt == nil {
+					err = nil
+				} else {
+					err = errInt.(error)
+				}
+			}
+			switch err.(type) {
+			case BucketRemoteIdenticalToSource:
+				return true, errorCodes.ToAPIErrWithErr(ErrBucketRemoteIdenticalToSource, fmt.Errorf("remote target endpoint %s is self referential", clnt.EndpointURL().String()))
+			default:
+			}
+		}
 		// validate replication ARN against target endpoint
-		c := globalBucketTargetSys.GetRemoteTargetClient(bucket, arnStr)
-		if c != nil {
-			if err := checkRemoteEndpoint(ctx, c.EndpointURL()); err != nil {
-				switch err.(type) {
-				case BucketRemoteIdenticalToSource:
-					return true, errorCodes.ToAPIErrWithErr(ErrBucketRemoteIdenticalToSource, fmt.Errorf("remote target endpoint %s is self referential", c.EndpointURL().String()))
-				default:
-				}
-			}
-			if c.EndpointURL().String() == clnt.EndpointURL().String() {
-				selfTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
-				if !sameTarget {
-					sameTarget = selfTarget
-				}
-				continue
-			}
+		selfTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
+		if !sameTarget {
+			sameTarget = selfTarget
 		}
 	}
 
@@ -765,21 +774,6 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
 	return "", false
 }
 
-func getCRCMeta(oi ObjectInfo, partNum int, h http.Header) map[string]string {
-	meta := make(map[string]string)
-	cs := oi.decryptChecksums(partNum, h)
-	for k, v := range cs {
-		cksum := hash.NewChecksumString(k, v)
-		if cksum == nil {
-			continue
-		}
-		if cksum.Valid() {
-			meta[cksum.Type.Key()] = v
-		}
-	}
-	return meta
-}
-
 func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, partNum int) (putOpts minio.PutObjectOptions, err error) {
 	meta := make(map[string]string)
 	isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined)
@@ -788,11 +782,6 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
 		// In case of SSE-C objects copy the allowed internal headers as well
 		if !isSSEC || !slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
 			if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
-				if strings.EqualFold(k, ReservedMetadataPrefixLower+"crc") {
-					for k, v := range getCRCMeta(objInfo, partNum, nil) {
-						meta[k] = v
-					}
-				}
 				continue
 			}
 			if isStandardHeader(k) {
@@ -811,8 +800,12 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
 		if isSSEC {
 			meta[ReplicationSsecChecksumHeader] = base64.StdEncoding.EncodeToString(objInfo.Checksum)
 		} else {
-			for k, v := range getCRCMeta(objInfo, 0, nil) {
-				meta[k] = v
+			for _, pi := range objInfo.Parts {
+				if pi.Number == partNum {
+					for k, v := range pi.Checksums {
+						meta[k] = v
+					}
+				}
 			}
 		}
 	}
@@ -1306,7 +1299,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 		HeaderSize: headerSize,
 	}
 	newCtx := ctx
-	if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) {
+	if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) && objInfo.Size < minLargeObjSize {
 		var cancel context.CancelFunc
 		newCtx, cancel = context.WithTimeout(ctx, throttleDeadline)
 		defer cancel()
@@ -1581,7 +1574,7 @@ applyAction:
 			HeaderSize: headerSize,
 		}
 		newCtx := ctx
-		if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) {
+		if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) && objInfo.Size < minLargeObjSize {
 			var cancel context.CancelFunc
 			newCtx, cancel = context.WithTimeout(ctx, throttleDeadline)
 			defer cancel()
@@ -1666,8 +1659,7 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 		cHeader := http.Header{}
 		cHeader.Add(xhttp.MinIOSourceReplicationRequest, "true")
 		if !isSSEC {
-			crc := getCRCMeta(objInfo, partInfo.Number, nil) // No SSE-C keys here.
-			for k, v := range crc {
+			for k, v := range partInfo.Checksums {
 				cHeader.Add(k, v)
 			}
 		}
@@ -1803,15 +1795,18 @@ var (
 type ReplicationPool struct {
 	// atomic ops:
 	activeWorkers    int32
+	activeLrgWorkers int32
 	activeMRFWorkers int32
 
-	objLayer   ObjectLayer
-	ctx        context.Context
-	priority   string
-	maxWorkers int
-	mu         sync.RWMutex
-	mrfMU      sync.Mutex
-	resyncer   *replicationResyncer
+	objLayer    ObjectLayer
+	ctx         context.Context
+	priority    string
+	maxWorkers  int
+	maxLWorkers int
+
+	mu       sync.RWMutex
+	mrfMU    sync.Mutex
+	resyncer *replicationResyncer
 
 	// workers:
 	workers    []chan ReplicationWorkerOperation
@@ -1882,9 +1877,13 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 	if maxWorkers > 0 && failedWorkers > maxWorkers {
 		failedWorkers = maxWorkers
 	}
+	maxLWorkers := LargeWorkerCount
+	if opts.MaxLWorkers > 0 {
+		maxLWorkers = opts.MaxLWorkers
+	}
 	pool := &ReplicationPool{
 		workers:         make([]chan ReplicationWorkerOperation, 0, workers),
-		lrgworkers:      make([]chan ReplicationWorkerOperation, 0, LargeWorkerCount),
+		lrgworkers:      make([]chan ReplicationWorkerOperation, 0, maxLWorkers),
 		mrfReplicaCh:    make(chan ReplicationWorkerOperation, 100000),
 		mrfWorkerKillCh: make(chan struct{}, failedWorkers),
 		resyncer:        newresyncer(),
@@ -1894,9 +1893,10 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 		objLayer:        o,
 		priority:        priority,
 		maxWorkers:      maxWorkers,
+		maxLWorkers:     maxLWorkers,
 	}
 
-	pool.AddLargeWorkers()
+	pool.ResizeLrgWorkers(maxLWorkers, 0)
 	pool.ResizeWorkers(workers, 0)
 	pool.ResizeFailedWorkers(failedWorkers)
 	go pool.resyncer.PersistToDisk(ctx, o)
@@ -1975,23 +1975,8 @@ func (p *ReplicationPool) AddWorker(input <-chan ReplicationWorkerOperation, opT
 	}
 }
 
-// AddLargeWorkers adds a static number of workers to handle large uploads
-func (p *ReplicationPool) AddLargeWorkers() {
-	for i := 0; i < LargeWorkerCount; i++ {
-		p.lrgworkers = append(p.lrgworkers, make(chan ReplicationWorkerOperation, 100000))
-		i := i
-		go p.AddLargeWorker(p.lrgworkers[i])
-	}
-	go func() {
-		<-p.ctx.Done()
-		for i := 0; i < LargeWorkerCount; i++ {
-			xioutil.SafeClose(p.lrgworkers[i])
-		}
-	}()
-}
-
 // AddLargeWorker adds a replication worker to the static pool for large uploads.
-func (p *ReplicationPool) AddLargeWorker(input <-chan ReplicationWorkerOperation) {
+func (p *ReplicationPool) AddLargeWorker(input <-chan ReplicationWorkerOperation, opTracker *int32) {
 	for {
 		select {
 		case <-p.ctx.Done():
@@ -2002,11 +1987,23 @@ func (p *ReplicationPool) AddLargeWorker(input <-chan ReplicationWorkerOperation
 			}
 			switch v := oi.(type) {
 			case ReplicateObjectInfo:
+				if opTracker != nil {
+					atomic.AddInt32(opTracker, 1)
+				}
 				globalReplicationStats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 				replicateObject(p.ctx, v, p.objLayer)
 				globalReplicationStats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
+				if opTracker != nil {
+					atomic.AddInt32(opTracker, -1)
+				}
 			case DeletedObjectReplicationInfo:
+				if opTracker != nil {
+					atomic.AddInt32(opTracker, 1)
+				}
 				replicateDelete(p.ctx, v, p.objLayer)
+				if opTracker != nil {
+					atomic.AddInt32(opTracker, -1)
+				}
 			default:
 				bugLogIf(p.ctx, fmt.Errorf("unknown replication type: %T", oi), "unknown-replicate-type")
 			}
@@ -2014,6 +2011,30 @@ func (p *ReplicationPool) AddLargeWorker(input <-chan ReplicationWorkerOperation
 	}
 }
 
+// ResizeLrgWorkers sets replication workers pool for large transfers(>=128MiB) to new size.
+// checkOld can be set to an expected value.
+// If the worker count changed
+func (p *ReplicationPool) ResizeLrgWorkers(n, checkOld int) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if (checkOld > 0 && len(p.lrgworkers) != checkOld) || n == len(p.lrgworkers) || n < 1 {
+		// Either already satisfied or worker count changed while we waited for the lock.
+		return
+	}
+	for len(p.lrgworkers) < n {
+		input := make(chan ReplicationWorkerOperation, 100000)
+		p.lrgworkers = append(p.lrgworkers, input)
+
+		go p.AddLargeWorker(input, &p.activeLrgWorkers)
+	}
+	for len(p.lrgworkers) > n {
+		worker := p.lrgworkers[len(p.lrgworkers)-1]
+		p.lrgworkers = p.lrgworkers[:len(p.lrgworkers)-1]
+		xioutil.SafeClose(worker)
+	}
+}
+
 // ActiveWorkers returns the number of active workers handling replication traffic.
 func (p *ReplicationPool) ActiveWorkers() int {
 	return int(atomic.LoadInt32(&p.activeWorkers))
@@ -2024,6 +2045,11 @@ func (p *ReplicationPool) ActiveMRFWorkers() int {
 	return int(atomic.LoadInt32(&p.activeMRFWorkers))
 }
 
+// ActiveLrgWorkers returns the number of active workers handling traffic > 128MiB object size.
+func (p *ReplicationPool) ActiveLrgWorkers() int {
+	return int(atomic.LoadInt32(&p.activeLrgWorkers))
+}
+
 // ResizeWorkers sets replication workers pool to new size.
 // checkOld can be set to an expected value.
 // If the worker count changed
@@ -2049,7 +2075,7 @@ func (p *ReplicationPool) ResizeWorkers(n, checkOld int) {
 }
 
 // ResizeWorkerPriority sets replication failed workers pool size
-func (p *ReplicationPool) ResizeWorkerPriority(pri string, maxWorkers int) {
+func (p *ReplicationPool) ResizeWorkerPriority(pri string, maxWorkers, maxLWorkers int) {
 	var workers, mrfWorkers int
 	p.mu.Lock()
 	switch pri {
@@ -2076,11 +2102,15 @@ func (p *ReplicationPool) ResizeWorkerPriority(pri string, maxWorkers int) {
 	if maxWorkers > 0 && mrfWorkers > maxWorkers {
 		mrfWorkers = maxWorkers
 	}
+	if maxLWorkers <= 0 {
+		maxLWorkers = LargeWorkerCount
+	}
 	p.priority = pri
 	p.maxWorkers = maxWorkers
 	p.mu.Unlock()
 	p.ResizeWorkers(workers, 0)
 	p.ResizeFailedWorkers(mrfWorkers)
+	p.ResizeLrgWorkers(maxLWorkers, 0)
 }
 
 // ResizeFailedWorkers sets replication failed workers pool size
@@ -2124,9 +2154,18 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		h := xxh3.HashString(ri.Bucket + ri.Name)
 		select {
 		case <-p.ctx.Done():
-		case p.lrgworkers[h%LargeWorkerCount] <- ri:
+		case p.lrgworkers[h%uint64(len(p.lrgworkers))] <- ri:
 		default:
 			globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
+			p.mu.RLock()
+			maxLWorkers := p.maxLWorkers
+			existing := len(p.lrgworkers)
+			p.mu.RUnlock()
+			maxLWorkers = min(maxLWorkers, LargeWorkerCount)
+			if p.ActiveLrgWorkers() < maxLWorkers {
+				workers := min(existing+1, maxLWorkers)
+				p.ResizeLrgWorkers(workers, existing)
+			}
 		}
 		return
 	}
@@ -2229,8 +2268,9 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 }
 
 type replicationPoolOpts struct {
-	Priority   string
-	MaxWorkers int
+	Priority    string
+	MaxWorkers  int
+	MaxLWorkers int
 }
 
 func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
@@ -3086,7 +3126,7 @@ func (p *ReplicationPool) deleteResyncMetadata(ctx context.Context, bucket strin
 }
 
 // initResync - initializes bucket replication resync for all buckets.
-func (p *ReplicationPool) initResync(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+func (p *ReplicationPool) initResync(ctx context.Context, buckets []string, objAPI ObjectLayer) error {
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
@@ -3095,7 +3135,7 @@ func (p *ReplicationPool) initResync(ctx context.Context, buckets []BucketInfo,
 	return nil
 }
 
-func (p *ReplicationPool) startResyncRoutine(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
+func (p *ReplicationPool) startResyncRoutine(ctx context.Context, buckets []string, objAPI ObjectLayer) {
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	// Run the replication resync in a loop
 	for {
@@ -3113,13 +3153,13 @@ func (p *ReplicationPool) startResyncRoutine(ctx context.Context, buckets []Buck
 }
 
 // Loads bucket replication resync statuses into memory.
-func (p *ReplicationPool) loadResync(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+func (p *ReplicationPool) loadResync(ctx context.Context, buckets []string, objAPI ObjectLayer) error {
 	// Make sure only one node running resync on the cluster.
 	ctx, cancel := globalLeaderLock.GetLock(ctx)
 	defer cancel()
 
 	for index := range buckets {
-		bucket := buckets[index].Name
+		bucket := buckets[index]
 
 		meta, err := loadBucketResyncMetadata(ctx, bucket, objAPI)
 		if err != nil {
@@ -3505,7 +3545,7 @@ func (p *ReplicationPool) persistToDrive(ctx context.Context, v MRFReplicateEntr
 	}
 
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 
 	for _, localDrive := range localDrives {
@@ -3572,7 +3612,7 @@ func (p *ReplicationPool) loadMRF() (mrfRec MRFReplicateEntries, err error) {
 	}
 
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 
 	for _, localDrive := range localDrives {
@@ -3693,3 +3733,12 @@ func (p *ReplicationPool) getMRF(ctx context.Context, bucket string) (ch <-chan
 
 	return mrfCh, nil
 }
+
+// validateReplicationDestinationOptions is used to configure the validation of the replication destination.
+// validateReplicationDestination uses this to configure the validation.
+type validateReplicationDestinationOptions struct {
+	CheckRemoteBucket bool
+	CheckReady        bool
+
+	checkReadyErr sync.Map
+}

cmd/bucket-targets.go

@@ -612,7 +612,7 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
 }
 
 // create minio-go clients for buckets having remote targets
-func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
+func (sys *BucketTargetSys) set(bucket string, meta BucketMetadata) {
 	cfg := meta.bucketTargetConfig
 	if cfg == nil || cfg.Empty() {
 		return
@@ -626,9 +626,9 @@ func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
 			continue
 		}
 		sys.arnRemotesMap[tgt.Arn] = arnTarget{Client: tgtClient}
-		sys.updateBandwidthLimit(bucket.Name, tgt.Arn, tgt.BandwidthLimit)
+		sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit)
 	}
-	sys.targetsMap[bucket.Name] = cfg.Targets
+	sys.targetsMap[bucket] = cfg.Targets
 }
 
 // Returns a minio-go Client configured to access remote host described in replication target config.

cmd/common-main.go

@@ -433,7 +433,6 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 	ctxt.RecvBufSize = ctx.Int("recv-buf-size")
 	ctxt.IdleTimeout = ctx.Duration("idle-timeout")
 	ctxt.UserTimeout = ctx.Duration("conn-user-timeout")
-	ctxt.ShutdownTimeout = ctx.Duration("shutdown-timeout")
 
 	if conf := ctx.String("config"); len(conf) > 0 {
 		err = mergeServerCtxtFromConfigFile(conf, ctxt)
@@ -686,16 +685,6 @@ func loadEnvVarsFromFiles() {
 		}
 	}
 
-	if env.IsSet(kms.EnvKMSSecretKeyFile) {
-		kmsSecret, err := readFromSecret(env.Get(kms.EnvKMSSecretKeyFile, ""))
-		if err != nil {
-			logger.Fatal(err, "Unable to read the KMS secret key inherited from secret file")
-		}
-		if kmsSecret != "" {
-			os.Setenv(kms.EnvKMSSecretKey, kmsSecret)
-		}
-	}
-
 	if env.IsSet(config.EnvConfigEnvFile) {
 		ekvs, err := minioEnvironFromFile(env.Get(config.EnvConfigEnvFile, ""))
 		if err != nil && !os.IsNotExist(err) {
@@ -835,7 +824,7 @@ func serverHandleEnvVars() {
 		}
 	}
 
-	globalDisableFreezeOnBoot = env.Get("_MINIO_DISABLE_API_FREEZE_ON_BOOT", "") == "true" || serverDebugLog
+	globalEnableSyncBoot = env.Get("MINIO_SYNC_BOOT", config.EnableOff) == config.EnableOn
 }
 
 func loadRootCredentials() {
@@ -844,6 +833,7 @@ func loadRootCredentials() {
 	// Check both cases and authenticate them if correctly defined
 	var user, password string
 	var hasCredentials bool
+	var legacyCredentials bool
 	//nolint:gocritic
 	if env.IsSet(config.EnvRootUser) && env.IsSet(config.EnvRootPassword) {
 		user = env.Get(config.EnvRootUser, "")
@@ -852,6 +842,7 @@ func loadRootCredentials() {
 	} else if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
 		user = env.Get(config.EnvAccessKey, "")
 		password = env.Get(config.EnvSecretKey, "")
+		legacyCredentials = true
 		hasCredentials = true
 	} else if globalServerCtxt.RootUser != "" && globalServerCtxt.RootPwd != "" {
 		user, password = globalServerCtxt.RootUser, globalServerCtxt.RootPwd
@@ -860,8 +851,13 @@ func loadRootCredentials() {
 	if hasCredentials {
 		cred, err := auth.CreateCredentials(user, password)
 		if err != nil {
-			logger.Fatal(config.ErrInvalidCredentials(err),
-				"Unable to validate credentials inherited from the shell environment")
+			if legacyCredentials {
+				logger.Fatal(config.ErrInvalidCredentials(err),
+					"Unable to validate credentials inherited from the shell environment")
+			} else {
+				logger.Fatal(config.ErrInvalidRootUserCredentials(err),
+					"Unable to validate credentials inherited from the shell environment")
+			}
 		}
 		if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
 			msg := fmt.Sprintf("WARNING: %s and %s are deprecated.\n"+
@@ -875,6 +871,12 @@ func loadRootCredentials() {
 	} else {
 		globalActiveCred = auth.DefaultCredentials
 	}
+
+	var err error
+	globalNodeAuthToken, err = authenticateNode(globalActiveCred.AccessKey, globalActiveCred.SecretKey)
+	if err != nil {
+		logger.Fatal(err, "Unable to generate internode credentials")
+	}
 }
 
 // Initialize KMS global variable after valiadating and loading the configuration.

cmd/config-current.go

@@ -101,7 +101,7 @@ func initHelp() {
 		config.HelpKV{
 			Key:         config.SubnetSubSys,
 			Type:        "string",
-			Description: "register the cluster to MinIO SUBNET",
+			Description: "register Enterprise license for the cluster",
 			Optional:    true,
 		},
 		config.HelpKV{

cmd/data-scanner.go

@@ -31,6 +31,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/minio/madmin-go/v3"
@@ -226,7 +227,9 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 				binary.LittleEndian.PutUint64(tmp, cycleInfo.next)
 				tmp, _ = cycleInfo.MarshalMsg(tmp)
 				err = saveConfig(ctx, objAPI, dataUsageBloomNamePath, tmp)
-				scannerLogIf(ctx, err, dataUsageBloomNamePath)
+				if err != nil {
+					scannerLogIf(ctx, fmt.Errorf("%w, Object %s", err, dataUsageBloomNamePath))
+				}
 			}
 		}
 	}
@@ -249,7 +252,8 @@ type folderScanner struct {
 	healObjectSelect      uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
 	scanMode              madmin.HealScanMode
 
-	weSleep func() bool
+	weSleep    func() bool
+	shouldHeal func() bool
 
 	disks       []StorageAPI
 	disksQuorum int
@@ -304,11 +308,12 @@ type folderScanner struct {
 // The returned cache will always be valid, but may not be updated from the existing.
 // Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
 // If the supplied context is canceled the function will return at the first chance.
-func scanDataFolder(ctx context.Context, disks []StorageAPI, basePath string, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode, weSleep func() bool) (dataUsageCache, error) {
+func scanDataFolder(ctx context.Context, disks []StorageAPI, drive *xlStorage, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode, weSleep func() bool) (dataUsageCache, error) {
 	switch cache.Info.Name {
 	case "", dataUsageRoot:
 		return cache, errors.New("internal error: root scan attempted")
 	}
+	basePath := drive.drivePath
 	updatePath, closeDisk := globalScannerMetrics.currentPathUpdater(basePath, cache.Info.Name)
 	defer closeDisk()
 
@@ -328,6 +333,26 @@ func scanDataFolder(ctx context.Context, disks []StorageAPI, basePath string, ca
 		disksQuorum:           len(disks) / 2,
 	}
 
+	var skipHeal atomic.Bool
+	if globalIsErasure || cache.Info.SkipHealing {
+		skipHeal.Store(true)
+	}
+
+	// Check if we should do healing at all.
+	s.shouldHeal = func() bool {
+		if skipHeal.Load() {
+			return false
+		}
+		if s.healObjectSelect == 0 {
+			return false
+		}
+		if di, _ := drive.DiskInfo(ctx, DiskInfoOptions{}); di.Healing {
+			skipHeal.Store(true)
+			return false
+		}
+		return true
+	}
+
 	// Enable healing in XL mode.
 	if globalIsErasure && !cache.Info.SkipHealing {
 		// Do a heal check on an object once every n cycles. Must divide into healFolderInclude
@@ -482,14 +507,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				replication: replicationCfg,
 			}
 
-			item.heal.enabled = thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure
+			item.heal.enabled = thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healObjectSelect/folder.objectHealProbDiv) && f.shouldHeal()
 			item.heal.bitrot = f.scanMode == madmin.HealDeepScan
 
-			// if the drive belongs to an erasure set
-			// that is already being healed, skip the
-			// healing attempt on this drive.
-			item.heal.enabled = item.heal.enabled && f.healObjectSelect > 0
-
 			sz, err := f.getSize(item)
 			if err != nil && err != errIgnoreFileContrib {
 				wait() // wait to proceed to next entry.
@@ -652,7 +672,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		}
 
 		// Scan for healing
-		if f.healObjectSelect == 0 || len(abandonedChildren) == 0 {
+		if len(abandonedChildren) == 0 || !f.shouldHeal() {
 			// If we are not heal scanning, return now.
 			break
 		}
@@ -687,6 +707,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 
 		healObjectsPrefix := color.Green("healObjects:")
 		for k := range abandonedChildren {
+			if !f.shouldHeal() {
+				break
+			}
 			bucket, prefix := path2BucketObject(k)
 			stopFn := globalScannerMetrics.time(scannerMetricCheckMissing)
 			f.updateCurrentPath(k)
@@ -720,6 +743,10 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				},
 				// Some disks have data for this.
 				partial: func(entries metaCacheEntries, errs []error) {
+					if !f.shouldHeal() {
+						cancel()
+						return
+					}
 					entry, ok := entries.resolve(&resolver)
 					if !ok {
 						// check if we can get one entry at least
@@ -772,7 +799,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						}, madmin.HealItemObject)
 						stopFn(int(ver.Size))
 						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-							scannerLogIf(ctx, err, fiv.Name)
+							if err != nil {
+								scannerLogIf(ctx, fmt.Errorf("%w, Object %s/%s/%s", err, bucket, fiv.Name, ver.VersionID))
+							}
 						}
 						if err == nil {
 							successVersions++
@@ -1246,7 +1275,7 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return false
 		}
-		ilmLogIf(ctx, err)
+		ilmLogIf(ctx, fmt.Errorf("expireTransitionedObject(%s, %s): %w", obj.Bucket, obj.Name, err))
 		return false
 	}
 	timeILM(1)
@@ -1299,7 +1328,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 			return false
 		}
 		// Assume it is still there.
-		ilmLogOnceIf(ctx, err, "non-transition-expiry")
+		ilmLogOnceIf(ctx, fmt.Errorf("DeleteObject(%s, %s): %w", obj.Bucket, obj.Name, err), "non-transition-expiry"+obj.Name)
 		return false
 	}
 	if dobj.Name == "" {

cmd/data-usage_test.go

@@ -26,6 +26,9 @@ import (
 	"path"
 	"path/filepath"
 	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/cachevalue"
 )
 
 type usageTestFile struct {
@@ -61,10 +64,13 @@ func TestDataUsageUpdate(t *testing.T) {
 		}
 		return
 	}
-
+	xls := xlStorage{drivePath: base, diskInfoCache: cachevalue.New[DiskInfo]()}
+	xls.diskInfoCache.InitOnce(time.Second, cachevalue.Opts{}, func(ctx context.Context) (DiskInfo, error) {
+		return DiskInfo{Total: 1 << 40, Free: 1 << 40}, nil
+	})
 	weSleep := func() bool { return false }
 
-	got, err := scanDataFolder(context.Background(), nil, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0, weSleep)
+	got, err := scanDataFolder(context.Background(), nil, &xls, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0, weSleep)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -174,7 +180,7 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = scanDataFolder(context.Background(), nil, base, got, getSize, 0, weSleep)
+		got, err = scanDataFolder(context.Background(), nil, &xls, got, getSize, 0, weSleep)
 		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
@@ -283,8 +289,12 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 
 	weSleep := func() bool { return false }
+	xls := xlStorage{drivePath: base, diskInfoCache: cachevalue.New[DiskInfo]()}
+	xls.diskInfoCache.InitOnce(time.Second, cachevalue.Opts{}, func(ctx context.Context) (DiskInfo, error) {
+		return DiskInfo{Total: 1 << 40, Free: 1 << 40}, nil
+	})
 
-	got, err := scanDataFolder(context.Background(), nil, base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize, 0, weSleep)
+	got, err := scanDataFolder(context.Background(), nil, &xls, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize, 0, weSleep)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -419,7 +429,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = scanDataFolder(context.Background(), nil, base, got, getSize, 0, weSleep)
+		got, err = scanDataFolder(context.Background(), nil, &xls, got, getSize, 0, weSleep)
 		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
@@ -567,8 +577,12 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 		}
 		return
 	}
+	xls := xlStorage{drivePath: base, diskInfoCache: cachevalue.New[DiskInfo]()}
+	xls.diskInfoCache.InitOnce(time.Second, cachevalue.Opts{}, func(ctx context.Context) (DiskInfo, error) {
+		return DiskInfo{Total: 1 << 40, Free: 1 << 40}, nil
+	})
 	weSleep := func() bool { return false }
-	want, err := scanDataFolder(context.Background(), nil, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0, weSleep)
+	want, err := scanDataFolder(context.Background(), nil, &xls, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0, weSleep)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/dummy-data-generator_test.go

@@ -61,10 +61,9 @@ func NewDummyDataGen(totalLength, skipOffset int64) io.ReadSeeker {
 	}
 
 	skipOffset %= int64(len(alphabets))
-	as := make([]byte, 2*len(alphabets))
-	copy(as, alphabets)
-	copy(as[len(alphabets):], alphabets)
-	b := as[skipOffset : skipOffset+int64(len(alphabets))]
+	const multiply = 100
+	as := bytes.Repeat(alphabets, multiply)
+	b := as[skipOffset : skipOffset+int64(len(alphabets)*(multiply-1))]
 	return &DummyDataGen{
 		length: totalLength,
 		b:      b,

cmd/encryption-v1.go

@@ -134,11 +134,16 @@ func DecryptETags(ctx context.Context, k *kms.KMS, objects []ObjectInfo) error {
 		SSES3SinglePartObjects := make(map[int]bool)
 		for i, object := range batch {
 			if kind, ok := crypto.IsEncrypted(object.UserDefined); ok && kind == crypto.S3 && !crypto.IsMultiPart(object.UserDefined) {
-				SSES3SinglePartObjects[i] = true
-
-				metadata = append(metadata, object.UserDefined)
-				buckets = append(buckets, object.Bucket)
-				names = append(names, object.Name)
+				ETag, err := etag.Parse(object.ETag)
+				if err != nil {
+					continue
+				}
+				if ETag.IsEncrypted() {
+					SSES3SinglePartObjects[i] = true
+					metadata = append(metadata, object.UserDefined)
+					buckets = append(buckets, object.Bucket)
+					names = append(names, object.Name)
+				}
 			}
 		}
 
@@ -190,7 +195,7 @@ func DecryptETags(ctx context.Context, k *kms.KMS, objects []ObjectInfo) error {
 				if err != nil {
 					return err
 				}
-				if SSES3SinglePartObjects[i] && ETag.IsEncrypted() {
+				if SSES3SinglePartObjects[i] {
 					ETag, err = etag.Decrypt(keys[0][:], ETag)
 					if err != nil {
 						return err
@@ -1167,6 +1172,14 @@ func (o *ObjectInfo) decryptChecksums(part int, h http.Header) map[string]string
 	if len(data) == 0 {
 		return nil
 	}
+	if part > 0 && !crypto.SSEC.IsEncrypted(o.UserDefined) {
+		// already decrypted in ToObjectInfo for multipart objects
+		for _, pi := range o.Parts {
+			if pi.Number == part {
+				return pi.Checksums
+			}
+		}
+	}
 	if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted {
 		decrypted, err := o.metadataDecrypter(h)("object-checksum", data)
 		if err != nil {

cmd/endpoint.go

@@ -253,6 +253,14 @@ type PoolEndpoints struct {
 // EndpointServerPools - list of list of endpoints
 type EndpointServerPools []PoolEndpoints
 
+// ESCount returns the total number of erasure sets in this cluster
+func (l EndpointServerPools) ESCount() (count int) {
+	for _, p := range l {
+		count += p.SetCount
+	}
+	return
+}
+
 // GetNodes returns a sorted list of nodes in this cluster
 func (l EndpointServerPools) GetNodes() (nodes []Node) {
 	nodesMap := make(map[string]Node)

cmd/erasure-healing-common_test.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 	"time"
 
@@ -147,6 +148,10 @@ func TestCommonTime(t *testing.T) {
 // TestListOnlineDisks - checks if listOnlineDisks and outDatedDisks
 // are consistent with each other.
 func TestListOnlineDisks(t *testing.T) {
+	if runtime.GOOS == globalWindowsOSName {
+		t.Skip()
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -230,7 +235,7 @@ func TestListOnlineDisks(t *testing.T) {
 	}
 
 	object := "object"
-	data := bytes.Repeat([]byte("a"), smallFileThreshold*16)
+	data := bytes.Repeat([]byte("a"), smallFileThreshold*32)
 	z := obj.(*erasureServerPools)
 
 	erasureDisks, err := z.GetDisks(0, 0)

cmd/erasure-healing.go

@@ -629,7 +629,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		}
 
 		for i, v := range result.Before.Drives {
-			if v.Endpoint == disk.String() {
+			if v.Endpoint == disk.Endpoint().String() {
 				result.After.Drives[i].State = madmin.DriveStateOk
 			}
 		}

cmd/erasure-metadata.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/amztime"
+	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash/sha256"
@@ -173,6 +174,7 @@ func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInf
 		}
 	}
 	objInfo.Checksum = fi.Checksum
+	objInfo.decryptPartsChecksums(nil)
 	objInfo.Inlined = fi.InlineData()
 	// Success.
 	return objInfo
@@ -456,6 +458,7 @@ func commonParity(parities []int, defaultParityCount int) int {
 }
 
 func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int) {
+	totalShards := len(partsMetadata)
 	parities = make([]int, len(partsMetadata))
 	for index, metadata := range partsMetadata {
 		if errs[index] != nil {
@@ -466,9 +469,13 @@ func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int)
 			parities[index] = -1
 			continue
 		}
+		//nolint:gocritic
 		// Delete marker or zero byte objects take highest parity.
 		if metadata.Deleted || metadata.Size == 0 {
-			parities[index] = len(partsMetadata) / 2
+			parities[index] = totalShards / 2
+		} else if metadata.TransitionStatus == lifecycle.TransitionComplete {
+			// For tiered objects, read quorum is N/2+1 to ensure simple majority on xl.meta. It is not equal to EcM because the data integrity is entrusted with the warm tier.
+			parities[index] = totalShards - (totalShards/2 + 1)
 		} else {
 			parities[index] = metadata.Erasure.ParityBlocks
 		}

cmd/erasure-metadata_test.go

@@ -19,6 +19,8 @@ package cmd
 
 import (
 	"context"
+	"fmt"
+	"slices"
 	"strconv"
 	"testing"
 	"time"
@@ -359,3 +361,124 @@ func TestSkipTierFreeVersion(t *testing.T) {
 		t.Fatal("Expected SkipTierFreeVersion to be set on FileInfo but wasn't")
 	}
 }
+
+func TestListObjectParities(t *testing.T) {
+	mkMetaArr := func(N, parity, agree int) []FileInfo {
+		fi := newFileInfo("obj-1", N-parity, parity)
+		fi.TransitionTier = "WARM-TIER"
+		fi.TransitionedObjName = mustGetUUID()
+		fi.TransitionStatus = "complete"
+		fi.Size = 1 << 20
+
+		metaArr := make([]FileInfo, N)
+		for i := range N {
+			fi.Erasure.Index = i + 1
+			metaArr[i] = fi
+			if i < agree {
+				continue
+			}
+			metaArr[i].TransitionTier, metaArr[i].TransitionedObjName = "", ""
+			metaArr[i].TransitionStatus = ""
+		}
+		return metaArr
+	}
+	mkParities := func(N, agreedParity, disagreedParity, agree int) []int {
+		ps := make([]int, N)
+		for i := range N {
+			if i < agree {
+				ps[i] = agreedParity
+				continue
+			}
+			ps[i] = disagreedParity // disagree
+		}
+		return ps
+	}
+
+	mkTest := func(N, parity, agree int) (res struct {
+		metaArr  []FileInfo
+		errs     []error
+		parities []int
+		parity   int
+	},
+	) {
+		res.metaArr = mkMetaArr(N, parity, agree)
+		res.parities = mkParities(N, N-(N/2+1), parity, agree)
+		res.errs = make([]error, N)
+		if agree >= N/2+1 { // simple majority consensus
+			res.parity = N - (N/2 + 1)
+		} else {
+			res.parity = -1
+		}
+		return res
+	}
+
+	nonTieredTest := func(N, parity, agree int) (res struct {
+		metaArr  []FileInfo
+		errs     []error
+		parities []int
+		parity   int
+	},
+	) {
+		fi := newFileInfo("obj-1", N-parity, parity)
+		fi.Size = 1 << 20
+		metaArr := make([]FileInfo, N)
+		parities := make([]int, N)
+		for i := range N {
+			fi.Erasure.Index = i + 1
+			metaArr[i] = fi
+			parities[i] = parity
+			if i < agree {
+				continue
+			}
+			metaArr[i].Erasure.Index = 0 // creates invalid fi on remaining drives
+			parities[i] = -1             // invalid fi are assigned parity -1
+		}
+		res.metaArr = metaArr
+		res.parities = parities
+		res.errs = make([]error, N)
+		if agree >= N-parity {
+			res.parity = parity
+		} else {
+			res.parity = -1
+		}
+
+		return res
+	}
+	tests := []struct {
+		metaArr  []FileInfo
+		errs     []error
+		parities []int
+		parity   int
+	}{
+		// More than simple majority consensus
+		mkTest(15, 3, 11),
+		// No simple majority consensus
+		mkTest(15, 3, 7),
+		// Exact simple majority consensus
+		mkTest(15, 3, 8),
+		// More than simple majority consensus
+		mkTest(16, 4, 11),
+		// No simple majority consensus
+		mkTest(16, 4, 8),
+		// Exact simple majority consensus
+		mkTest(16, 4, 9),
+		// non-tiered object require read quorum of EcM
+		nonTieredTest(15, 3, 12),
+		// non-tiered object with fewer than EcM in consensus
+		nonTieredTest(15, 3, 11),
+		// non-tiered object require read quorum of EcM
+		nonTieredTest(16, 4, 12),
+		// non-tiered object with fewer than EcM in consensus
+		nonTieredTest(16, 4, 11),
+	}
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
+			if got := listObjectParities(test.metaArr, test.errs); !slices.Equal(got, test.parities) {
+				t.Fatalf("Expected parities %v but got %v", test.parities, got)
+			}
+			if got := commonParity(test.parities, len(test.metaArr)/2); got != test.parity {
+				t.Fatalf("Expected common parity %v but got %v", test.parity, got)
+			}
+		})
+	}
+}

cmd/erasure-multipart.go

@@ -103,10 +103,9 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 	return fi, partsMetadata, err
 }
 
-// Removes part.meta given by partName belonging to a multipart upload from minioMetaBucket
-func (er erasureObjects) removePartMeta(bucket, object, uploadID, dataDir string, partNumber int) {
-	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
-	curpartPath := pathJoin(uploadIDPath, dataDir, fmt.Sprintf("part.%d", partNumber))
+// cleanMultipartPath removes all extraneous files and parts from the multipart folder, this is used per CompleteMultipart.
+// do not use this function outside of completeMultipartUpload()
+func (er erasureObjects) cleanupMultipartPath(ctx context.Context, paths ...string) {
 	storageDisks := er.getDisks()
 
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -116,42 +115,7 @@ func (er erasureObjects) removePartMeta(bucket, object, uploadID, dataDir string
 		}
 		index := index
 		g.Go(func() error {
-			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath+".meta", DeleteOptions{
-				Recursive: false,
-				Immediate: false,
-			})
-
-			return nil
-		}, index)
-	}
-	g.Wait()
-}
-
-// Removes part given by partName belonging to a multipart upload from minioMetaBucket
-func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir string, partNumber int) {
-	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
-	curpartPath := pathJoin(uploadIDPath, dataDir, fmt.Sprintf("part.%d", partNumber))
-	storageDisks := er.getDisks()
-
-	g := errgroup.WithNErrs(len(storageDisks))
-	for index, disk := range storageDisks {
-		if disk == nil {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			// Ignoring failure to remove parts that weren't present in CompleteMultipartUpload
-			// requests. xl.meta is the authoritative source of truth on which parts constitute
-			// the object. The presence of parts that don't belong in the object doesn't affect correctness.
-			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath, DeleteOptions{
-				Recursive: false,
-				Immediate: false,
-			})
-			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath+".meta", DeleteOptions{
-				Recursive: false,
-				Immediate: false,
-			})
-
+			_ = storageDisks[index].DeleteBulk(ctx, minioMetaMultipartBucket, paths...)
 			return nil
 		}, index)
 	}
@@ -159,7 +123,7 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
 }
 
 // Clean-up the old multipart uploads. Should be run in a Go routine.
-func (er erasureObjects) cleanupStaleUploads(ctx context.Context, expiry time.Duration) {
+func (er erasureObjects) cleanupStaleUploads(ctx context.Context) {
 	// run multiple cleanup's local to this server.
 	var wg sync.WaitGroup
 	for _, disk := range er.getLocalDisks() {
@@ -167,7 +131,7 @@ func (er erasureObjects) cleanupStaleUploads(ctx context.Context, expiry time.Du
 			wg.Add(1)
 			go func(disk StorageAPI) {
 				defer wg.Done()
-				er.cleanupStaleUploadsOnDisk(ctx, disk, expiry)
+				er.cleanupStaleUploadsOnDisk(ctx, disk)
 			}(disk)
 		}
 	}
@@ -193,7 +157,7 @@ func (er erasureObjects) deleteAll(ctx context.Context, bucket, prefix string) {
 }
 
 // Remove the old multipart uploads on the given disk.
-func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk StorageAPI, expiry time.Duration) {
+func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk StorageAPI) {
 	drivePath := disk.Endpoint().Path
 
 	readDirFn(pathJoin(drivePath, minioMetaMultipartBucket), func(shaDir string, typ os.FileMode) error {
@@ -219,7 +183,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 				modTime = fi.ModTime
 				wait()
 			}
-			if time.Since(modTime) < expiry {
+			if time.Since(modTime) < globalAPIConfig.getStaleUploadsExpiry() {
 				return nil
 			}
 			w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
@@ -238,7 +202,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 			return nil
 		}
 		// Modtime is returned in the Created field. See (*xlStorage).StatVol
-		if time.Since(vi.Created) < expiry {
+		if time.Since(vi.Created) < globalAPIConfig.getStaleUploadsExpiry() {
 			return nil
 		}
 		w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
@@ -267,7 +231,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 		return w.Run(func() error {
 			wait := deleteMultipartCleanupSleeper.Timer(ctx)
-			if time.Since(vi.Created) > expiry {
+			if time.Since(vi.Created) > globalAPIConfig.getStaleUploadsExpiry() {
 				pathUUID := mustGetUUID()
 				targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID)
 
@@ -1359,10 +1323,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 
+	paths := make([]string, 0, len(currentFI.Parts))
 	// Remove parts that weren't present in CompleteMultipartUpload request.
 	for _, curpart := range currentFI.Parts {
-		// Remove part.meta which is not needed anymore.
-		er.removePartMeta(bucket, object, uploadID, currentFI.DataDir, curpart.Number)
+		paths = append(paths, pathJoin(uploadIDPath, currentFI.DataDir, fmt.Sprintf("part.%d.meta", curpart.Number)))
 
 		if objectPartIndex(fi.Parts, curpart.Number) == -1 {
 			// Delete the missing part files. e.g,
@@ -1371,10 +1335,12 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			// Request 3: PutObjectPart 2
 			// Request 4: CompleteMultipartUpload --part 2
 			// N.B. 1st part is not present. This part should be removed from the storage.
-			er.removeObjectPart(bucket, object, uploadID, currentFI.DataDir, curpart.Number)
+			paths = append(paths, pathJoin(uploadIDPath, currentFI.DataDir, fmt.Sprintf("part.%d", curpart.Number)))
 		}
 	}
 
+	er.cleanupMultipartPath(ctx, paths...) // cleanup all part.N.meta, and skipped part.N's before final rename().
+
 	defer func() {
 		if err == nil {
 			er.deleteAll(context.Background(), minioMetaMultipartBucket, uploadIDPath)

cmd/erasure-object.go

@@ -283,7 +283,7 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 	}
 
 	if unlockOnDefer {
-		unlockOnDefer = fi.InlineData()
+		unlockOnDefer = fi.InlineData() || len(fi.Data) > 0
 	}
 
 	pr, pw := xioutil.WaitPipe()
@@ -576,35 +576,11 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 }
 
 func fileInfoFromRaw(ri RawFileInfo, bucket, object string, readData, inclFreeVers, allParts bool) (FileInfo, error) {
-	var xl xlMetaV2
-	if err := xl.LoadOrConvert(ri.Buf); err != nil {
-		return FileInfo{}, errFileCorrupt
-	}
-
-	fi, err := xl.ToFileInfo(bucket, object, "", inclFreeVers, allParts)
-	if err != nil {
-		return FileInfo{}, err
-	}
-
-	if !fi.IsValid() {
-		return FileInfo{}, errFileCorrupt
-	}
-
-	versionID := fi.VersionID
-	if versionID == "" {
-		versionID = nullVersionID
-	}
-
-	fileInfo, err := xl.ToFileInfo(bucket, object, versionID, inclFreeVers, allParts)
-	if err != nil {
-		return FileInfo{}, err
-	}
-
-	if readData {
-		fileInfo.Data = xl.data.find(versionID)
-	}
-
-	return fileInfo, nil
+	return getFileInfo(ri.Buf, bucket, object, "", fileInfoOpts{
+		Data:             readData,
+		InclFreeVersions: inclFreeVers,
+		AllParts:         allParts,
+	})
 }
 
 func readAllRawFileInfo(ctx context.Context, disks []StorageAPI, bucket, object string, readData bool) ([]RawFileInfo, []error) {
@@ -756,8 +732,9 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	disks := er.getDisks()
 
 	ropts := ReadOptions{
-		ReadData: readData,
-		Healing:  false,
+		ReadData:         readData,
+		InclFreeVersions: opts.InclFreeVersions,
+		Healing:          false,
 	}
 
 	mrfCheck := make(chan FileInfo)
@@ -931,6 +908,8 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		}
 
 		rw.Lock()
+		// when its a versioned bucket and empty versionID - at totalResp == setDriveCount
+		// we must use rawFileInfo to resolve versions to figure out the latest version.
 		if opts.VersionID == "" && totalResp == er.setDriveCount {
 			fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(pickLatestQuorumFilesInfo(ctx,
 				rawArr, errs, bucket, object, readData, opts.InclFreeVersions, true))
@@ -938,7 +917,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 			fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(metaArr, errs)
 		}
 		rw.Unlock()
-		if err == nil && fi.InlineData() {
+		if err == nil && (fi.InlineData() || len(fi.Data) > 0) {
 			break
 		}
 	}
@@ -1422,7 +1401,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	writers := make([]io.Writer, len(onlineDisks))
 	var inlineBuffers []*bytes.Buffer
 	if shardFileSize >= 0 {
-		if !opts.Versioned && shardFileSize < inlineBlock {
+		if !opts.Versioned && shardFileSize <= inlineBlock {
 			inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
 		} else if shardFileSize < inlineBlock/8 {
 			inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
@@ -1430,7 +1409,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	} else {
 		// If compressed, use actual size to determine.
 		if sz := erasure.ShardFileSize(data.ActualSize()); sz > 0 {
-			if !opts.Versioned && sz < inlineBlock {
+			if !opts.Versioned && sz <= inlineBlock {
 				inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
 			} else if sz < inlineBlock/8 {
 				inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))

cmd/erasure-server-pool-decom.go

@@ -358,7 +358,7 @@ func (p *poolMeta) validate(pools []*erasureSets) (bool, error) {
 			update = true
 		}
 		if ok && pi.completed {
-			return false, fmt.Errorf("pool(%s) = %s is decommissioned, please remove from server command line", humanize.Ordinal(pi.position+1), k)
+			logger.LogIf(GlobalContext, "decommission", fmt.Errorf("pool(%s) = %s is decommissioned, please remove from server command line", humanize.Ordinal(pi.position+1), k))
 		}
 	}
 

cmd/erasure-server-pool-decom_test.go

@@ -134,7 +134,7 @@ func TestPoolMetaValidate(t *testing.T) {
 			meta:           nmeta1,
 			pools:          pools,
 			name:           "Invalid-Completed-Pool-Not-Removed",
-			expectedErr:    true,
+			expectedErr:    false,
 			expectedUpdate: false,
 		},
 		{

cmd/erasure-server-pool-rebalance.go

@@ -119,11 +119,8 @@ func (z *erasureServerPools) loadRebalanceMeta(ctx context.Context) error {
 	}
 
 	z.rebalMu.Lock()
-	if len(r.PoolStats) == len(z.serverPools) {
-		z.rebalMeta = r
-	} else {
-		z.updateRebalanceStats(ctx)
-	}
+	z.rebalMeta = r
+	z.updateRebalanceStats(ctx)
 	z.rebalMu.Unlock()
 
 	return nil
@@ -147,24 +144,16 @@ func (z *erasureServerPools) updateRebalanceStats(ctx context.Context) error {
 		}
 	}
 	if ok {
-		lock := z.serverPools[0].NewNSLock(minioMetaBucket, rebalMetaName)
-		lkCtx, err := lock.GetLock(ctx, globalOperationTimeout)
-		if err != nil {
-			rebalanceLogIf(ctx, fmt.Errorf("failed to acquire write lock on %s/%s: %w", minioMetaBucket, rebalMetaName, err))
-			return err
-		}
-		defer lock.Unlock(lkCtx)
-
-		ctx = lkCtx.Context()
-
-		noLockOpts := ObjectOptions{NoLock: true}
-		return z.rebalMeta.saveWithOpts(ctx, z.serverPools[0], noLockOpts)
+		return z.rebalMeta.save(ctx, z.serverPools[0])
 	}
 
 	return nil
 }
 
 func (z *erasureServerPools) findIndex(index int) int {
+	if z.rebalMeta == nil {
+		return 0
+	}
 	for i := 0; i < len(z.rebalMeta.PoolStats); i++ {
 		if i == index {
 			return index
@@ -277,6 +266,10 @@ func (z *erasureServerPools) bucketRebalanceDone(bucket string, poolIdx int) {
 	z.rebalMu.Lock()
 	defer z.rebalMu.Unlock()
 
+	if z.rebalMeta == nil {
+		return
+	}
+
 	ps := z.rebalMeta.PoolStats[poolIdx]
 	if ps == nil {
 		return
@@ -331,6 +324,10 @@ func (r *rebalanceMeta) loadWithOpts(ctx context.Context, store objectIO, opts O
 }
 
 func (r *rebalanceMeta) saveWithOpts(ctx context.Context, store objectIO, opts ObjectOptions) error {
+	if r == nil {
+		return nil
+	}
+
 	data := make([]byte, 4, r.Msgsize()+4)
 
 	// Initialize the header.
@@ -353,8 +350,15 @@ func (z *erasureServerPools) IsRebalanceStarted() bool {
 	z.rebalMu.RLock()
 	defer z.rebalMu.RUnlock()
 
-	if r := z.rebalMeta; r != nil {
-		if r.StoppedAt.IsZero() {
+	r := z.rebalMeta
+	if r == nil {
+		return false
+	}
+	if !r.StoppedAt.IsZero() {
+		return false
+	}
+	for _, ps := range r.PoolStats {
+		if ps.Participating && ps.Info.Status != rebalCompleted {
 			return true
 		}
 	}
@@ -369,7 +373,7 @@ func (z *erasureServerPools) IsPoolRebalancing(poolIndex int) bool {
 		if !r.StoppedAt.IsZero() {
 			return false
 		}
-		ps := z.rebalMeta.PoolStats[poolIndex]
+		ps := r.PoolStats[poolIndex]
 		return ps.Participating && ps.Info.Status == rebalStarted
 	}
 	return false
@@ -794,7 +798,9 @@ func (z *erasureServerPools) saveRebalanceStats(ctx context.Context, poolIdx int
 	case rebalSaveStoppedAt:
 		r.StoppedAt = time.Now()
 	case rebalSaveStats:
-		r.PoolStats[poolIdx] = z.rebalMeta.PoolStats[poolIdx]
+		if z.rebalMeta != nil {
+			r.PoolStats[poolIdx] = z.rebalMeta.PoolStats[poolIdx]
+		}
 	}
 	z.rebalMeta = r
 

cmd/erasure-server-pool.go

@@ -168,7 +168,7 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 
 	if !globalIsDistErasure {
 		globalLocalDrivesMu.Lock()
-		globalLocalDrives = localDrives
+		globalLocalDrivesMap = make(map[string]StorageAPI, len(localDrives))
 		for _, drive := range localDrives {
 			globalLocalDrivesMap[drive.Endpoint().String()] = drive
 		}
@@ -1526,14 +1526,12 @@ func (z *erasureServerPools) listObjectsGeneric(ctx context.Context, bucket, pre
 			loi.NextMarker = last.Name
 		}
 
-		if merged.lastSkippedEntry != "" {
-			if merged.lastSkippedEntry > loi.NextMarker {
-				// An object hidden by ILM was found during listing. Since the number of entries
-				// fetched from drives is limited, set IsTruncated to true to ask the s3 client
-				// to continue listing if it wishes in order to find if there is more objects.
-				loi.IsTruncated = true
-				loi.NextMarker = merged.lastSkippedEntry
-			}
+		if loi.IsTruncated && merged.lastSkippedEntry > loi.NextMarker {
+			// An object hidden by ILM was found during a truncated listing. Since the number of entries
+			// fetched from drives is limited by max-keys, we should use the last ILM filtered entry
+			// as a continuation token if it is lexially higher than the last visible object so that the
+			// next call of WalkDir() with the max-keys can reach new objects not seen previously.
+			loi.NextMarker = merged.lastSkippedEntry
 		}
 
 		if loi.NextMarker != "" {
@@ -2008,10 +2006,12 @@ func (z *erasureServerPools) ListBuckets(ctx context.Context, opts BucketOptions
 				if err != nil {
 					return nil, err
 				}
-				for i := range buckets {
-					createdAt, err := globalBucketMetadataSys.CreatedAt(buckets[i].Name)
-					if err == nil {
-						buckets[i].Created = createdAt
+				if !opts.NoMetadata {
+					for i := range buckets {
+						createdAt, err := globalBucketMetadataSys.CreatedAt(buckets[i].Name)
+						if err == nil {
+							buckets[i].Created = createdAt
+						}
 					}
 				}
 				return buckets, nil
@@ -2025,10 +2025,13 @@ func (z *erasureServerPools) ListBuckets(ctx context.Context, opts BucketOptions
 	if err != nil {
 		return nil, err
 	}
-	for i := range buckets {
-		createdAt, err := globalBucketMetadataSys.CreatedAt(buckets[i].Name)
-		if err == nil {
-			buckets[i].Created = createdAt
+
+	if !opts.NoMetadata {
+		for i := range buckets {
+			createdAt, err := globalBucketMetadataSys.CreatedAt(buckets[i].Name)
+			if err == nil {
+				buckets[i].Created = createdAt
+			}
 		}
 	}
 	return buckets, nil
@@ -2082,7 +2085,6 @@ func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts
 
 // Walk a bucket, optionally prefix recursively, until we have returned
 // all the contents of the provided bucket+prefix.
-// TODO: Note that most errors will result in a truncated listing.
 func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error {
 	if err := checkListObjsArgs(ctx, bucket, prefix, ""); err != nil {
 		xioutil.SafeClose(results)
@@ -2199,9 +2201,8 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 	// Convert and filter merged entries.
 	merged := make(chan metaCacheEntry, 100)
 	vcfg, _ := globalBucketVersioningSys.Get(bucket)
+	errCh := make(chan error, 1)
 	go func() {
-		defer cancelCause(nil)
-		defer xioutil.SafeClose(results)
 		sentErr := false
 		sendErr := func(err error) {
 			if !sentErr {
@@ -2212,6 +2213,15 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 				}
 			}
 		}
+		defer func() {
+			select {
+			case <-ctx.Done():
+				sendErr(ctx.Err())
+			default:
+			}
+			xioutil.SafeClose(results)
+			cancelCause(nil)
+		}()
 		send := func(oi ObjectInfo) bool {
 			select {
 			case results <- itemOrErr[ObjectInfo]{Item: oi}:
@@ -2266,12 +2276,16 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 				}
 			}
 		}
+		if err := <-errCh; err != nil {
+			sendErr(err)
+		}
 	}()
 	go func() {
+		defer close(errCh)
 		// Merge all entries from all disks.
 		// We leave quorum at 1, since entries are already resolved to have the desired quorum.
 		// mergeEntryChannels will close 'merged' channel upon completion or cancellation.
-		storageLogIf(ctx, mergeEntryChannels(ctx, entries, merged, 1))
+		errCh <- mergeEntryChannels(ctx, entries, merged, 1)
 	}()
 
 	return nil
@@ -2327,12 +2341,18 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 
 	var poolErrs [][]error
 	for idx, erasureSet := range z.serverPools {
+		if opts.Pool != nil && *opts.Pool != idx {
+			continue
+		}
 		if z.IsSuspended(idx) {
 			continue
 		}
 		errs := make([]error, len(erasureSet.sets))
 		var wg sync.WaitGroup
 		for idx, set := range erasureSet.sets {
+			if opts.Set != nil && *opts.Set != idx {
+				continue
+			}
 			wg.Add(1)
 			go func(idx int, set *erasureObjects) {
 				defer wg.Done()
@@ -2425,6 +2445,7 @@ const (
 type HealthOptions struct {
 	Maintenance    bool
 	DeploymentType string
+	NoLogging      bool
 }
 
 // HealthResult returns the current state of the system, also
@@ -2449,6 +2470,24 @@ type HealthResult struct {
 	UsingDefaults bool
 }
 
+func (hr HealthResult) String() string {
+	var str strings.Builder
+	for i, es := range hr.ESHealth {
+		str.WriteString("(Pool: ")
+		str.WriteString(strconv.Itoa(es.PoolID))
+		str.WriteString(" Set: ")
+		str.WriteString(strconv.Itoa(es.SetID))
+		str.WriteString(" Healthy: ")
+		str.WriteString(strconv.FormatBool(es.Healthy))
+		if i == 0 {
+			str.WriteString(")")
+		} else {
+			str.WriteString(") | ")
+		}
+	}
+	return str.String()
+}
+
 // Health - returns current status of the object layer health,
 // provides if write access exists across sets, additionally
 // can be used to query scenarios if health may be lost
@@ -2566,18 +2605,18 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 			})
 
 			healthy := erasureSetUpCount[poolIdx][setIdx].online >= poolWriteQuorums[poolIdx]
-			if !healthy {
+			if !healthy && !opts.NoLogging {
 				storageLogIf(logger.SetReqInfo(ctx, reqInfo),
-					fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
-						poolIdx, setIdx, poolWriteQuorums[poolIdx]), logger.FatalKind)
+					fmt.Errorf("Write quorum could not be established on pool: %d, set: %d, expected write quorum: %d, drives-online: %d",
+						poolIdx, setIdx, poolWriteQuorums[poolIdx], erasureSetUpCount[poolIdx][setIdx].online), logger.FatalKind)
 			}
 			result.Healthy = result.Healthy && healthy
 
 			healthyRead := erasureSetUpCount[poolIdx][setIdx].online >= poolReadQuorums[poolIdx]
-			if !healthyRead {
+			if !healthyRead && !opts.NoLogging {
 				storageLogIf(logger.SetReqInfo(ctx, reqInfo),
-					fmt.Errorf("Read quorum may be lost on pool: %d, set: %d, expected read quorum: %d",
-						poolIdx, setIdx, poolReadQuorums[poolIdx]))
+					fmt.Errorf("Read quorum could not be established on pool: %d, set: %d, expected read quorum: %d, drives-online: %d",
+						poolIdx, setIdx, poolReadQuorums[poolIdx], erasureSetUpCount[poolIdx][setIdx].online))
 			}
 			result.HealthyRead = result.HealthyRead && healthyRead
 		}

cmd/erasure-sets.go

@@ -36,7 +36,6 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/dsync"
-	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/console"
 	"github.com/minio/pkg/v3/sync/errgroup"
@@ -80,10 +79,6 @@ type erasureSets struct {
 
 	poolIndex int
 
-	// A channel to send the set index to the MRF when
-	// any disk belonging to that set is connected
-	setReconnectEvent chan int
-
 	// Distribution algorithm of choice.
 	distributionAlgo string
 	deploymentID     [16]byte
@@ -125,13 +120,6 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 
 	format, err := loadFormatErasure(disk, false)
 	if err != nil {
-		if errors.Is(err, errUnformattedDisk) {
-			info, derr := disk.DiskInfo(context.TODO(), DiskInfoOptions{})
-			if derr != nil && info.RootDisk {
-				disk.Close()
-				return nil, nil, fmt.Errorf("Drive: %s is a root drive", disk)
-			}
-		}
 		disk.Close()
 		return nil, nil, fmt.Errorf("Drive: %s returned %w", disk, err) // make sure to '%w' to wrap the error
 	}
@@ -201,7 +189,7 @@ func (s *erasureSets) Legacy() (ok bool) {
 
 // connectDisks - attempt to connect all the endpoints, loads format
 // and re-arranges the disks in proper position.
-func (s *erasureSets) connectDisks() {
+func (s *erasureSets) connectDisks(log bool) {
 	defer func() {
 		s.lastConnectDisksOpTime = time.Now()
 	}()
@@ -235,8 +223,10 @@ func (s *erasureSets) connectDisks() {
 			if err != nil {
 				if endpoint.IsLocal && errors.Is(err, errUnformattedDisk) {
 					globalBackgroundHealState.pushHealLocalDisks(endpoint)
-				} else {
-					printEndpointError(endpoint, err, true)
+				} else if !errors.Is(err, errDriveIsRoot) {
+					if log {
+						printEndpointError(endpoint, err, true)
+					}
 				}
 				return
 			}
@@ -272,13 +262,7 @@ func (s *erasureSets) connectDisks() {
 				if globalIsDistErasure {
 					globalLocalSetDrives[s.poolIndex][setIndex][diskIndex] = disk
 				}
-				for i, ldisk := range globalLocalDrives {
-					_, k, l := ldisk.GetDiskLoc()
-					if k == setIndex && l == diskIndex {
-						globalLocalDrives[i] = disk
-						break
-					}
-				}
+				globalLocalDrivesMap[disk.Endpoint().String()] = disk
 				globalLocalDrivesMu.Unlock()
 			}
 			s.erasureDisksMu.Unlock()
@@ -297,7 +281,7 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 	time.Sleep(time.Duration(r.Float64() * float64(time.Second)))
 
 	// Pre-emptively connect the disks if possible.
-	s.connectDisks()
+	s.connectDisks(false)
 
 	monitor := time.NewTimer(monitorInterval)
 	defer monitor.Stop()
@@ -311,7 +295,7 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 				console.Debugln("running drive monitoring")
 			}
 
-			s.connectDisks()
+			s.connectDisks(true)
 
 			// Reset the timer for next interval
 			monitor.Reset(monitorInterval)
@@ -380,7 +364,6 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 		setDriveCount:      setDriveCount,
 		defaultParityCount: defaultParityCount,
 		format:             format,
-		setReconnectEvent:  make(chan int),
 		distributionAlgo:   format.Erasure.DistributionAlgo,
 		deploymentID:       uuid.MustParse(format.ID),
 		poolIndex:          poolIdx,
@@ -542,7 +525,7 @@ func (s *erasureSets) cleanupStaleUploads(ctx context.Context) {
 					if set == nil {
 						return
 					}
-					set.cleanupStaleUploads(ctx, globalAPIConfig.getStaleUploadsExpiry())
+					set.cleanupStaleUploads(ctx)
 				}(set)
 			}
 			wg.Wait()
@@ -662,14 +645,6 @@ func (s *erasureSets) Shutdown(ctx context.Context) error {
 			return err
 		}
 	}
-	select {
-	case _, ok := <-s.setReconnectEvent:
-		if ok {
-			xioutil.SafeClose(s.setReconnectEvent)
-		}
-	default:
-		xioutil.SafeClose(s.setReconnectEvent)
-	}
 	return nil
 }
 
@@ -1154,13 +1129,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 					if globalIsDistErasure {
 						globalLocalSetDrives[s.poolIndex][m][n] = disk
 					}
-					for i, ldisk := range globalLocalDrives {
-						_, k, l := ldisk.GetDiskLoc()
-						if k == m && l == n {
-							globalLocalDrives[i] = disk
-							break
-						}
-					}
+					globalLocalDrivesMap[disk.Endpoint().String()] = disk
 					globalLocalDrivesMu.Unlock()
 				}
 			}

cmd/erasure.go

@@ -102,6 +102,8 @@ func diskErrToDriveState(err error) (state string) {
 		state = madmin.DriveStatePermission
 	case errors.Is(err, errFaultyDisk):
 		state = madmin.DriveStateFaulty
+	case errors.Is(err, errDriveIsRoot):
+		state = madmin.DriveStateRootMount
 	case err == nil:
 		state = madmin.DriveStateOk
 	default:

cmd/event-notification.go

@@ -49,23 +49,18 @@ func NewEventNotifier(ctx context.Context) *EventNotifier {
 }
 
 // GetARNList - returns available ARNs.
-func (evnot *EventNotifier) GetARNList(onlyActive bool) []string {
+func (evnot *EventNotifier) GetARNList() []string {
 	arns := []string{}
 	if evnot == nil {
 		return arns
 	}
 	region := globalSite.Region()
-	for targetID, target := range evnot.targetList.TargetMap() {
+	for targetID := range evnot.targetList.TargetMap() {
 		// httpclient target is part of ListenNotification
 		// which doesn't need to be listed as part of the ARN list
 		// This list is only meant for external targets, filter
 		// this out pro-actively.
 		if !strings.HasPrefix(targetID.ID, "httpclient+") {
-			if onlyActive {
-				if _, err := target.IsActive(); err != nil {
-					continue
-				}
-			}
 			arns = append(arns, targetID.ToARN(region).String())
 		}
 	}
@@ -74,7 +69,7 @@ func (evnot *EventNotifier) GetARNList(onlyActive bool) []string {
 }
 
 // Loads notification policies for all buckets into EventNotifier.
-func (evnot *EventNotifier) set(bucket BucketInfo, meta BucketMetadata) {
+func (evnot *EventNotifier) set(bucket string, meta BucketMetadata) {
 	config := meta.notificationConfig
 	if config == nil {
 		return
@@ -86,7 +81,7 @@ func (evnot *EventNotifier) set(bucket BucketInfo, meta BucketMetadata) {
 			internalLogIf(GlobalContext, err)
 		}
 	}
-	evnot.AddRulesMap(bucket.Name, config.ToRulesMap())
+	evnot.AddRulesMap(bucket, config.ToRulesMap())
 }
 
 // Targets returns all the registered targets

cmd/fmt-gen.go

@@ -0,0 +1,120 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build fmtgen
+
+package cmd
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/klauspost/compress/zip"
+	"github.com/minio/cli"
+)
+
+var fmtGenFlags = []cli.Flag{
+	cli.IntFlag{
+		Name:  "parity",
+		Usage: "specify erasure code parity",
+	},
+	cli.StringFlag{
+		Name:  "deployment-id",
+		Usage: "deployment-id of the MinIO cluster for which format.json is needed",
+	},
+	cli.StringFlag{
+		Name:   "address",
+		Value:  ":" + GlobalMinioDefaultPort,
+		Usage:  "bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname",
+		EnvVar: "MINIO_ADDRESS",
+	},
+}
+
+var fmtGenCmd = cli.Command{
+	Name:   "fmt-gen",
+	Usage:  "Generate format.json files for an erasure server pool",
+	Flags:  append(fmtGenFlags, GlobalFlags...),
+	Action: fmtGenMain,
+	CustomHelpTemplate: `NAME:
+  {{.HelpName}} - {{.Usage}}
+
+USAGE:
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR1 [DIR2..]
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR{1...64}
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR{1...64} DIR{65...128}
+
+DIR:
+  DIR points to a directory on a filesystem. When you want to combine
+  multiple drives into a single large system, pass one directory per
+  filesystem separated by space. You may also use a '...' convention
+  to abbreviate the directory arguments. Remote directories in a
+  distributed setup are encoded as HTTP(s) URIs.
+{{if .VisibleFlags}}
+FLAGS:
+  {{range .VisibleFlags}}{{.}}
+  {{end}}{{end}}
+EXAMPLES:
+  1. Generate format.json.zip containing format.json files for all drives in a distributed MinIO server pool of 32 nodes with 32 drives each.
+     {{.Prompt}} {{.HelpName}} http://node{1...32}.example.com/mnt/export{1...32}
+
+`,
+}
+
+func fmtGenMain(ctxt *cli.Context) {
+	deploymentID := ctxt.String("deployment-id")
+	err := buildServerCtxt(ctxt, &globalServerCtxt)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	handleCommonArgs(globalServerCtxt)
+	pools, _, err := createServerEndpoints(globalMinioAddr, globalServerCtxt.Layout.pools, globalServerCtxt.Layout.legacy)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	zipFile, err := os.Create("format.json.zip")
+	if err != nil {
+		log.Fatalf("failed to create format.json.zip: %v", err)
+	}
+	defer zipFile.Close()
+	fmtZipW := zip.NewWriter(zipFile)
+	defer fmtZipW.Close()
+	for _, pool := range pools { // for each pool
+		setCount, setDriveCount := pool.SetCount, pool.DrivesPerSet
+		format := newFormatErasureV3(setCount, setDriveCount)
+		format.ID = deploymentID
+		for i := 0; i < setCount; i++ { // for each erasure set
+			for j := 0; j < setDriveCount; j++ {
+				newFormat := format.Clone()
+				newFormat.Erasure.This = format.Erasure.Sets[i][j]
+				if deploymentID != "" {
+					newFormat.ID = deploymentID
+				}
+				drive := pool.Endpoints[i*setDriveCount+j]
+				fmtBytes, err := json.Marshal(newFormat)
+				if err != nil {
+					//nolint:gocritic
+					log.Fatalf("failed to marshal format.json for %s: %v", drive.String(), err)
+				}
+				fmtJSON := filepath.Join(drive.Host, drive.Path, minioMetaBucket, "format.json")
+				embedFileInZip(fmtZipW, fmtJSON, fmtBytes, 0o600)
+			}
+		}
+	}
+}

cmd/fmt-gen_notsupported.go

@@ -1,6 +1,6 @@
-// Copyright (c) 2015-2022 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
-// This file is part of MinIO Object Storage stack
+// # This file is part of MinIO Object Storage stack
 //
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
@@ -15,9 +15,10 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-//go:build go1.19
-// +build go1.19
+//go:build !fmtgen
 
-package fips
+package cmd
 
-const go19 = true
+import "github.com/minio/cli"
+
+var fmtGenCmd cli.Command

cmd/generic-handlers.go

@@ -247,11 +247,14 @@ func guessIsRPCReq(req *http.Request) bool {
 	if req == nil {
 		return false
 	}
-	if req.Method == http.MethodGet && req.URL != nil && req.URL.Path == grid.RoutePath {
-		return true
+	if req.Method == http.MethodGet && req.URL != nil {
+		switch req.URL.Path {
+		case grid.RoutePath, grid.RouteLockPath:
+			return true
+		}
 	}
 
-	return req.Method == http.MethodPost &&
+	return (req.Method == http.MethodPost || req.Method == http.MethodGet) &&
 		strings.HasPrefix(req.URL.Path, minioReservedBucketPath+SlashSeparator)
 }
 
@@ -469,7 +472,7 @@ func setBucketForwardingMiddleware(h http.Handler) http.Handler {
 		}
 		if globalDNSConfig == nil || !globalBucketFederation ||
 			guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
-			guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) {
+			guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) || isKMSReq(r) {
 			h.ServeHTTP(w, r)
 			return
 		}

cmd/generic-handlers_test.go

@@ -51,9 +51,10 @@ func TestGuessIsRPC(t *testing.T) {
 	r = &http.Request{
 		Proto:  "HTTP/1.1",
 		Method: http.MethodGet,
+		URL:    u,
 	}
-	if guessIsRPCReq(r) {
-		t.Fatal("Test shouldn't report as net/rpc for a non net/rpc request.")
+	if !guessIsRPCReq(r) {
+		t.Fatal("Test shouldn't fail for a possible net/rpc request.")
 	}
 	r = &http.Request{
 		Proto:  "HTTP/1.1",
@@ -63,6 +64,14 @@ func TestGuessIsRPC(t *testing.T) {
 	if !guessIsRPCReq(r) {
 		t.Fatal("Grid RPC path not detected")
 	}
+	r = &http.Request{
+		Proto:  "HTTP/1.1",
+		Method: http.MethodGet,
+		URL:    &url.URL{Path: grid.RouteLockPath},
+	}
+	if !guessIsRPCReq(r) {
+		t.Fatal("Grid RPC path not detected")
+	}
 }
 
 var isHTTPHeaderSizeTooLargeTests = []struct {

cmd/global-heal.go

@@ -150,6 +150,11 @@ type healEntryResult struct {
 
 // healErasureSet lists and heals all objects in a specific erasure set
 func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string, tracker *healingTracker) error {
+	bgSeq, found := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
+	if !found {
+		return errors.New("no local healing sequence initialized, unable to heal the drive")
+	}
+
 	scanMode := madmin.HealNormalScan
 
 	// Make sure to copy since `buckets slice`
@@ -163,8 +168,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 	}
 
 	for _, bucket := range healBuckets {
-		_, err := objAPI.HealBucket(ctx, bucket, madmin.HealOpts{ScanMode: scanMode})
-		if err != nil {
+		if err := bgSeq.healBucket(objAPI, bucket, true); err != nil {
 			// Log bucket healing error if any, we shall retry again.
 			healingLogIf(ctx, err)
 		}
@@ -261,9 +265,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		tracker.setBucket(bucket)
 		// Heal current bucket again in case if it is failed
 		// in the beginning of erasure set healing
-		if _, err := objAPI.HealBucket(ctx, bucket, madmin.HealOpts{
-			ScanMode: scanMode,
-		}); err != nil {
+		if err := bgSeq.healBucket(objAPI, bucket, true); err != nil {
 			// Set this such that when we return this function
 			// we let the caller retry this disk again for the
 			// buckets that failed healing.
@@ -362,6 +364,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				}
 				return false
 			case results <- result:
+				bgSeq.countScanned(madmin.HealItemObject)
 				return true
 			}
 		}
@@ -412,8 +415,10 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 						return
 					}
 					result = healEntryFailure(0)
+					bgSeq.countFailed(madmin.HealItemObject)
 					healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, entry.name, err))
 				} else {
+					bgSeq.countHealed(madmin.HealItemObject)
 					result = healEntrySuccess(uint64(res.ObjectSize))
 				}
 
@@ -437,6 +442,8 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					continue
 				}
 
+				var versionHealed bool
+
 				res, err := er.HealObject(ctx, bucket, encodedEntryName,
 					version.VersionID, madmin.HealOpts{
 						ScanMode: scanMode,
@@ -449,15 +456,24 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 						versionNotFound++
 						continue
 					}
-					// If not deleted, assume they failed.
+				} else {
+					// Look for the healing results
+					if res.After.Drives[tracker.DiskIndex].State == madmin.DriveStateOk {
+						versionHealed = true
+					}
+				}
+
+				if versionHealed {
+					bgSeq.countHealed(madmin.HealItemObject)
+					result = healEntrySuccess(uint64(version.Size))
+				} else {
+					bgSeq.countFailed(madmin.HealItemObject)
 					result = healEntryFailure(uint64(version.Size))
 					if version.VersionID != "" {
 						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
 					} else {
 						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, version.Name, err))
 					}
-				} else {
-					result = healEntrySuccess(uint64(res.ObjectSize))
 				}
 
 				if !send(result) {
@@ -505,7 +521,11 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				jt.Take()
 				go healEntry(bucket, *entry)
 			},
-			finished: nil,
+			finished: func(errs []error) {
+				if countErrs(errs, nil) != len(errs) {
+					retErr = fmt.Errorf("one or more errors reported during listing: %v", errors.Join(errs...))
+				}
+			},
 		})
 		jt.Wait() // synchronize all the concurrent heal jobs
 		if err != nil {
@@ -513,7 +533,10 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			// we let the caller retry this disk again for the
 			// buckets it failed to list.
 			retErr = err
-			healingLogIf(ctx, err)
+		}
+
+		if retErr != nil {
+			healingLogIf(ctx, fmt.Errorf("listing failed with: %v on bucket: %v", retErr, bucket))
 			continue
 		}
 

cmd/globals.go

@@ -163,7 +163,6 @@ type serverCtxt struct {
 	MemLimit uint64
 
 	UserTimeout         time.Duration
-	ShutdownTimeout     time.Duration
 	IdleTimeout         time.Duration
 	ReadHeaderTimeout   time.Duration
 	MaxIdleConnsPerHost int
@@ -311,6 +310,7 @@ var (
 	globalBootTime = UTCNow()
 
 	globalActiveCred         auth.Credentials
+	globalNodeAuthToken      string
 	globalSiteReplicatorCred siteReplicatorCred
 
 	// Captures if root credentials are set via ENV.
@@ -414,10 +414,9 @@ var (
 	globalServiceFreezeCnt int32
 	globalServiceFreezeMu  sync.Mutex // Updates.
 
-	// List of local drives to this node, this is only set during server startup,
-	// and is only mutated by HealFormat. Hold globalLocalDrivesMu to access.
-	globalLocalDrives    []StorageAPI
-	globalLocalDrivesMap = make(map[string]StorageAPI)
+	// Map of local drives to this node, this is set during server startup,
+	// disk reconnect and mutated by HealFormat. Hold globalLocalDrivesMu to access.
+	globalLocalDrivesMap map[string]StorageAPI
 	globalLocalDrivesMu  sync.RWMutex
 
 	globalDriveMonitoring = env.Get("_MINIO_DRIVE_ACTIVE_MONITORING", config.EnableOn) == config.EnableOn
@@ -450,8 +449,8 @@ var (
 	// dynamic sleeper for multipart expiration routine
 	deleteMultipartCleanupSleeper = newDynamicSleeper(5, 25*time.Millisecond, false)
 
-	// Is _MINIO_DISABLE_API_FREEZE_ON_BOOT set?
-	globalDisableFreezeOnBoot bool
+	// Is MINIO_SYNC_BOOT set?
+	globalEnableSyncBoot bool
 
 	// Contains NIC interface name used for internode communication
 	globalInternodeInterface     string

cmd/grid.go

@@ -31,9 +31,15 @@ import (
 // globalGrid is the global grid manager.
 var globalGrid atomic.Pointer[grid.Manager]
 
+// globalLockGrid is the global lock grid manager.
+var globalLockGrid atomic.Pointer[grid.Manager]
+
 // globalGridStart is a channel that will block startup of grid connections until closed.
 var globalGridStart = make(chan struct{})
 
+// globalLockGridStart is a channel that will block startup of lock grid connections until closed.
+var globalLockGridStart = make(chan struct{})
+
 func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error {
 	hosts, local := eps.GridHosts()
 	lookupHost := globalDNSCache.LookupHost
@@ -41,21 +47,24 @@ func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error {
 		// Pass Dialer for websocket grid, make sure we do not
 		// provide any DriveOPTimeout() function, as that is not
 		// useful over persistent connections.
-		Dialer:       grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions.ForWebsocket()))),
+		Dialer: grid.ConnectWS(
+			grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions.ForWebsocket()))),
+			newCachedAuthToken(),
+			&tls.Config{
+				RootCAs:          globalRootCAs,
+				CipherSuites:     fips.TLSCiphers(),
+				CurvePreferences: fips.TLSCurveIDs(),
+			}),
 		Local:        local,
 		Hosts:        hosts,
-		AddAuth:      newCachedAuthToken(),
-		AuthRequest:  storageServerRequestValidate,
+		AuthToken:    validateStorageRequestToken,
+		AuthFn:       newCachedAuthToken(),
 		BlockConnect: globalGridStart,
-		TLSConfig: &tls.Config{
-			RootCAs:          globalRootCAs,
-			CipherSuites:     fips.TLSCiphers(),
-			CurvePreferences: fips.TLSCurveIDs(),
-		},
 		// Record incoming and outgoing bytes.
-		Incoming: globalConnStats.incInternodeInputBytes,
-		Outgoing: globalConnStats.incInternodeOutputBytes,
-		TraceTo:  globalTrace,
+		Incoming:  globalConnStats.incInternodeInputBytes,
+		Outgoing:  globalConnStats.incInternodeOutputBytes,
+		TraceTo:   globalTrace,
+		RoutePath: grid.RoutePath,
 	})
 	if err != nil {
 		return err
@@ -63,3 +72,36 @@ func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error {
 	globalGrid.Store(g)
 	return nil
 }
+
+func initGlobalLockGrid(ctx context.Context, eps EndpointServerPools) error {
+	hosts, local := eps.GridHosts()
+	lookupHost := globalDNSCache.LookupHost
+	g, err := grid.NewManager(ctx, grid.ManagerOptions{
+		// Pass Dialer for websocket grid, make sure we do not
+		// provide any DriveOPTimeout() function, as that is not
+		// useful over persistent connections.
+		Dialer: grid.ConnectWSWithRoutePath(
+			grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions.ForWebsocket()))),
+			newCachedAuthToken(),
+			&tls.Config{
+				RootCAs:          globalRootCAs,
+				CipherSuites:     fips.TLSCiphers(),
+				CurvePreferences: fips.TLSCurveIDs(),
+			}, grid.RouteLockPath),
+		Local:        local,
+		Hosts:        hosts,
+		AuthToken:    validateStorageRequestToken,
+		AuthFn:       newCachedAuthToken(),
+		BlockConnect: globalGridStart,
+		// Record incoming and outgoing bytes.
+		Incoming:  globalConnStats.incInternodeInputBytes,
+		Outgoing:  globalConnStats.incInternodeOutputBytes,
+		TraceTo:   globalTrace,
+		RoutePath: grid.RouteLockPath,
+	})
+	if err != nil {
+		return err
+	}
+	globalLockGrid.Store(g)
+	return nil
+}

cmd/handler-api.go

@@ -39,14 +39,15 @@ import (
 type apiConfig struct {
 	mu sync.RWMutex
 
-	requestsDeadline      time.Duration
-	requestsPool          chan struct{}
-	clusterDeadline       time.Duration
-	listQuorum            string
-	corsAllowOrigins      []string
-	replicationPriority   string
-	replicationMaxWorkers int
-	transitionWorkers     int
+	requestsDeadline       time.Duration
+	requestsPool           chan struct{}
+	clusterDeadline        time.Duration
+	listQuorum             string
+	corsAllowOrigins       []string
+	replicationPriority    string
+	replicationMaxWorkers  int
+	replicationMaxLWorkers int
+	transitionWorkers      int
 
 	staleUploadsExpiry          time.Duration
 	staleUploadsCleanupInterval time.Duration
@@ -170,11 +171,12 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) {
 	}
 	t.listQuorum = listQuorum
 	if globalReplicationPool != nil &&
-		(cfg.ReplicationPriority != t.replicationPriority || cfg.ReplicationMaxWorkers != t.replicationMaxWorkers) {
-		globalReplicationPool.ResizeWorkerPriority(cfg.ReplicationPriority, cfg.ReplicationMaxWorkers)
+		(cfg.ReplicationPriority != t.replicationPriority || cfg.ReplicationMaxWorkers != t.replicationMaxWorkers || cfg.ReplicationMaxLWorkers != t.replicationMaxLWorkers) {
+		globalReplicationPool.ResizeWorkerPriority(cfg.ReplicationPriority, cfg.ReplicationMaxWorkers, cfg.ReplicationMaxLWorkers)
 	}
 	t.replicationPriority = cfg.ReplicationPriority
 	t.replicationMaxWorkers = cfg.ReplicationMaxWorkers
+	t.replicationMaxLWorkers = cfg.ReplicationMaxLWorkers
 
 	// N B api.transition_workers will be deprecated
 	if globalTransitionState != nil {
@@ -381,14 +383,16 @@ func (t *apiConfig) getReplicationOpts() replicationPoolOpts {
 
 	if t.replicationPriority == "" {
 		return replicationPoolOpts{
-			Priority:   "auto",
-			MaxWorkers: WorkerMaxLimit,
+			Priority:    "auto",
+			MaxWorkers:  WorkerMaxLimit,
+			MaxLWorkers: LargeWorkerCount,
 		}
 	}
 
 	return replicationPoolOpts{
-		Priority:   t.replicationPriority,
-		MaxWorkers: t.replicationMaxWorkers,
+		Priority:    t.replicationPriority,
+		MaxWorkers:  t.replicationMaxWorkers,
+		MaxLWorkers: t.replicationMaxLWorkers,
 	}
 }
 

cmd/healthcheck-handler.go

@@ -29,14 +29,35 @@ import (
 
 const unavailable = "offline"
 
-// ClusterCheckHandler returns if the server is ready for requests.
-func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ClusterCheckHandler")
-
+func checkHealth(w http.ResponseWriter) ObjectLayer {
 	objLayer := newObjectLayerFn()
 	if objLayer == nil {
 		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		return nil
+	}
+
+	if !globalBucketMetadataSys.Initialized() {
+		w.Header().Set(xhttp.MinIOServerStatus, "bucket-metadata-offline")
+		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		return nil
+	}
+
+	if !globalIAMSys.Initialized() {
+		w.Header().Set(xhttp.MinIOServerStatus, "iam-offline")
+		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		return nil
+	}
+
+	return objLayer
+}
+
+// ClusterCheckHandler returns if the server is ready for requests.
+func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ClusterCheckHandler")
+
+	objLayer := checkHealth(w)
+	if objLayer == nil {
 		return
 	}
 
@@ -72,10 +93,8 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ClusterReadCheckHandler")
 
-	objLayer := newObjectLayerFn()
+	objLayer := checkHealth(w)
 	if objLayer == nil {
-		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
-		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
 

cmd/http-stats.go

@@ -27,10 +27,6 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-const (
-	apiGetObject = "GetObject"
-)
-
 // connStats - Network statistics
 // Count total input/output transferred bytes during
 // the server's life.
@@ -132,7 +128,7 @@ func (bh *bucketHTTPStats) updateHTTPStats(bucket, api string, w *xhttp.Response
 		return
 	}
 
-	if w != nil && api == apiGetObject {
+	if w != nil {
 		// Increment the prometheus http request response histogram with API, Bucket
 		bucketHTTPRequestsDuration.With(prometheus.Labels{
 			"api":    api,
@@ -437,9 +433,7 @@ func (st *HTTPStats) updateStats(api string, w *xhttp.ResponseRecorder) {
 	st.totalS3Requests.Inc(api)
 
 	// Increment the prometheus http request response histogram with appropriate label
-	if api == apiGetObject {
-		httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(w.TimeToFirstByte.Seconds())
-	}
+	httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(w.TimeToFirstByte.Seconds())
 
 	code := w.StatusCode
 

cmd/iam-object-store.go

@@ -439,23 +439,44 @@ func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (res map
 	return res, nil
 }
 
+const (
+	maxIAMLoadOpTime = 5 * time.Second
+)
+
 // Assumes cache is locked by caller.
-func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iamCache) error {
+func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iamCache, firstTime bool) error {
+	bootstrapTraceMsgFirstTime := func(s string) {
+		if firstTime {
+			bootstrapTraceMsg(s)
+		}
+	}
+
 	if iamOS.objAPI == nil {
 		return errServerNotInitialized
 	}
 
-	bootstrapTraceMsg("loading all IAM items")
+	bootstrapTraceMsgFirstTime("loading all IAM items")
 
+	setDefaultCannedPolicies(cache.iamPolicyDocsMap)
+
+	listStartTime := UTCNow()
 	listedConfigItems, err := iamOS.listAllIAMConfigItems(ctx)
 	if err != nil {
 		return fmt.Errorf("unable to list IAM data: %w", err)
 	}
+	if took := time.Since(listStartTime); took > maxIAMLoadOpTime {
+		var s strings.Builder
+		for k, v := range listedConfigItems {
+			s.WriteString(fmt.Sprintf("    %s: %d items\n", k, len(v)))
+		}
+		logger.Info("listAllIAMConfigItems took %.2fs with contents:\n%s", took.Seconds(), s.String())
+	}
 
 	// Loads things in the same order as `LoadIAMCache()`
 
-	bootstrapTraceMsg("loading policy documents")
+	bootstrapTraceMsgFirstTime("loading policy documents")
 
+	policyLoadStartTime := UTCNow()
 	policiesList := listedConfigItems[policiesListKey]
 	for _, item := range policiesList {
 		policyName := path.Dir(item)
@@ -463,58 +484,88 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 			return fmt.Errorf("unable to load the policy doc `%s`: %w", policyName, err)
 		}
 	}
-	setDefaultCannedPolicies(cache.iamPolicyDocsMap)
+	if took := time.Since(policyLoadStartTime); took > maxIAMLoadOpTime {
+		logger.Info("Policy docs load took %.2fs (for %d items)", took.Seconds(), len(policiesList))
+	}
 
 	if iamOS.usersSysType == MinIOUsersSysType {
-		bootstrapTraceMsg("loading regular IAM users")
+		bootstrapTraceMsgFirstTime("loading regular IAM users")
+		regUsersLoadStartTime := UTCNow()
 		regUsersList := listedConfigItems[usersListKey]
 		for _, item := range regUsersList {
 			userName := path.Dir(item)
 			if err := iamOS.loadUser(ctx, userName, regUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
-				return fmt.Errorf("unable to load the user `%s`: %w", userName, err)
+				return fmt.Errorf("unable to load the user: %w", err)
 			}
 		}
+		if took := time.Since(regUsersLoadStartTime); took > maxIAMLoadOpTime {
+			actualLoaded := len(cache.iamUsersMap)
+			logger.Info("Reg. users load took %.2fs (for %d items with %d expired items)", took.Seconds(),
+				len(regUsersList), len(regUsersList)-actualLoaded)
+		}
 
-		bootstrapTraceMsg("loading regular IAM groups")
+		bootstrapTraceMsgFirstTime("loading regular IAM groups")
+		groupsLoadStartTime := UTCNow()
 		groupsList := listedConfigItems[groupsListKey]
 		for _, item := range groupsList {
 			group := path.Dir(item)
 			if err := iamOS.loadGroup(ctx, group, cache.iamGroupsMap); err != nil && err != errNoSuchGroup {
-				return fmt.Errorf("unable to load the group `%s`: %w", group, err)
+				return fmt.Errorf("unable to load the group: %w", err)
 			}
 		}
+		if took := time.Since(groupsLoadStartTime); took > maxIAMLoadOpTime {
+			logger.Info("Groups load took %.2fs (for %d items)", took.Seconds(), len(groupsList))
+		}
 	}
 
-	bootstrapTraceMsg("loading user policy mapping")
+	bootstrapTraceMsgFirstTime("loading user policy mapping")
+	userPolicyMappingLoadStartTime := UTCNow()
 	userPolicyMappingsList := listedConfigItems[policyDBUsersListKey]
 	for _, item := range userPolicyMappingsList {
 		userName := strings.TrimSuffix(item, ".json")
 		if err := iamOS.loadMappedPolicy(ctx, userName, regUser, false, cache.iamUserPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return fmt.Errorf("unable to load the policy mapping for the user `%s`: %w", userName, err)
+			return fmt.Errorf("unable to load the policy mapping for the user: %w", err)
 		}
 	}
+	if took := time.Since(userPolicyMappingLoadStartTime); took > maxIAMLoadOpTime {
+		logger.Info("User policy mappings load took %.2fs (for %d items)", took.Seconds(), len(userPolicyMappingsList))
+	}
 
-	bootstrapTraceMsg("loading group policy mapping")
+	bootstrapTraceMsgFirstTime("loading group policy mapping")
+	groupPolicyMappingLoadStartTime := UTCNow()
 	groupPolicyMappingsList := listedConfigItems[policyDBGroupsListKey]
 	for _, item := range groupPolicyMappingsList {
 		groupName := strings.TrimSuffix(item, ".json")
 		if err := iamOS.loadMappedPolicy(ctx, groupName, regUser, true, cache.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return fmt.Errorf("unable to load the policy mapping for the group `%s`: %w", groupName, err)
+			return fmt.Errorf("unable to load the policy mapping for the group: %w", err)
 		}
 	}
+	if took := time.Since(groupPolicyMappingLoadStartTime); took > maxIAMLoadOpTime {
+		logger.Info("Group policy mappings load took %.2fs (for %d items)", took.Seconds(), len(groupPolicyMappingsList))
+	}
 
-	bootstrapTraceMsg("loading service accounts")
+	bootstrapTraceMsgFirstTime("loading service accounts")
+	svcAccLoadStartTime := UTCNow()
 	svcAccList := listedConfigItems[svcAccListKey]
 	svcUsersMap := make(map[string]UserIdentity, len(svcAccList))
 	for _, item := range svcAccList {
 		userName := path.Dir(item)
 		if err := iamOS.loadUser(ctx, userName, svcUser, svcUsersMap); err != nil && err != errNoSuchUser {
-			return fmt.Errorf("unable to load the service account `%s`: %w", userName, err)
+			return fmt.Errorf("unable to load the service account: %w", err)
 		}
 	}
+	if took := time.Since(svcAccLoadStartTime); took > maxIAMLoadOpTime {
+		logger.Info("Service accounts load took %.2fs (for %d items with %d expired items)", took.Seconds(),
+			len(svcAccList), len(svcAccList)-len(svcUsersMap))
+	}
+
+	bootstrapTraceMsg("loading STS account policy mapping")
+	stsPolicyMappingLoadStartTime := UTCNow()
+	var stsPolicyMappingsCount int
 	for _, svcAcc := range svcUsersMap {
 		svcParent := svcAcc.Credentials.ParentUser
 		if _, ok := cache.iamUsersMap[svcParent]; !ok {
+			stsPolicyMappingsCount++
 			// If a service account's parent user is not in iamUsersMap, the
 			// parent is an STS account. Such accounts may have a policy mapped
 			// on the parent user, so we load them. This is not needed for the
@@ -529,10 +580,14 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 			// OIDC/AssumeRoleWithCustomToken/AssumeRoleWithCertificate).
 			err := iamOS.loadMappedPolicy(ctx, svcParent, stsUser, false, cache.iamSTSPolicyMap)
 			if err != nil && !errors.Is(err, errNoSuchPolicy) {
-				return fmt.Errorf("unable to load the policy mapping for the STS user `%s`: %w", svcParent, err)
+				return fmt.Errorf("unable to load the policy mapping for the STS user: %w", err)
 			}
 		}
 	}
+	if took := time.Since(stsPolicyMappingLoadStartTime); took > maxIAMLoadOpTime {
+		logger.Info("STS policy mappings load took %.2fs (for %d items)", took.Seconds(), stsPolicyMappingsCount)
+	}
+
 	// Copy svcUsersMap to cache.iamUsersMap
 	for k, v := range svcUsersMap {
 		cache.iamUsersMap[k] = v

cmd/iam-store.go

@@ -431,8 +431,41 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 		}
 	}
 
-	// returned policy could be empty
-	policies := mp.toSlice()
+	// returned policy could be empty, we use set to de-duplicate.
+	policies := set.CreateStringSet(mp.toSlice()...)
+
+	for _, group := range u.Credentials.Groups {
+		if store.getUsersSysType() == MinIOUsersSysType {
+			g, ok := c.iamGroupsMap[group]
+			if !ok {
+				if err := store.loadGroup(context.Background(), group, c.iamGroupsMap); err != nil {
+					return nil, time.Time{}, err
+				}
+				g, ok = c.iamGroupsMap[group]
+				if !ok {
+					return nil, time.Time{}, errNoSuchGroup
+				}
+			}
+
+			// Group is disabled, so we return no policy - this
+			// ensures the request is denied.
+			if g.Status == statusDisabled {
+				return nil, time.Time{}, nil
+			}
+		}
+
+		policy, ok := c.iamGroupPolicyMap.Load(group)
+		if !ok {
+			if err := store.loadMappedPolicyWithRetry(context.TODO(), group, regUser, true, c.iamGroupPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
+				return nil, time.Time{}, err
+			}
+			policy, _ = c.iamGroupPolicyMap.Load(group)
+		}
+
+		for _, p := range policy.toSlice() {
+			policies.Add(p)
+		}
+	}
 
 	for _, group := range c.iamUserGroupMemberships[name].ToSlice() {
 		if store.getUsersSysType() == MinIOUsersSysType {
@@ -462,10 +495,12 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 			policy, _ = c.iamGroupPolicyMap.Load(group)
 		}
 
-		policies = append(policies, policy.toSlice()...)
+		for _, p := range policy.toSlice() {
+			policies.Add(p)
+		}
 	}
 
-	return policies, mp.UpdatedAt, nil
+	return policies.ToSlice(), mp.UpdatedAt, nil
 }
 
 func (c *iamCache) updateUserWithClaims(key string, u UserIdentity) error {
@@ -537,25 +572,25 @@ func setDefaultCannedPolicies(policies map[string]PolicyDoc) {
 // LoadIAMCache reads all IAM items and populates a new iamCache object and
 // replaces the in-memory cache object.
 func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) error {
-	bootstrapTraceMsg := func(s string) {
+	bootstrapTraceMsgFirstTime := func(s string) {
 		if firstTime {
 			bootstrapTraceMsg(s)
 		}
 	}
-	bootstrapTraceMsg("loading IAM data")
+	bootstrapTraceMsgFirstTime("loading IAM data")
 
 	newCache := newIamCache()
 
 	loadedAt := time.Now()
 
 	if iamOS, ok := store.IAMStorageAPI.(*IAMObjectStore); ok {
-		err := iamOS.loadAllFromObjStore(ctx, newCache)
+		err := iamOS.loadAllFromObjStore(ctx, newCache, firstTime)
 		if err != nil {
 			return err
 		}
 	} else {
-
-		bootstrapTraceMsg("loading policy documents")
+		// Only non-object IAM store (i.e. only etcd backend).
+		bootstrapTraceMsgFirstTime("loading policy documents")
 		if err := store.loadPolicyDocs(ctx, newCache.iamPolicyDocsMap); err != nil {
 			return err
 		}
@@ -564,29 +599,29 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) erro
 		setDefaultCannedPolicies(newCache.iamPolicyDocsMap)
 
 		if store.getUsersSysType() == MinIOUsersSysType {
-			bootstrapTraceMsg("loading regular users")
+			bootstrapTraceMsgFirstTime("loading regular users")
 			if err := store.loadUsers(ctx, regUser, newCache.iamUsersMap); err != nil {
 				return err
 			}
-			bootstrapTraceMsg("loading regular groups")
+			bootstrapTraceMsgFirstTime("loading regular groups")
 			if err := store.loadGroups(ctx, newCache.iamGroupsMap); err != nil {
 				return err
 			}
 		}
 
-		bootstrapTraceMsg("loading user policy mapping")
+		bootstrapTraceMsgFirstTime("loading user policy mapping")
 		// load polices mapped to users
 		if err := store.loadMappedPolicies(ctx, regUser, false, newCache.iamUserPolicyMap); err != nil {
 			return err
 		}
 
-		bootstrapTraceMsg("loading group policy mapping")
+		bootstrapTraceMsgFirstTime("loading group policy mapping")
 		// load policies mapped to groups
 		if err := store.loadMappedPolicies(ctx, regUser, true, newCache.iamGroupPolicyMap); err != nil {
 			return err
 		}
 
-		bootstrapTraceMsg("loading service accounts")
+		bootstrapTraceMsgFirstTime("loading service accounts")
 		// load service accounts
 		if err := store.loadUsers(ctx, svcUser, newCache.iamUsersMap); err != nil {
 			return err
@@ -937,12 +972,7 @@ func (store *IAMStoreSys) GetGroupDescription(group string) (gd madmin.GroupDesc
 	}, nil
 }
 
-// ListGroups - lists groups. Since this is not going to be a frequent
-// operation, we fetch this info from storage, and refresh the cache as well.
-func (store *IAMStoreSys) ListGroups(ctx context.Context) (res []string, err error) {
-	cache := store.lock()
-	defer store.unlock()
-
+func (store *IAMStoreSys) updateGroups(ctx context.Context, cache *iamCache) (res []string, err error) {
 	if store.getUsersSysType() == MinIOUsersSysType {
 		m := map[string]GroupInfo{}
 		err = store.loadGroups(ctx, m)
@@ -970,7 +1000,16 @@ func (store *IAMStoreSys) ListGroups(ctx context.Context) (res []string, err err
 		})
 	}
 
-	return
+	return res, nil
+}
+
+// ListGroups - lists groups. Since this is not going to be a frequent
+// operation, we fetch this info from storage, and refresh the cache as well.
+func (store *IAMStoreSys) ListGroups(ctx context.Context) (res []string, err error) {
+	cache := store.lock()
+	defer store.unlock()
+
+	return store.updateGroups(ctx, cache)
 }
 
 // listGroups - lists groups - fetch groups from cache
@@ -1445,16 +1484,51 @@ func filterPolicies(cache *iamCache, policyName string, bucketName string) (stri
 	return strings.Join(policies, ","), policy.MergePolicies(toMerge...)
 }
 
-// FilterPolicies - accepts a comma separated list of policy names as a string
-// and bucket and returns only policies that currently exist in MinIO. If
-// bucketName is non-empty, additionally filters policies matching the bucket.
-// The first returned value is the list of currently existing policies, and the
-// second is their combined policy definition.
-func (store *IAMStoreSys) FilterPolicies(policyName string, bucketName string) (string, policy.Policy) {
-	cache := store.rlock()
-	defer store.runlock()
+// MergePolicies - accepts a comma separated list of policy names as a string
+// and returns only policies that currently exist in MinIO. It includes hot loading
+// of policies if not in the memory
+func (store *IAMStoreSys) MergePolicies(policyName string) (string, policy.Policy) {
+	var policies []string
+	var missingPolicies []string
+	var toMerge []policy.Policy
 
-	return filterPolicies(cache, policyName, bucketName)
+	cache := store.rlock()
+	for _, policy := range newMappedPolicy(policyName).toSlice() {
+		if policy == "" {
+			continue
+		}
+		p, found := cache.iamPolicyDocsMap[policy]
+		if !found {
+			missingPolicies = append(missingPolicies, policy)
+			continue
+		}
+		policies = append(policies, policy)
+		toMerge = append(toMerge, p.Policy)
+	}
+	store.runlock()
+
+	if len(missingPolicies) > 0 {
+		m := make(map[string]PolicyDoc)
+		for _, policy := range missingPolicies {
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			_ = store.loadPolicyDoc(ctx, policy, m)
+			cancel()
+		}
+
+		cache := store.lock()
+		for policy, p := range m {
+			cache.iamPolicyDocsMap[policy] = p
+		}
+		store.unlock()
+
+		for policy, p := range m {
+			policies = append(policies, policy)
+			toMerge = append(toMerge, p.Policy)
+		}
+
+	}
+
+	return strings.Join(policies, ","), policy.MergePolicies(toMerge...)
 }
 
 // GetBucketUsers - returns users (not STS or service accounts) that have access
@@ -1907,6 +1981,11 @@ func (store *IAMStoreSys) GetAllParentUsers() map[string]ParentUserInfo {
 	cache := store.rlock()
 	defer store.runlock()
 
+	return store.getParentUsers(cache)
+}
+
+// assumes store is locked by caller.
+func (store *IAMStoreSys) getParentUsers(cache *iamCache) map[string]ParentUserInfo {
 	res := map[string]ParentUserInfo{}
 	for _, ui := range cache.iamUsersMap {
 		cred := ui.Credentials
@@ -1977,50 +2056,104 @@ func (store *IAMStoreSys) GetAllParentUsers() map[string]ParentUserInfo {
 	return res
 }
 
-// Assumes store is locked by caller. If users is empty, returns all user mappings.
-func (store *IAMStoreSys) listUserPolicyMappings(cache *iamCache, users []string,
-	userPredicate func(string) bool,
+// GetAllSTSUserMappings - Loads all STS user policy mappings from storage and
+// returns them. Also gets any STS users that do not have policy mappings but have
+// Service Accounts or STS keys (This is useful if the user is part of a group)
+func (store *IAMStoreSys) GetAllSTSUserMappings(userPredicate func(string) bool) (map[string]string, error) {
+	cache := store.rlock()
+	defer store.runlock()
+
+	stsMap := make(map[string]string)
+	m := xsync.NewMapOf[string, MappedPolicy]()
+	if err := store.loadMappedPolicies(context.Background(), stsUser, false, m); err != nil {
+		return nil, err
+	}
+
+	m.Range(func(user string, mappedPolicy MappedPolicy) bool {
+		if userPredicate != nil && !userPredicate(user) {
+			return true
+		}
+		stsMap[user] = mappedPolicy.Policies
+		return true
+	})
+
+	for user := range store.getParentUsers(cache) {
+		if _, ok := stsMap[user]; !ok {
+			if userPredicate != nil && !userPredicate(user) {
+				continue
+			}
+			stsMap[user] = ""
+		}
+	}
+	return stsMap, nil
+}
+
+// Assumes store is locked by caller. If userMap is empty, returns all user mappings.
+func (store *IAMStoreSys) listUserPolicyMappings(cache *iamCache, userMap map[string]set.StringSet,
+	userPredicate func(string) bool, decodeFunc func(string) string,
 ) []madmin.UserPolicyEntities {
+	stsMap := xsync.NewMapOf[string, MappedPolicy]()
+	resMap := make(map[string]madmin.UserPolicyEntities, len(userMap))
+
+	for user, groupSet := range userMap {
+		// Attempt to load parent user mapping for STS accounts
+		store.loadMappedPolicy(context.TODO(), user, stsUser, false, stsMap)
+		decodeUser := user
+		if decodeFunc != nil {
+			decodeUser = decodeFunc(user)
+		}
+		blankEntities := madmin.UserPolicyEntities{User: decodeUser}
+		if !groupSet.IsEmpty() {
+			blankEntities.MemberOfMappings = store.listGroupPolicyMappings(cache, groupSet, nil, decodeFunc)
+		}
+		resMap[user] = blankEntities
+	}
+
 	var r []madmin.UserPolicyEntities
-	usersSet := set.CreateStringSet(users...)
 	cache.iamUserPolicyMap.Range(func(user string, mappedPolicy MappedPolicy) bool {
 		if userPredicate != nil && !userPredicate(user) {
 			return true
 		}
 
-		if !usersSet.IsEmpty() && !usersSet.Contains(user) {
-			return true
+		entitiesWithMemberOf, ok := resMap[user]
+		if !ok {
+			if len(userMap) > 0 {
+				return true
+			}
+			decodeUser := user
+			if decodeFunc != nil {
+				decodeUser = decodeFunc(user)
+			}
+			entitiesWithMemberOf = madmin.UserPolicyEntities{User: decodeUser}
 		}
 
 		ps := mappedPolicy.toSlice()
 		sort.Strings(ps)
-		r = append(r, madmin.UserPolicyEntities{
-			User:     user,
-			Policies: ps,
-		})
+		entitiesWithMemberOf.Policies = ps
+		resMap[user] = entitiesWithMemberOf
 		return true
 	})
 
-	stsMap := xsync.NewMapOf[string, MappedPolicy]()
-	for _, user := range users {
-		// Attempt to load parent user mapping for STS accounts
-		store.loadMappedPolicy(context.TODO(), user, stsUser, false, stsMap)
-	}
-
 	stsMap.Range(func(user string, mappedPolicy MappedPolicy) bool {
 		if userPredicate != nil && !userPredicate(user) {
 			return true
 		}
 
+		entitiesWithMemberOf := resMap[user]
+
 		ps := mappedPolicy.toSlice()
 		sort.Strings(ps)
-		r = append(r, madmin.UserPolicyEntities{
-			User:     user,
-			Policies: ps,
-		})
+		entitiesWithMemberOf.Policies = ps
+		resMap[user] = entitiesWithMemberOf
 		return true
 	})
 
+	for _, v := range resMap {
+		if v.Policies != nil || v.MemberOfMappings != nil {
+			r = append(r, v)
+		}
+	}
+
 	sort.Slice(r, func(i, j int) bool {
 		return r[i].User < r[j].User
 	})
@@ -2029,11 +2162,11 @@ func (store *IAMStoreSys) listUserPolicyMappings(cache *iamCache, users []string
 }
 
 // Assumes store is locked by caller. If groups is empty, returns all group mappings.
-func (store *IAMStoreSys) listGroupPolicyMappings(cache *iamCache, groups []string,
-	groupPredicate func(string) bool,
+func (store *IAMStoreSys) listGroupPolicyMappings(cache *iamCache, groupsSet set.StringSet,
+	groupPredicate func(string) bool, decodeFunc func(string) string,
 ) []madmin.GroupPolicyEntities {
 	var r []madmin.GroupPolicyEntities
-	groupsSet := set.CreateStringSet(groups...)
+
 	cache.iamGroupPolicyMap.Range(func(group string, mappedPolicy MappedPolicy) bool {
 		if groupPredicate != nil && !groupPredicate(group) {
 			return true
@@ -2043,10 +2176,15 @@ func (store *IAMStoreSys) listGroupPolicyMappings(cache *iamCache, groups []stri
 			return true
 		}
 
+		decodeGroup := group
+		if decodeFunc != nil {
+			decodeGroup = decodeFunc(group)
+		}
+
 		ps := mappedPolicy.toSlice()
 		sort.Strings(ps)
 		r = append(r, madmin.GroupPolicyEntities{
-			Group:    group,
+			Group:    decodeGroup,
 			Policies: ps,
 		})
 		return true
@@ -2060,17 +2198,20 @@ func (store *IAMStoreSys) listGroupPolicyMappings(cache *iamCache, groups []stri
 }
 
 // Assumes store is locked by caller. If policies is empty, returns all policy mappings.
-func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
-	userPredicate, groupPredicate func(string) bool,
+func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, queryPolSet set.StringSet,
+	userPredicate, groupPredicate func(string) bool, decodeFunc func(string) string,
 ) []madmin.PolicyEntities {
-	queryPolSet := set.CreateStringSet(policies...)
-
 	policyToUsersMap := make(map[string]set.StringSet)
 	cache.iamUserPolicyMap.Range(func(user string, mappedPolicy MappedPolicy) bool {
 		if userPredicate != nil && !userPredicate(user) {
 			return true
 		}
 
+		decodeUser := user
+		if decodeFunc != nil {
+			decodeUser = decodeFunc(user)
+		}
+
 		commonPolicySet := mappedPolicy.policySet()
 		if !queryPolSet.IsEmpty() {
 			commonPolicySet = commonPolicySet.Intersection(queryPolSet)
@@ -2078,9 +2219,9 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 		for _, policy := range commonPolicySet.ToSlice() {
 			s, ok := policyToUsersMap[policy]
 			if !ok {
-				policyToUsersMap[policy] = set.CreateStringSet(user)
+				policyToUsersMap[policy] = set.CreateStringSet(decodeUser)
 			} else {
-				s.Add(user)
+				s.Add(decodeUser)
 				policyToUsersMap[policy] = s
 			}
 		}
@@ -2094,6 +2235,11 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 				continue
 			}
 
+			decodeUser := user
+			if decodeFunc != nil {
+				decodeUser = decodeFunc(user)
+			}
+
 			var mappedPolicy MappedPolicy
 			store.loadIAMConfig(context.Background(), &mappedPolicy, getMappedPolicyPath(user, stsUser, false))
 
@@ -2104,9 +2250,9 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 			for _, policy := range commonPolicySet.ToSlice() {
 				s, ok := policyToUsersMap[policy]
 				if !ok {
-					policyToUsersMap[policy] = set.CreateStringSet(user)
+					policyToUsersMap[policy] = set.CreateStringSet(decodeUser)
 				} else {
-					s.Add(user)
+					s.Add(decodeUser)
 					policyToUsersMap[policy] = s
 				}
 			}
@@ -2121,6 +2267,11 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 					return true
 				}
 
+				decodeUser := user
+				if decodeFunc != nil {
+					decodeUser = decodeFunc(user)
+				}
+
 				commonPolicySet := mappedPolicy.policySet()
 				if !queryPolSet.IsEmpty() {
 					commonPolicySet = commonPolicySet.Intersection(queryPolSet)
@@ -2128,9 +2279,9 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 				for _, policy := range commonPolicySet.ToSlice() {
 					s, ok := policyToUsersMap[policy]
 					if !ok {
-						policyToUsersMap[policy] = set.CreateStringSet(user)
+						policyToUsersMap[policy] = set.CreateStringSet(decodeUser)
 					} else {
-						s.Add(user)
+						s.Add(decodeUser)
 						policyToUsersMap[policy] = s
 					}
 				}
@@ -2145,6 +2296,11 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 			return true
 		}
 
+		decodeGroup := group
+		if decodeFunc != nil {
+			decodeGroup = decodeFunc(group)
+		}
+
 		commonPolicySet := mappedPolicy.policySet()
 		if !queryPolSet.IsEmpty() {
 			commonPolicySet = commonPolicySet.Intersection(queryPolSet)
@@ -2152,9 +2308,9 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 		for _, policy := range commonPolicySet.ToSlice() {
 			s, ok := policyToGroupsMap[policy]
 			if !ok {
-				policyToGroupsMap[policy] = set.CreateStringSet(group)
+				policyToGroupsMap[policy] = set.CreateStringSet(decodeGroup)
 			} else {
-				s.Add(group)
+				s.Add(decodeGroup)
 				policyToGroupsMap[policy] = s
 			}
 		}
@@ -2194,24 +2350,24 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 }
 
 // ListPolicyMappings - return users/groups mapped to policies.
-func (store *IAMStoreSys) ListPolicyMappings(q madmin.PolicyEntitiesQuery,
-	userPredicate, groupPredicate func(string) bool,
+func (store *IAMStoreSys) ListPolicyMappings(q cleanEntitiesQuery,
+	userPredicate, groupPredicate func(string) bool, decodeFunc func(string) string,
 ) madmin.PolicyEntitiesResult {
 	cache := store.rlock()
 	defer store.runlock()
 
 	var result madmin.PolicyEntitiesResult
 
-	isAllPoliciesQuery := len(q.Users) == 0 && len(q.Groups) == 0 && len(q.Policy) == 0
+	isAllPoliciesQuery := len(q.Users) == 0 && len(q.Groups) == 0 && len(q.Policies) == 0
 
 	if len(q.Users) > 0 {
-		result.UserMappings = store.listUserPolicyMappings(cache, q.Users, userPredicate)
+		result.UserMappings = store.listUserPolicyMappings(cache, q.Users, userPredicate, decodeFunc)
 	}
 	if len(q.Groups) > 0 {
-		result.GroupMappings = store.listGroupPolicyMappings(cache, q.Groups, groupPredicate)
+		result.GroupMappings = store.listGroupPolicyMappings(cache, q.Groups, groupPredicate, decodeFunc)
 	}
-	if len(q.Policy) > 0 || isAllPoliciesQuery {
-		result.PolicyMappings = store.listPolicyMappings(cache, q.Policy, userPredicate, groupPredicate)
+	if len(q.Policies) > 0 || isAllPoliciesQuery {
+		result.PolicyMappings = store.listPolicyMappings(cache, q.Policies, userPredicate, groupPredicate, decodeFunc)
 	}
 	return result
 }
@@ -2638,6 +2794,18 @@ func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) error
 			}
 		}
 
+		load := len(cache.iamGroupsMap) == 0
+		if store.getUsersSysType() == LDAPUsersSysType && cache.iamGroupPolicyMap.Size() == 0 {
+			load = true
+		}
+		if load {
+			if _, err = store.updateGroups(ctx, cache); err != nil {
+				return "done", err
+			}
+		}
+
+		cache.buildUserGroupMemberships()
+
 		return "done", err
 	})
 

cmd/iam.go

@@ -217,6 +217,9 @@ func (sys *IAMSys) Load(ctx context.Context, firstTime bool) error {
 
 	if firstTime {
 		bootstrapTraceMsg(fmt.Sprintf("globalIAMSys.Load(): (duration: %s)", loadDuration))
+		if globalIsDistErasure {
+			logger.Info("IAM load(startup) finished. (duration: %s)", loadDuration)
+		}
 	}
 
 	select {
@@ -315,6 +318,24 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 		break
 	}
 
+	cache := sys.store.lock()
+	setDefaultCannedPolicies(cache.iamPolicyDocsMap)
+	sys.store.unlock()
+
+	// Load RoleARNs
+	sys.rolesMap = make(map[arn.ARN]string)
+
+	// From OpenID
+	if riMap := sys.OpenIDConfig.GetRoleInfo(); riMap != nil {
+		sys.validateAndAddRolePolicyMappings(ctx, riMap)
+	}
+
+	// From AuthN plugin if enabled.
+	if authn := newGlobalAuthNPluginFn(); authn != nil {
+		riMap := authn.GetRoleInfo()
+		sys.validateAndAddRolePolicyMappings(ctx, riMap)
+	}
+
 	// Load IAM data from storage.
 	for {
 		if err := sys.Load(retryCtx, true); err != nil {
@@ -334,20 +355,6 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 
 	go sys.periodicRoutines(ctx, refreshInterval)
 
-	// Load RoleARNs
-	sys.rolesMap = make(map[arn.ARN]string)
-
-	// From OpenID
-	if riMap := sys.OpenIDConfig.GetRoleInfo(); riMap != nil {
-		sys.validateAndAddRolePolicyMappings(ctx, riMap)
-	}
-
-	// From AuthN plugin if enabled.
-	if authn := newGlobalAuthNPluginFn(); authn != nil {
-		riMap := authn.GetRoleInfo()
-		sys.validateAndAddRolePolicyMappings(ctx, riMap)
-	}
-
 	sys.printIAMRoles()
 
 	bootstrapTraceMsg("finishing IAM loading")
@@ -396,12 +403,12 @@ func (sys *IAMSys) periodicRoutines(ctx context.Context, baseInterval time.Durat
 			// Load all IAM items (except STS creds) periodically.
 			refreshStart := time.Now()
 			if err := sys.Load(ctx, false); err != nil {
-				iamLogIf(ctx, fmt.Errorf("Failure in periodic refresh for IAM (took %.2fs): %v", time.Since(refreshStart).Seconds(), err), logger.WarningKind)
+				iamLogIf(ctx, fmt.Errorf("Failure in periodic refresh for IAM (duration: %s): %v", time.Since(refreshStart), err), logger.WarningKind)
 			} else {
 				took := time.Since(refreshStart).Seconds()
 				if took > maxDurationSecondsForLog {
 					// Log if we took a lot of time to load.
-					logger.Info("IAM refresh took %.2fs", took)
+					logger.Info("IAM refresh took (duration: %s)", took)
 				}
 			}
 
@@ -436,7 +443,7 @@ func (sys *IAMSys) validateAndAddRolePolicyMappings(ctx context.Context, m map[a
 	// running server by creating the policies after start up.
 	for arn, rolePolicies := range m {
 		specifiedPoliciesSet := newMappedPolicy(rolePolicies).policySet()
-		validPolicies, _ := sys.store.FilterPolicies(rolePolicies, "")
+		validPolicies, _ := sys.store.MergePolicies(rolePolicies)
 		knownPoliciesSet := newMappedPolicy(validPolicies).policySet()
 		unknownPoliciesSet := specifiedPoliciesSet.Difference(knownPoliciesSet)
 		if len(unknownPoliciesSet) > 0 {
@@ -672,7 +679,7 @@ func (sys *IAMSys) CurrentPolicies(policyName string) string {
 		return ""
 	}
 
-	policies, _ := sys.store.FilterPolicies(policyName, "")
+	policies, _ := sys.store.MergePolicies(policyName)
 	return policies
 }
 
@@ -786,11 +793,15 @@ func (sys *IAMSys) ListLDAPUsers(ctx context.Context) (map[string]madmin.UserInf
 
 	select {
 	case <-sys.configLoaded:
-		ldapUsers := make(map[string]madmin.UserInfo)
-		for user, policy := range sys.store.GetUsersWithMappedPolicies() {
+		stsMap, err := sys.store.GetAllSTSUserMappings(sys.LDAPConfig.IsLDAPUserDN)
+		if err != nil {
+			return nil, err
+		}
+		ldapUsers := make(map[string]madmin.UserInfo, len(stsMap))
+		for user, policy := range stsMap {
 			ldapUsers[user] = madmin.UserInfo{
 				PolicyName: policy,
-				Status:     madmin.AccountEnabled,
+				Status:     statusEnabled,
 			}
 		}
 		return ldapUsers, nil
@@ -799,6 +810,57 @@ func (sys *IAMSys) ListLDAPUsers(ctx context.Context) (map[string]madmin.UserInf
 	}
 }
 
+type cleanEntitiesQuery struct {
+	Users    map[string]set.StringSet
+	Groups   set.StringSet
+	Policies set.StringSet
+}
+
+// createCleanEntitiesQuery - maps users to their groups and normalizes user or group DNs if ldap.
+func (sys *IAMSys) createCleanEntitiesQuery(q madmin.PolicyEntitiesQuery, ldap bool) cleanEntitiesQuery {
+	cleanQ := cleanEntitiesQuery{
+		Users:    make(map[string]set.StringSet),
+		Groups:   set.CreateStringSet(q.Groups...),
+		Policies: set.CreateStringSet(q.Policy...),
+	}
+
+	if ldap {
+		// Validate and normalize users, then fetch and normalize their groups
+		// Also include unvalidated users for backward compatibility.
+		for _, user := range q.Users {
+			lookupRes, actualGroups, _ := sys.LDAPConfig.GetValidatedDNWithGroups(user)
+			if lookupRes != nil {
+				groupSet := set.CreateStringSet(actualGroups...)
+
+				// duplicates can be overwritten, fetched groups should be identical.
+				cleanQ.Users[lookupRes.NormDN] = groupSet
+			}
+			// Search for non-normalized DN as well for backward compatibility.
+			if _, ok := cleanQ.Users[user]; !ok {
+				cleanQ.Users[user] = nil
+			}
+		}
+
+		// Validate and normalize groups.
+		for _, group := range q.Groups {
+			lookupRes, underDN, _ := sys.LDAPConfig.GetValidatedGroupDN(nil, group)
+			if lookupRes != nil && underDN {
+				cleanQ.Groups.Add(lookupRes.NormDN)
+			}
+		}
+	} else {
+		for _, user := range q.Users {
+			info, err := sys.store.GetUserInfo(user)
+			var groupSet set.StringSet
+			if err == nil {
+				groupSet = set.CreateStringSet(info.MemberOf...)
+			}
+			cleanQ.Users[user] = groupSet
+		}
+	}
+	return cleanQ
+}
+
 // QueryLDAPPolicyEntities - queries policy associations for LDAP users/groups/policies.
 func (sys *IAMSys) QueryLDAPPolicyEntities(ctx context.Context, q madmin.PolicyEntitiesQuery) (*madmin.PolicyEntitiesResult, error) {
 	if !sys.Initialized() {
@@ -811,7 +873,8 @@ func (sys *IAMSys) QueryLDAPPolicyEntities(ctx context.Context, q madmin.PolicyE
 
 	select {
 	case <-sys.configLoaded:
-		pe := sys.store.ListPolicyMappings(q, sys.LDAPConfig.IsLDAPUserDN, sys.LDAPConfig.IsLDAPGroupDN)
+		cleanQuery := sys.createCleanEntitiesQuery(q, true)
+		pe := sys.store.ListPolicyMappings(cleanQuery, sys.LDAPConfig.IsLDAPUserDN, sys.LDAPConfig.IsLDAPGroupDN, sys.LDAPConfig.DecodeDN)
 		pe.Timestamp = UTCNow()
 		return &pe, nil
 	case <-ctx.Done():
@@ -885,6 +948,7 @@ func (sys *IAMSys) QueryPolicyEntities(ctx context.Context, q madmin.PolicyEntit
 
 	select {
 	case <-sys.configLoaded:
+		cleanQuery := sys.createCleanEntitiesQuery(q, false)
 		var userPredicate, groupPredicate func(string) bool
 		if sys.LDAPConfig.Enabled() {
 			userPredicate = func(s string) bool {
@@ -894,7 +958,7 @@ func (sys *IAMSys) QueryPolicyEntities(ctx context.Context, q madmin.PolicyEntit
 				return !sys.LDAPConfig.IsLDAPGroupDN(s)
 			}
 		}
-		pe := sys.store.ListPolicyMappings(q, userPredicate, groupPredicate)
+		pe := sys.store.ListPolicyMappings(cleanQuery, userPredicate, groupPredicate, nil)
 		pe.Timestamp = UTCNow()
 		return &pe, nil
 	case <-ctx.Done():
@@ -1378,11 +1442,13 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 	// 1. Collect all LDAP users with active creds.
 	allCreds := sys.store.GetSTSAndServiceAccounts()
 	// List of unique LDAP (parent) user DNs that have active creds
-	var parentUsers []string
-	// Map of LDAP user to list of active credential objects
+	var parentUserActualDNList []string
+	// Map of LDAP user (internal representation) to list of active credential objects
 	parentUserToCredsMap := make(map[string][]auth.Credentials)
 	// DN to ldap username mapping for each LDAP user
-	parentUserToLDAPUsernameMap := make(map[string]string)
+	actualDNToLDAPUsernameMap := make(map[string]string)
+	// External (actual) LDAP DN to internal normalized representation
+	actualDNToParentUserMap := make(map[string]string)
 	for _, cred := range allCreds {
 		// Expired credentials don't need parent user updates.
 		if cred.IsExpired() {
@@ -1425,25 +1491,28 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 				continue
 			}
 
-			ldapUsername, ok := jwtClaims.Lookup(ldapUserN)
-			if !ok {
+			ldapUsername, okUserN := jwtClaims.Lookup(ldapUserN)
+			ldapActualDN, okDN := jwtClaims.Lookup(ldapActualUser)
+			if !okUserN || !okDN {
 				// skip this cred - we dont have the
 				// username info needed
 				continue
 			}
 
 			// Collect each new cred.ParentUser into parentUsers
-			parentUsers = append(parentUsers, cred.ParentUser)
+			parentUserActualDNList = append(parentUserActualDNList, ldapActualDN)
 
 			// Update the ldapUsernameMap
-			parentUserToLDAPUsernameMap[cred.ParentUser] = ldapUsername
+			actualDNToLDAPUsernameMap[ldapActualDN] = ldapUsername
+
+			// Update the actualDNToParentUserMap
+			actualDNToParentUserMap[ldapActualDN] = cred.ParentUser
 		}
 		parentUserToCredsMap[cred.ParentUser] = append(parentUserToCredsMap[cred.ParentUser], cred)
-
 	}
 
 	// 2. Query LDAP server for groups of the LDAP users collected.
-	updatedGroups, err := sys.LDAPConfig.LookupGroupMemberships(parentUsers, parentUserToLDAPUsernameMap)
+	updatedGroups, err := sys.LDAPConfig.LookupGroupMemberships(parentUserActualDNList, actualDNToLDAPUsernameMap)
 	if err != nil {
 		// Log and return on error - perhaps it'll work the next time.
 		iamLogIf(GlobalContext, err)
@@ -1451,8 +1520,9 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 	}
 
 	// 3. Update creds for those users whose groups are changed
-	for _, parentUser := range parentUsers {
-		currGroupsSet := updatedGroups[parentUser]
+	for _, parentActualDN := range parentUserActualDNList {
+		currGroupsSet := updatedGroups[parentActualDN]
+		parentUser := actualDNToParentUserMap[parentActualDN]
 		currGroups := currGroupsSet.ToSlice()
 		for _, cred := range parentUserToCredsMap[parentUser] {
 			gSet := set.CreateStringSet(cred.Groups...)
@@ -1510,11 +1580,11 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap
 		// server and is under a configured base DN.
 		validatedParent, isUnderBaseDN, err := sys.LDAPConfig.GetValidatedUserDN(conn, parent)
 		if err != nil {
-			collectedErrors = append(collectedErrors, fmt.Errorf("could not validate `%s` exists in LDAP directory: %w", parent, err))
+			collectedErrors = append(collectedErrors, fmt.Errorf("could not validate parent exists in LDAP directory: %w", err))
 			continue
 		}
 		if validatedParent == nil || !isUnderBaseDN {
-			err := fmt.Errorf("DN `%s` was not found in the LDAP directory", parent)
+			err := fmt.Errorf("DN parent was not found in the LDAP directory")
 			collectedErrors = append(collectedErrors, err)
 			continue
 		}
@@ -1529,11 +1599,11 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap
 			// configured base DN.
 			validatedGroup, _, err := sys.LDAPConfig.GetValidatedGroupDN(conn, group)
 			if err != nil {
-				collectedErrors = append(collectedErrors, fmt.Errorf("could not validate `%s` exists in LDAP directory: %w", group, err))
+				collectedErrors = append(collectedErrors, fmt.Errorf("could not validate group exists in LDAP directory: %w", err))
 				continue
 			}
 			if validatedGroup == nil {
-				err := fmt.Errorf("DN `%s` was not found in the LDAP directory", group)
+				err := fmt.Errorf("DN group was not found in the LDAP directory")
 				collectedErrors = append(collectedErrors, err)
 				continue
 			}
@@ -1623,7 +1693,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 			continue
 		}
 		if validatedDN == nil || !underBaseDN {
-			err := fmt.Errorf("DN `%s` was not found in the LDAP directory", k)
+			err := fmt.Errorf("DN was not found in the LDAP directory")
 			collectedErrors = append(collectedErrors, err)
 			continue
 		}
@@ -1966,7 +2036,7 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
 		if dnResult == nil {
 			// dn not found - still attempt to detach if provided user is a DN.
 			if !isAttach && sys.LDAPConfig.IsLDAPUserDN(r.User) {
-				dn = r.User
+				dn = sys.LDAPConfig.QuickNormalizeDN(r.User)
 			} else {
 				err = errNoSuchUser
 				return
@@ -1983,7 +2053,7 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
 		}
 		if dnResult == nil || !underBaseDN {
 			if !isAttach {
-				dn = r.Group
+				dn = sys.LDAPConfig.QuickNormalizeDN(r.Group)
 			} else {
 				err = errNoSuchGroup
 				return
@@ -2118,7 +2188,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args policy.Args, parentUser string)
 	var combinedPolicy policy.Policy
 	// Policies were found, evaluate all of them.
 	if !isOwnerDerived {
-		availablePoliciesStr, c := sys.store.FilterPolicies(strings.Join(svcPolicies, ","), "")
+		availablePoliciesStr, c := sys.store.MergePolicies(strings.Join(svcPolicies, ","))
 		if availablePoliciesStr == "" {
 			return false
 		}
@@ -2210,22 +2280,16 @@ func (sys *IAMSys) IsAllowedSTS(args policy.Args, parentUser string) bool {
 	// 2. Combine the mapped policies into a single combined policy.
 
 	var combinedPolicy policy.Policy
+	// Policies were found, evaluate all of them.
 	if !isOwnerDerived {
-		var err error
-		combinedPolicy, err = sys.store.GetPolicy(strings.Join(policies, ","))
-		if errors.Is(err, errNoSuchPolicy) {
-			for _, pname := range policies {
-				_, err := sys.store.GetPolicy(pname)
-				if errors.Is(err, errNoSuchPolicy) {
-					// all policies presented in the claim should exist
-					iamLogIf(GlobalContext, fmt.Errorf("expected policy (%s) missing from the JWT claim %s, rejecting the request", pname, iamPolicyClaimNameOpenID()))
-					return false
-				}
-			}
-			iamLogIf(GlobalContext, fmt.Errorf("all policies were unexpectedly present!"))
+		availablePoliciesStr, c := sys.store.MergePolicies(strings.Join(policies, ","))
+		if availablePoliciesStr == "" {
+			// all policies presented in the claim should exist
+			iamLogIf(GlobalContext, fmt.Errorf("expected policy (%s) missing from the JWT claim %s, rejecting the request", policies, iamPolicyClaimNameOpenID()))
+
 			return false
 		}
-
+		combinedPolicy = c
 	}
 
 	// 3. If an inline session-policy is present, evaluate it.
@@ -2346,7 +2410,7 @@ func isAllowedBySessionPolicy(args policy.Args) (hasSessionPolicy bool, isAllowe
 
 // GetCombinedPolicy returns a combined policy combining all policies
 func (sys *IAMSys) GetCombinedPolicy(policies ...string) policy.Policy {
-	_, policy := sys.store.FilterPolicies(strings.Join(policies, ","), "")
+	_, policy := sys.store.MergePolicies(strings.Join(policies, ","))
 	return policy
 }
 

cmd/jwt.go

@@ -24,10 +24,8 @@ import (
 
 	jwtgo "github.com/golang-jwt/jwt/v4"
 	jwtreq "github.com/golang-jwt/jwt/v4/request"
-	"github.com/hashicorp/golang-lru/v2/expirable"
 	"github.com/minio/minio/internal/auth"
 	xjwt "github.com/minio/minio/internal/jwt"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/policy"
 )
 
@@ -37,8 +35,8 @@ const (
 	// Default JWT token for web handlers is one day.
 	defaultJWTExpiry = 24 * time.Hour
 
-	// Inter-node JWT token expiry is 15 minutes.
-	defaultInterNodeJWTExpiry = 15 * time.Minute
+	// Inter-node JWT token expiry is 100 years approx.
+	defaultInterNodeJWTExpiry = 100 * 365 * 24 * time.Hour
 )
 
 var (
@@ -50,17 +48,10 @@ var (
 	errMalformedAuth      = errors.New("Malformed authentication input")
 )
 
-type cacheKey struct {
-	accessKey, secretKey, audience string
-}
-
-var cacheLRU = expirable.NewLRU[cacheKey, string](1000, nil, 15*time.Second)
-
-func authenticateNode(accessKey, secretKey, audience string) (string, error) {
+func authenticateNode(accessKey, secretKey string) (string, error) {
 	claims := xjwt.NewStandardClaims()
 	claims.SetExpiry(UTCNow().Add(defaultInterNodeJWTExpiry))
 	claims.SetAccessKey(accessKey)
-	claims.SetAudience(audience)
 
 	jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, claims)
 	return jwt.SignedString([]byte(secretKey))
@@ -141,27 +132,9 @@ func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, b
 	return claims, groups, owner, nil
 }
 
-// newCachedAuthToken returns a token that is cached up to 15 seconds.
-// If globalActiveCred is updated it is reflected at once.
-func newCachedAuthToken() func(audience string) string {
-	fn := func(accessKey, secretKey, audience string) (s string, err error) {
-		k := cacheKey{accessKey: accessKey, secretKey: secretKey, audience: audience}
-
-		var ok bool
-		s, ok = cacheLRU.Get(k)
-		if !ok {
-			s, err = authenticateNode(accessKey, secretKey, audience)
-			if err != nil {
-				return "", err
-			}
-			cacheLRU.Add(k, s)
-		}
-		return s, nil
-	}
-	return func(audience string) string {
-		cred := globalActiveCred
-		token, err := fn(cred.AccessKey, cred.SecretKey, audience)
-		logger.CriticalIf(GlobalContext, err)
-		return token
+// newCachedAuthToken returns the cached token.
+func newCachedAuthToken() func() string {
+	return func() string {
+		return globalNodeAuthToken
 	}
 }

cmd/jwt_test.go

@@ -107,7 +107,7 @@ func BenchmarkParseJWTStandardClaims(b *testing.B) {
 	}
 
 	creds := globalActiveCred
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey, "")
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -138,7 +138,7 @@ func BenchmarkParseJWTMapClaims(b *testing.B) {
 	}
 
 	creds := globalActiveCred
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey, "")
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -176,7 +176,7 @@ func BenchmarkAuthenticateNode(b *testing.B) {
 		b.ResetTimer()
 		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
-			fn(creds.AccessKey, creds.SecretKey, "aud")
+			fn(creds.AccessKey, creds.SecretKey)
 		}
 	})
 	b.Run("cached", func(b *testing.B) {
@@ -184,7 +184,7 @@ func BenchmarkAuthenticateNode(b *testing.B) {
 		b.ResetTimer()
 		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
-			fn("aud")
+			fn()
 		}
 	})
 }

cmd/kms-handlers.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/minio/kms-go/kes"
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/policy"
@@ -56,7 +57,7 @@ func (a kmsAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Request)
 	writeSuccessResponseJSON(w, resp)
 }
 
-// KMSMetricsHandler - POST /minio/kms/v1/metrics
+// KMSMetricsHandler - GET /minio/kms/v1/metrics
 func (a kmsAPIHandlers) KMSMetricsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSMetrics")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -83,7 +84,7 @@ func (a kmsAPIHandlers) KMSMetricsHandler(w http.ResponseWriter, r *http.Request
 	}
 }
 
-// KMSAPIsHandler - POST /minio/kms/v1/apis
+// KMSAPIsHandler - GET /minio/kms/v1/apis
 func (a kmsAPIHandlers) KMSAPIsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSAPIs")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -114,7 +115,7 @@ type versionResponse struct {
 	Version string `json:"version"`
 }
 
-// KMSVersionHandler - POST /minio/kms/v1/version
+// KMSVersionHandler - GET /minio/kms/v1/version
 func (a kmsAPIHandlers) KMSVersionHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSVersion")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -159,7 +160,20 @@ func (a kmsAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	if err := GlobalKMS.CreateKey(ctx, &kms.CreateKeyRequest{Name: r.Form.Get("key-id")}); err != nil {
+	keyID := r.Form.Get("key-id")
+
+	// Ensure policy allows the user to create this key name
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+	if !checkKMSActionAllowed(r, owner, cred, policy.KMSCreateKeyAction, keyID) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
+	if err := GlobalKMS.CreateKey(ctx, &kms.CreateKeyRequest{Name: keyID}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -171,6 +185,9 @@ func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Reques
 	ctx := newContext(r, w, "KMSListKeys")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
+	// This only checks if the action (kms:ListKeys) is allowed, it does not check
+	// each key name against the policy's Resources. We check that below, once
+	// we have the list of key names from the KMS.
 	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSListKeysAction)
 	if objectAPI == nil {
 		return
@@ -180,7 +197,7 @@ func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Reques
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
 		return
 	}
-	names, _, err := GlobalKMS.ListKeyNames(ctx, &kms.ListRequest{
+	allKeyNames, _, err := GlobalKMS.ListKeyNames(ctx, &kms.ListRequest{
 		Prefix: r.Form.Get("pattern"),
 	})
 	if err != nil {
@@ -188,8 +205,24 @@ func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	values := make([]kes.KeyInfo, 0, len(names))
-	for _, name := range names {
+	// Get the cred and owner for checking authz below.
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	// Now we have all the key names, for each of them, check whether the policy grants permission for
+	// the user to list it.
+	keyNames := []string{}
+	for _, name := range allKeyNames {
+		if checkKMSActionAllowed(r, owner, cred, policy.KMSListKeysAction, name) {
+			keyNames = append(keyNames, name)
+		}
+	}
+
+	values := make([]kes.KeyInfo, 0, len(keyNames))
+	for _, name := range keyNames {
 		values = append(values, kes.KeyInfo{
 			Name: name,
 		})
@@ -224,6 +257,17 @@ func (a kmsAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Reque
 		KeyID: keyID,
 	}
 
+	// Ensure policy allows the user to get this key's status
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+	if !checkKMSActionAllowed(r, owner, cred, policy.KMSKeyStatusAction, keyID) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
 	kmsContext := kms.Context{"MinIO admin API": "KMSKeyStatusHandler"} // Context for a test key operation
 	// 1. Generate a new key using the KMS.
 	key, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{Name: keyID, AssociatedData: kmsContext})
@@ -274,3 +318,16 @@ func (a kmsAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Reque
 	}
 	writeSuccessResponseJSON(w, resp)
 }
+
+// checkKMSActionAllowed checks for authorization for a specific action on a resource.
+func checkKMSActionAllowed(r *http.Request, owner bool, cred auth.Credentials, action policy.KMSAction, resource string) bool {
+	return globalIAMSys.IsAllowed(policy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          policy.Action(action),
+		ConditionValues: getConditionValues(r, "", cred),
+		IsOwner:         owner,
+		Claims:          cred.Claims,
+		BucketName:      resource, // overloading BucketName as that's what the policy engine uses to assemble a Resource.
+	})
+}

cmd/kms-handlers_test.go

@@ -0,0 +1,851 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/pkg/v3/policy"
+)
+
+const (
+	// KMS API paths
+	// For example: /minio/kms/v1/key/list?pattern=*
+	kmsURL           = kmsPathPrefix + kmsAPIVersionPrefix
+	kmsStatusPath    = kmsURL + "/status"
+	kmsMetricsPath   = kmsURL + "/metrics"
+	kmsAPIsPath      = kmsURL + "/apis"
+	kmsVersionPath   = kmsURL + "/version"
+	kmsKeyCreatePath = kmsURL + "/key/create"
+	kmsKeyListPath   = kmsURL + "/key/list"
+	kmsKeyStatusPath = kmsURL + "/key/status"
+
+	// Admin API paths
+	// For example: /minio/admin/v3/kms/status
+	adminURL              = adminPathPrefix + adminAPIVersionPrefix
+	kmsAdminStatusPath    = adminURL + "/kms/status"
+	kmsAdminKeyStatusPath = adminURL + "/kms/key/status"
+	kmsAdminKeyCreate     = adminURL + "/kms/key/create"
+)
+
+const (
+	userAccessKey = "miniofakeuseraccesskey"
+	userSecretKey = "miniofakeusersecret"
+)
+
+type kmsTestCase struct {
+	name   string
+	method string
+	path   string
+	query  map[string]string
+
+	// User credentials and policy for request
+	policy string
+	asRoot bool
+
+	// Wanted in response.
+	wantStatusCode int
+	wantKeyNames   []string
+	wantResp       []string
+}
+
+func TestKMSHandlersCreateKey(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, true)
+	defer tearDown()
+
+	tests := []kmsTestCase{
+		// Create key test
+		{
+			name:   "create key as user with no policy want forbidden",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "new-test-key"},
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "create key as user with no resources specified want success",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "new-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:CreateKey"] }`,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "create key as user set policy to allow want success",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "second-new-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:CreateKey"],
+				"Resource": ["arn:minio:kms:::second-new-test-*"] }`,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "create key as user set policy to non matching resource want forbidden",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "third-new-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:CreateKey"],
+				"Resource": ["arn:minio:kms:::non-matching-key-name"] }`,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+	}
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) {
+			execKMSTest(t, test, adminTestBed)
+		})
+	}
+}
+
+func TestKMSHandlersKeyStatus(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, true)
+	defer tearDown()
+
+	tests := []kmsTestCase{
+		{
+			name:   "create a first key root user",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "key status as root want success",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"abc-test-key"},
+		},
+		{
+			name:   "key status as user no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "key status as user legacy no resources specified want success",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:KeyStatus"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"abc-test-key"},
+		},
+		{
+			name:   "key status as user set policy to allow only one key",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:KeyStatus"],
+					"Resource": ["arn:minio:kms:::abc-test-*"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"abc-test-key"},
+		},
+		{
+			name:   "key status as user set policy to allow non-matching key",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:KeyStatus"],
+					"Resource": ["arn:minio:kms:::xyz-test-key"] }`,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+	}
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) {
+			execKMSTest(t, test, adminTestBed)
+		})
+	}
+}
+
+func TestKMSHandlersAPIs(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, true)
+	defer tearDown()
+
+	tests := []kmsTestCase{
+		// Version test
+		{
+			name:   "version as root want success",
+			method: http.MethodGet,
+			path:   kmsVersionPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"version"},
+		},
+		{
+			name:   "version as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsVersionPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "version as user with policy ignores resource want success",
+			method: http.MethodGet,
+			path:   kmsVersionPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:Version"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"version"},
+		},
+
+		// APIs test
+		{
+			name:   "apis as root want success",
+			method: http.MethodGet,
+			path:   kmsAPIsPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"stub/path"},
+		},
+		{
+			name:   "apis as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsAPIsPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "apis as user with policy ignores resource want success",
+			method: http.MethodGet,
+			path:   kmsAPIsPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:API"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"stub/path"},
+		},
+
+		// Metrics test
+		{
+			name:   "metrics as root want success",
+			method: http.MethodGet,
+			path:   kmsMetricsPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"kms"},
+		},
+		{
+			name:   "metrics as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsMetricsPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "metrics as user with policy ignores resource want success",
+			method: http.MethodGet,
+			path:   kmsMetricsPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:Metrics"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"kms"},
+		},
+
+		// Status tests
+		{
+			name:   "status as root want success",
+			method: http.MethodGet,
+			path:   kmsStatusPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"MinIO builtin"},
+		},
+		{
+			name:   "status as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsStatusPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "status as user with policy ignores resource want success",
+			method: http.MethodGet,
+			path:   kmsStatusPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:Status"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"]}`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"MinIO builtin"},
+		},
+	}
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) {
+			execKMSTest(t, test, adminTestBed)
+		})
+	}
+}
+
+func TestKMSHandlersListKeys(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, true)
+	defer tearDown()
+
+	tests := []kmsTestCase{
+		{
+			name:   "create a first key root user",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "create a second key root user",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "xyz-test-key"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+		},
+
+		// List keys tests
+		{
+			name:   "list keys as root want all to be returned",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{"default-test-key", "abc-test-key", "xyz-test-key"},
+		},
+		{
+			name:   "list keys as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "list keys as user with no resources specified want success",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:ListKeys"]
+			}`,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{"default-test-key", "abc-test-key", "xyz-test-key"},
+		},
+		{
+			name:   "list keys as user set policy resource to allow only one key",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:ListKeys"],
+					"Resource": ["arn:minio:kms:::abc*"]}`,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{"abc-test-key"},
+		},
+		{
+			name:   "list keys as user set policy to allow only one key, use pattern that includes correct key",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "abc*"},
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:ListKeys"],
+					"Resource": ["arn:minio:kms:::abc*"]}`,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{"abc-test-key"},
+		},
+		{
+			name:   "list keys as user set policy to allow only one key, use pattern that excludes correct key",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "xyz*"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:ListKeys"],
+					"Resource": ["arn:minio:kms:::abc*"]}`,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{},
+		},
+		{
+			name:   "list keys as user set policy that has no matching key resources",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:ListKeys"],
+					"Resource": ["arn:minio:kms:::nonematch*"]}`,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{},
+		},
+		{
+			name:   "list keys as user set policy that allows listing but denies specific keys",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: false,
+
+			// It looks like this should allow listing any key that isn't "default-test-key", however
+			// the policy engine matches all Deny statements first, without regard to Resources (for KMS).
+			// This is for backwards compatibility where historically KMS statements ignored Resources.
+			policy: `{
+						"Effect": "Allow",
+						"Action": ["kms:ListKeys"]
+					},{
+						"Effect": "Deny",
+						"Action": ["kms:ListKeys"],
+						"Resource": ["arn:minio:kms:::default-test-key"]
+					}`,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+	}
+
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) {
+			execKMSTest(t, test, adminTestBed)
+		})
+	}
+}
+
+func TestKMSHandlerAdminAPI(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, true)
+	defer tearDown()
+
+	tests := []kmsTestCase{
+		// Create key tests
+		{
+			name:   "create a key root user",
+			method: http.MethodPost,
+			path:   kmsAdminKeyCreate,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "create key as user with no policy want forbidden",
+			method: http.MethodPost,
+			path:   kmsAdminKeyCreate,
+			query:  map[string]string{"key-id": "new-test-key"},
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "create key as user with no resources specified want success",
+			method: http.MethodPost,
+			path:   kmsAdminKeyCreate,
+			query:  map[string]string{"key-id": "new-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["admin:KMSCreateKey"] }`,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "create key as user set policy to non matching resource want success",
+			method: http.MethodPost,
+			path:   kmsAdminKeyCreate,
+			query:  map[string]string{"key-id": "third-new-test-key"},
+			asRoot: false,
+
+			// Admin actions ignore Resources
+			policy: `{"Effect": "Allow",
+				"Action": ["admin:KMSCreateKey"],
+				"Resource": ["arn:minio:kms:::this-is-disregarded"] }`,
+
+			wantStatusCode: http.StatusOK,
+		},
+
+		// Status tests
+		{
+			name:   "status as root want success",
+			method: http.MethodPost,
+			path:   kmsAdminStatusPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"MinIO builtin"},
+		},
+		{
+			name:   "status as user with no policy want forbidden",
+			method: http.MethodPost,
+			path:   kmsAdminStatusPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "status as user with policy ignores resource want success",
+			method: http.MethodPost,
+			path:   kmsAdminStatusPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["admin:KMSKeyStatus"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"MinIO builtin"},
+		},
+
+		// Key status tests
+		{
+			name:   "key status as root want success",
+			method: http.MethodGet,
+			path:   kmsAdminKeyStatusPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"key-id"},
+		},
+		{
+			name:   "key status as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsAdminKeyStatusPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "key status as user with policy ignores resource want success",
+			method: http.MethodGet,
+			path:   kmsAdminKeyStatusPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["admin:KMSKeyStatus"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"key-id"},
+		},
+	}
+
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) {
+			execKMSTest(t, test, adminTestBed)
+		})
+	}
+}
+
+// execKMSTest runs a single test case for KMS handlers
+func execKMSTest(t *testing.T, test kmsTestCase, adminTestBed *adminErasureTestBed) {
+	var accessKey, secretKey string
+	if test.asRoot {
+		accessKey, secretKey = globalActiveCred.AccessKey, globalActiveCred.SecretKey
+	} else {
+		setupKMSUser(t, userAccessKey, userSecretKey, test.policy)
+		accessKey = userAccessKey
+		secretKey = userSecretKey
+	}
+
+	req := buildKMSRequest(t, test.method, test.path, accessKey, secretKey, test.query)
+	rec := httptest.NewRecorder()
+	adminTestBed.router.ServeHTTP(rec, req)
+
+	t.Logf("HTTP req: %s, resp code: %d, resp body: %s", req.URL.String(), rec.Code, rec.Body.String())
+
+	// Check status code
+	if rec.Code != test.wantStatusCode {
+		t.Errorf("want status code %d, got %d", test.wantStatusCode, rec.Code)
+	}
+
+	// Check returned key list is correct
+	if test.wantKeyNames != nil {
+		gotKeyNames := keyNamesFromListKeysResp(t, rec.Body.Bytes())
+		if len(test.wantKeyNames) != len(gotKeyNames) {
+			t.Fatalf("want keys len: %d, got len: %d", len(test.wantKeyNames), len(gotKeyNames))
+		}
+		for i, wantKeyName := range test.wantKeyNames {
+			if gotKeyNames[i] != wantKeyName {
+				t.Fatalf("want key name %s, in position %d, got %s", wantKeyName, i, gotKeyNames[i])
+			}
+		}
+	}
+
+	// Check generic text in the response
+	if test.wantResp != nil {
+		for _, want := range test.wantResp {
+			if !strings.Contains(rec.Body.String(), want) {
+				t.Fatalf("want response to contain %s, got %s", want, rec.Body.String())
+			}
+		}
+	}
+}
+
+// TestKMSHandlerNotConfiguredOrInvalidCreds tests KMS handlers for situations where KMS is not configured
+// or invalid credentials are provided.
+func TestKMSHandlerNotConfiguredOrInvalidCreds(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, false)
+	defer tearDown()
+
+	tests := []struct {
+		name   string
+		method string
+		path   string
+		query  map[string]string
+	}{
+		{
+			name:   "GET status",
+			method: http.MethodGet,
+			path:   kmsStatusPath,
+		},
+		{
+			name:   "GET metrics",
+			method: http.MethodGet,
+			path:   kmsMetricsPath,
+		},
+		{
+			name:   "GET apis",
+			method: http.MethodGet,
+			path:   kmsAPIsPath,
+		},
+		{
+			name:   "GET version",
+			method: http.MethodGet,
+			path:   kmsVersionPath,
+		},
+		{
+			name:   "POST key create",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "master-key-id"},
+		},
+		{
+			name:   "GET key list",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+		},
+		{
+			name:   "GET key status",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "master-key-id"},
+		},
+	}
+
+	// Test when the GlobalKMS is not configured
+	for _, test := range tests {
+		t.Run(test.name+" not configured", func(t *testing.T) {
+			req := buildKMSRequest(t, test.method, test.path, "", "", test.query)
+			rec := httptest.NewRecorder()
+			adminTestBed.router.ServeHTTP(rec, req)
+			if rec.Code != http.StatusNotImplemented {
+				t.Errorf("want status code %d, got %d", http.StatusNotImplemented, rec.Code)
+			}
+		})
+	}
+
+	// Test when the GlobalKMS is configured but the credentials are invalid
+	GlobalKMS = kms.NewStub("default-test-key")
+	for _, test := range tests {
+		t.Run(test.name+" invalid credentials", func(t *testing.T) {
+			req := buildKMSRequest(t, test.method, test.path, userAccessKey, userSecretKey, test.query)
+			rec := httptest.NewRecorder()
+			adminTestBed.router.ServeHTTP(rec, req)
+			if rec.Code != http.StatusForbidden {
+				t.Errorf("want status code %d, got %d", http.StatusForbidden, rec.Code)
+			}
+		})
+	}
+}
+
+func setupKMSTest(t *testing.T, enableKMS bool) (*adminErasureTestBed, func()) {
+	adminTestBed, err := prepareAdminErasureTestBed(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	registerKMSRouter(adminTestBed.router)
+
+	if enableKMS {
+		GlobalKMS = kms.NewStub("default-test-key")
+	}
+
+	tearDown := func() {
+		adminTestBed.TearDown()
+		GlobalKMS = nil
+	}
+	return adminTestBed, tearDown
+}
+
+func buildKMSRequest(t *testing.T, method, path, accessKey, secretKey string, query map[string]string) *http.Request {
+	if len(query) > 0 {
+		queryVal := url.Values{}
+		for k, v := range query {
+			queryVal.Add(k, v)
+		}
+		path = path + "?" + queryVal.Encode()
+	}
+
+	if accessKey == "" && secretKey == "" {
+		accessKey = globalActiveCred.AccessKey
+		secretKey = globalActiveCred.SecretKey
+	}
+
+	req, err := newTestSignedRequestV4(method, path, 0, nil, accessKey, secretKey, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return req
+}
+
+// setupKMSUser is a test helper that creates a new user with the provided access key and secret key
+// and applies the given policy to the user.
+func setupKMSUser(t *testing.T, accessKey, secretKey, p string) {
+	ctx := context.Background()
+	createUserParams := madmin.AddOrUpdateUserReq{
+		SecretKey: secretKey,
+		Status:    madmin.AccountEnabled,
+	}
+	_, err := globalIAMSys.CreateUser(ctx, accessKey, createUserParams)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testKMSPolicyName := "testKMSPolicy"
+	if p != "" {
+		p = `{"Version":"2012-10-17","Statement":[` + p + `]}`
+		policyData, err := policy.ParseConfig(strings.NewReader(p))
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, err = globalIAMSys.SetPolicy(ctx, testKMSPolicyName, *policyData)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, err = globalIAMSys.PolicyDBSet(ctx, accessKey, testKMSPolicyName, regUser, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+	} else {
+		err = globalIAMSys.DeletePolicy(ctx, testKMSPolicyName, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, err = globalIAMSys.PolicyDBSet(ctx, accessKey, "", regUser, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func keyNamesFromListKeysResp(t *testing.T, b []byte) []string {
+	var keyInfos []madmin.KMSKeyInfo
+	err := json.Unmarshal(b, &keyInfos)
+	if err != nil {
+		t.Fatalf("cannot unmarshal '%s', err: %v", b, err)
+	}
+
+	var gotKeyNames []string
+	for _, keyInfo := range keyInfos {
+		gotKeyNames = append(gotKeyNames, keyInfo.Name)
+	}
+	return gotKeyNames
+}

cmd/license-update.go

@@ -1,117 +0,0 @@
-// Copyright (c) 2015-2023 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"math/rand"
-	"time"
-
-	"github.com/tidwall/gjson"
-)
-
-const (
-	licUpdateCycle = 24 * time.Hour * 30
-	licRenewPath   = "/api/cluster/renew-license"
-)
-
-// initlicenseUpdateJob start the periodic license update job in the background.
-func initLicenseUpdateJob(ctx context.Context, objAPI ObjectLayer) {
-	go func() {
-		r := rand.New(rand.NewSource(time.Now().UnixNano()))
-		// Leader node (that successfully acquires the lock inside licenceUpdaterLoop)
-		// will keep performing the license update. If the leader goes down for some
-		// reason, the lock will be released and another node will acquire it and
-		// take over because of this loop.
-		for {
-			licenceUpdaterLoop(ctx, objAPI)
-
-			// license update stopped for some reason.
-			// sleep for some time and try again.
-			duration := time.Duration(r.Float64() * float64(time.Hour))
-			if duration < time.Second {
-				// Make sure to sleep at least a second to avoid high CPU ticks.
-				duration = time.Second
-			}
-			time.Sleep(duration)
-		}
-	}()
-}
-
-func licenceUpdaterLoop(ctx context.Context, objAPI ObjectLayer) {
-	ctx, cancel := globalLeaderLock.GetLock(ctx)
-	defer cancel()
-
-	licenseUpdateTimer := time.NewTimer(licUpdateCycle)
-	defer licenseUpdateTimer.Stop()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-licenseUpdateTimer.C:
-
-			if globalSubnetConfig.Registered() {
-				performLicenseUpdate(ctx, objAPI)
-			}
-
-			// Reset the timer for next cycle.
-			licenseUpdateTimer.Reset(licUpdateCycle)
-		}
-	}
-}
-
-func performLicenseUpdate(ctx context.Context, objectAPI ObjectLayer) {
-	// the subnet license renewal api renews the license only
-	// if required e.g. when it is expiring soon
-	url := globalSubnetConfig.BaseURL + licRenewPath
-
-	resp, err := globalSubnetConfig.Post(url, nil)
-	if err != nil {
-		subnetLogIf(ctx, fmt.Errorf("error from %s: %w", url, err))
-		return
-	}
-
-	r := gjson.Parse(resp).Get("license_v2")
-	if r.Index == 0 {
-		internalLogIf(ctx, fmt.Errorf("license not found in response from %s", url))
-		return
-	}
-
-	lic := r.String()
-	if lic == globalSubnetConfig.License {
-		// license hasn't changed.
-		return
-	}
-
-	kv := "subnet license=" + lic
-	result, err := setConfigKV(ctx, objectAPI, []byte(kv))
-	if err != nil {
-		internalLogIf(ctx, fmt.Errorf("error setting subnet license config: %w", err))
-		return
-	}
-
-	if result.Dynamic {
-		if err := applyDynamicConfigForSubSys(GlobalContext, objectAPI, result.Cfg, result.SubSys); err != nil {
-			subnetLogIf(ctx, fmt.Errorf("error applying subnet dynamic config: %w", err))
-			return
-		}
-		globalNotificationSys.SignalConfigReload(result.SubSys)
-	}
-}

cmd/local-locker.go

@@ -31,16 +31,16 @@ import (
 
 // lockRequesterInfo stores various info from the client for each lock that is requested.
 type lockRequesterInfo struct {
-	Name            string    // name of the resource lock was requested for
-	Writer          bool      // Bool whether write or read lock.
-	UID             string    // UID to uniquely identify request of client.
-	Timestamp       time.Time // Timestamp set at the time of initialization.
-	TimeLastRefresh time.Time // Timestamp for last lock refresh.
-	Source          string    // Contains line, function and filename requesting the lock.
-	Group           bool      // indicates if it was a group lock.
-	Owner           string    // Owner represents the UUID of the owner who originally requested the lock.
-	Quorum          int       // Quorum represents the quorum required for this lock to be active.
-	idx             int       `msg:"-"` // index of the lock in the lockMap.
+	Name            string // name of the resource lock was requested for
+	Writer          bool   // Bool whether write or read lock.
+	UID             string // UID to uniquely identify request of client.
+	Timestamp       int64  // Timestamp set at the time of initialization.
+	TimeLastRefresh int64  // Timestamp for last lock refresh.
+	Source          string // Contains line, function and filename requesting the lock.
+	Group           bool   // indicates if it was a group lock.
+	Owner           string // Owner represents the UUID of the owner who originally requested the lock.
+	Quorum          int    // Quorum represents the quorum required for this lock to be active.
+	idx             int    `msg:"-"` // index of the lock in the lockMap.
 }
 
 // isWriteLock returns whether the lock is a write or read lock.
@@ -87,6 +87,7 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
 
 	// No locks held on the all resources, so claim write
 	// lock on all resources at once.
+	now := UTCNow()
 	for i, resource := range args.Resources {
 		l.lockMap[resource] = []lockRequesterInfo{
 			{
@@ -95,10 +96,10 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
 				Source:          args.Source,
 				Owner:           args.Owner,
 				UID:             args.UID,
-				Timestamp:       UTCNow(),
-				TimeLastRefresh: UTCNow(),
+				Timestamp:       now.UnixNano(),
+				TimeLastRefresh: now.UnixNano(),
 				Group:           len(args.Resources) > 1,
-				Quorum:          args.Quorum,
+				Quorum:          *args.Quorum,
 				idx:             i,
 			},
 		}
@@ -108,7 +109,7 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
 }
 
 func formatUUID(s string, idx int) string {
-	return s + strconv.Itoa(idx)
+	return concat(s, strconv.Itoa(idx))
 }
 
 func (l *localLocker) Unlock(_ context.Context, args dsync.LockArgs) (reply bool, err error) {
@@ -167,17 +168,19 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
 	resource := args.Resources[0]
+	now := UTCNow()
 	lrInfo := lockRequesterInfo{
 		Name:            resource,
 		Writer:          false,
 		Source:          args.Source,
 		Owner:           args.Owner,
 		UID:             args.UID,
-		Timestamp:       UTCNow(),
-		TimeLastRefresh: UTCNow(),
-		Quorum:          args.Quorum,
+		Timestamp:       now.UnixNano(),
+		TimeLastRefresh: now.UnixNano(),
+		Quorum:          *args.Quorum,
 	}
-	if lri, ok := l.lockMap[resource]; ok {
+	lri, ok := l.lockMap[resource]
+	if ok {
 		if reply = !isWriteLock(lri); reply {
 			// Unless there is a write lock
 			l.lockMap[resource] = append(l.lockMap[resource], lrInfo)
@@ -351,9 +354,10 @@ func (l *localLocker) Refresh(ctx context.Context, args dsync.LockArgs) (refresh
 				delete(l.lockUID, formatUUID(args.UID, idx))
 				return idx > 0, nil
 			}
+			now := UTCNow()
 			for i := range lris {
 				if lris[i].UID == args.UID {
-					lris[i].TimeLastRefresh = UTCNow()
+					lris[i].TimeLastRefresh = now.UnixNano()
 				}
 			}
 			idx++
@@ -376,7 +380,7 @@ func (l *localLocker) expireOldLocks(interval time.Duration) {
 		modified := false
 		for i := 0; i < len(lris); {
 			lri := &lris[i]
-			if time.Since(lri.TimeLastRefresh) > interval {
+			if time.Since(time.Unix(0, lri.TimeLastRefresh)) > interval {
 				delete(l.lockUID, formatUUID(lri.UID, lri.idx))
 				if len(lris) == 1 {
 					// Remove the write lock.

cmd/local-locker_gen.go

@@ -204,13 +204,13 @@ func (z *lockRequesterInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "Timestamp":
-			z.Timestamp, err = dc.ReadTime()
+			z.Timestamp, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "Timestamp")
 				return
 			}
 		case "TimeLastRefresh":
-			z.TimeLastRefresh, err = dc.ReadTime()
+			z.TimeLastRefresh, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "TimeLastRefresh")
 				return
@@ -288,7 +288,7 @@ func (z *lockRequesterInfo) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteTime(z.Timestamp)
+	err = en.WriteInt64(z.Timestamp)
 	if err != nil {
 		err = msgp.WrapError(err, "Timestamp")
 		return
@@ -298,7 +298,7 @@ func (z *lockRequesterInfo) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteTime(z.TimeLastRefresh)
+	err = en.WriteInt64(z.TimeLastRefresh)
 	if err != nil {
 		err = msgp.WrapError(err, "TimeLastRefresh")
 		return
@@ -361,10 +361,10 @@ func (z *lockRequesterInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendString(o, z.UID)
 	// string "Timestamp"
 	o = append(o, 0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70)
-	o = msgp.AppendTime(o, z.Timestamp)
+	o = msgp.AppendInt64(o, z.Timestamp)
 	// string "TimeLastRefresh"
 	o = append(o, 0xaf, 0x54, 0x69, 0x6d, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68)
-	o = msgp.AppendTime(o, z.TimeLastRefresh)
+	o = msgp.AppendInt64(o, z.TimeLastRefresh)
 	// string "Source"
 	o = append(o, 0xa6, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65)
 	o = msgp.AppendString(o, z.Source)
@@ -417,13 +417,13 @@ func (z *lockRequesterInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				return
 			}
 		case "Timestamp":
-			z.Timestamp, bts, err = msgp.ReadTimeBytes(bts)
+			z.Timestamp, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Timestamp")
 				return
 			}
 		case "TimeLastRefresh":
-			z.TimeLastRefresh, bts, err = msgp.ReadTimeBytes(bts)
+			z.TimeLastRefresh, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "TimeLastRefresh")
 				return
@@ -466,7 +466,7 @@ func (z *lockRequesterInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *lockRequesterInfo) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 7 + msgp.BoolSize + 4 + msgp.StringPrefixSize + len(z.UID) + 10 + msgp.TimeSize + 16 + msgp.TimeSize + 7 + msgp.StringPrefixSize + len(z.Source) + 6 + msgp.BoolSize + 6 + msgp.StringPrefixSize + len(z.Owner) + 7 + msgp.IntSize
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 7 + msgp.BoolSize + 4 + msgp.StringPrefixSize + len(z.UID) + 10 + msgp.Int64Size + 16 + msgp.Int64Size + 7 + msgp.StringPrefixSize + len(z.Source) + 6 + msgp.BoolSize + 6 + msgp.StringPrefixSize + len(z.Owner) + 7 + msgp.IntSize
 	return
 }
 

cmd/local-locker_test.go

@@ -32,6 +32,7 @@ import (
 func TestLocalLockerExpire(t *testing.T) {
 	wResources := make([]string, 1000)
 	rResources := make([]string, 1000)
+	quorum := 0
 	l := newLocker()
 	ctx := context.Background()
 	for i := range wResources {
@@ -40,7 +41,7 @@ func TestLocalLockerExpire(t *testing.T) {
 			Resources: []string{mustGetUUID()},
 			Source:    t.Name(),
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.Lock(ctx, arg)
 		if err != nil {
@@ -58,14 +59,14 @@ func TestLocalLockerExpire(t *testing.T) {
 			Resources: []string{name},
 			Source:    t.Name(),
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.RLock(ctx, arg)
 		if err != nil {
 			t.Fatal(err)
 		}
 		if !ok {
-			t.Fatal("did not get write lock")
+			t.Fatal("did not get read lock")
 		}
 		// RLock twice
 		ok, err = l.RLock(ctx, arg)
@@ -112,6 +113,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 	rUIDs := make([]string, 0, n*2)
 	l := newLocker()
 	ctx := context.Background()
+	quorum := 0
 	for i := range wResources {
 		names := [m]string{}
 		for j := range names {
@@ -123,7 +125,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 			Resources: names[:],
 			Source:    t.Name(),
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.Lock(ctx, arg)
 		if err != nil {
@@ -144,7 +146,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 			Resources: []string{name},
 			Source:    t.Name(),
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.RLock(ctx, arg)
 		if err != nil {
@@ -183,7 +185,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 			Resources: []string{name},
 			Source:    t.Name(),
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.RUnlock(ctx, arg)
 		if err != nil {
@@ -203,6 +205,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 	if len(l.lockUID) != len(rResources)+len(wResources)*m {
 		t.Fatalf("lockUID len, got %d, want %d + %d", len(l.lockUID), len(rResources), len(wResources)*m)
 	}
+
 	// RUnlock again, different uids
 	for i, name := range rResources {
 		arg := dsync.LockArgs{
@@ -210,7 +213,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 			Resources: []string{name},
 			Source:    "minio",
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.RUnlock(ctx, arg)
 		if err != nil {
@@ -238,7 +241,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 			Resources: names[:],
 			Source:    "minio",
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.Unlock(ctx, arg)
 		if err != nil {
@@ -261,6 +264,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 
 func Test_localLocker_expireOldLocksExpire(t *testing.T) {
 	rng := rand.New(rand.NewSource(0))
+	quorum := 0
 	// Numbers of unique locks
 	for _, locks := range []int{100, 1000, 1e6} {
 		if testing.Short() && locks > 100 {
@@ -289,7 +293,7 @@ func Test_localLocker_expireOldLocksExpire(t *testing.T) {
 								Resources: res,
 								Source:    hex.EncodeToString(tmp[:8]),
 								Owner:     hex.EncodeToString(tmp[8:]),
-								Quorum:    0,
+								Quorum:    &quorum,
 							})
 							if !ok || err != nil {
 								t.Fatal("failed:", err, ok)
@@ -311,7 +315,7 @@ func Test_localLocker_expireOldLocksExpire(t *testing.T) {
 					for _, v := range l.lockMap {
 						for i := range v {
 							if rng.Intn(2) == 0 {
-								v[i].TimeLastRefresh = expired
+								v[i].TimeLastRefresh = expired.UnixNano()
 							}
 						}
 					}
@@ -347,6 +351,7 @@ func Test_localLocker_expireOldLocksExpire(t *testing.T) {
 
 func Test_localLocker_RUnlock(t *testing.T) {
 	rng := rand.New(rand.NewSource(0))
+	quorum := 0
 	// Numbers of unique locks
 	for _, locks := range []int{1, 100, 1000, 1e6} {
 		if testing.Short() && locks > 100 {
@@ -375,7 +380,7 @@ func Test_localLocker_RUnlock(t *testing.T) {
 								Resources: res,
 								Source:    hex.EncodeToString(tmp[:8]),
 								Owner:     hex.EncodeToString(tmp[8:]),
-								Quorum:    0,
+								Quorum:    &quorum,
 							})
 							if !ok || err != nil {
 								t.Fatal("failed:", err, ok)

cmd/lock-rest-client.go

@@ -107,5 +107,5 @@ func newLockAPI(endpoint Endpoint) dsync.NetLocker {
 
 // Returns a lock rest client.
 func newlockRESTClient(ep Endpoint) *lockRESTClient {
-	return &lockRESTClient{globalGrid.Load().Connection(ep.GridHost())}
+	return &lockRESTClient{globalLockGrid.Load().Connection(ep.GridHost())}
 }

cmd/lock-rest-client_test.go

@@ -39,7 +39,7 @@ func TestLockRESTlient(t *testing.T) {
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	err = initGlobalGrid(ctx, []PoolEndpoints{{Endpoints: Endpoints{endpoint, endpointLocal}}})
+	err = initGlobalLockGrid(ctx, []PoolEndpoints{{Endpoints: Endpoints{endpoint, endpointLocal}}})
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/lock-rest-server-common_test.go

@@ -44,7 +44,7 @@ func createLockTestServer(ctx context.Context, t *testing.T) (string, *lockRESTS
 		},
 	}
 	creds := globalActiveCred
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey, "")
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -63,15 +63,15 @@ func TestLockRpcServerRemoveEntry(t *testing.T) {
 		Owner:           "owner",
 		Writer:          true,
 		UID:             "0123-4567",
-		Timestamp:       UTCNow(),
-		TimeLastRefresh: UTCNow(),
+		Timestamp:       UTCNow().UnixNano(),
+		TimeLastRefresh: UTCNow().UnixNano(),
 	}
 	lockRequesterInfo2 := lockRequesterInfo{
 		Owner:           "owner",
 		Writer:          true,
 		UID:             "89ab-cdef",
-		Timestamp:       UTCNow(),
-		TimeLastRefresh: UTCNow(),
+		Timestamp:       UTCNow().UnixNano(),
+		TimeLastRefresh: UTCNow().UnixNano(),
 	}
 
 	locker.ll.lockMap["name"] = []lockRequesterInfo{

cmd/lock-rest-server.go

@@ -111,17 +111,17 @@ func newLockHandler(h grid.HandlerID) *grid.SingleHandler[*dsync.LockArgs, *dsyn
 }
 
 // registerLockRESTHandlers - register lock rest router.
-func registerLockRESTHandlers() {
+func registerLockRESTHandlers(gm *grid.Manager) {
 	lockServer := &lockRESTServer{
 		ll: newLocker(),
 	}
 
-	logger.FatalIf(lockRPCForceUnlock.Register(globalGrid.Load(), lockServer.ForceUnlockHandler), "unable to register handler")
-	logger.FatalIf(lockRPCRefresh.Register(globalGrid.Load(), lockServer.RefreshHandler), "unable to register handler")
-	logger.FatalIf(lockRPCLock.Register(globalGrid.Load(), lockServer.LockHandler), "unable to register handler")
-	logger.FatalIf(lockRPCUnlock.Register(globalGrid.Load(), lockServer.UnlockHandler), "unable to register handler")
-	logger.FatalIf(lockRPCRLock.Register(globalGrid.Load(), lockServer.RLockHandler), "unable to register handler")
-	logger.FatalIf(lockRPCRUnlock.Register(globalGrid.Load(), lockServer.RUnlockHandler), "unable to register handler")
+	logger.FatalIf(lockRPCForceUnlock.Register(gm, lockServer.ForceUnlockHandler), "unable to register handler")
+	logger.FatalIf(lockRPCRefresh.Register(gm, lockServer.RefreshHandler), "unable to register handler")
+	logger.FatalIf(lockRPCLock.Register(gm, lockServer.LockHandler), "unable to register handler")
+	logger.FatalIf(lockRPCUnlock.Register(gm, lockServer.UnlockHandler), "unable to register handler")
+	logger.FatalIf(lockRPCRLock.Register(gm, lockServer.RLockHandler), "unable to register handler")
+	logger.FatalIf(lockRPCRUnlock.Register(gm, lockServer.RUnlockHandler), "unable to register handler")
 
 	globalLockServer = lockServer.ll
 

cmd/logging.go

@@ -184,10 +184,6 @@ func etcdLogOnceIf(ctx context.Context, err error, id string, errKind ...interfa
 	logger.LogOnceIf(ctx, "etcd", err, id, errKind...)
 }
 
-func subnetLogIf(ctx context.Context, err error, errKind ...interface{}) {
-	logger.LogIf(ctx, "subnet", err, errKind...)
-}
-
 func metricsLogIf(ctx context.Context, err error, errKind ...interface{}) {
 	logger.LogIf(ctx, "metrics", err, errKind...)
 }

cmd/main.go

@@ -107,6 +107,11 @@ func newApp(name string) *cli.App {
 
 	// registerCommand registers a cli command.
 	registerCommand := func(command cli.Command) {
+		// avoid registering commands which are not being built (via
+		// go:build tags)
+		if command.Name == "" {
+			return
+		}
 		commands = append(commands, command)
 		commandsTree.Insert(command.Name)
 	}
@@ -134,6 +139,7 @@ func newApp(name string) *cli.App {
 
 	// Register all commands.
 	registerCommand(serverCmd)
+	registerCommand(fmtGenCmd)
 
 	// Set up app.
 	cli.HelpFlag = cli.BoolFlag{
@@ -189,12 +195,14 @@ func printMinIOVersion(c *cli.Context) {
 	io.Copy(c.App.Writer, versionBanner(c))
 }
 
+var debugNoExit = env.Get("_MINIO_DEBUG_NO_EXIT", "") != ""
+
 // Main main for minio server.
 func Main(args []string) {
 	// Set the minio app name.
 	appName := filepath.Base(args[0])
 
-	if env.Get("_MINIO_DEBUG_NO_EXIT", "") != "" {
+	if debugNoExit {
 		freeze := func(_ int) {
 			// Infinite blocking op
 			<-make(chan struct{})

cmd/metacache-entries.go

@@ -259,7 +259,7 @@ func (e *metaCacheEntry) fileInfo(bucket string) (FileInfo, error) {
 		}
 		return e.cached.ToFileInfo(bucket, e.name, "", false, false)
 	}
-	return getFileInfo(e.metadata, bucket, e.name, "", false, false)
+	return getFileInfo(e.metadata, bucket, e.name, "", fileInfoOpts{})
 }
 
 // xlmeta returns the decoded metadata.
@@ -532,6 +532,9 @@ func (m *metaCacheEntriesSorted) fileInfoVersions(bucket, prefix, delimiter, aft
 			}
 
 			for _, version := range fiVersions {
+				if !version.VersionPurgeStatus().Empty() {
+					continue
+				}
 				versioned := vcfg != nil && vcfg.Versioned(entry.name)
 				versions = append(versions, version.ToObjectInfo(bucket, entry.name, versioned))
 			}
@@ -593,7 +596,7 @@ func (m *metaCacheEntriesSorted) fileInfos(bucket, prefix, delimiter string) (ob
 			}
 
 			fi, err := entry.fileInfo(bucket)
-			if err == nil {
+			if err == nil && fi.VersionPurgeStatus().Empty() {
 				versioned := vcfg != nil && vcfg.Versioned(entry.name)
 				objects = append(objects, fi.ToObjectInfo(bucket, entry.name, versioned))
 			}

cmd/metacache-server-pool.go

@@ -153,19 +153,7 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 			} else {
 				// Continue listing
 				o.ID = c.id
-				go func(meta metacache) {
-					// Continuously update while we wait.
-					t := time.NewTicker(metacacheMaxClientWait / 10)
-					defer t.Stop()
-					select {
-					case <-ctx.Done():
-						// Request is done, stop updating.
-						return
-					case <-t.C:
-						meta.lastHandout = time.Now()
-						meta, _ = rpc.UpdateMetacacheListing(ctx, meta)
-					}
-				}(*c)
+				go c.keepAlive(ctx, rpc)
 			}
 		}
 	}
@@ -219,6 +207,9 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 		o.ID = ""
 	}
 
+	if contextCanceled(ctx) {
+		return entries, ctx.Err()
+	}
 	// Do listing in-place.
 	// Create output for our results.
 	// Create filter for results.
@@ -449,5 +440,10 @@ func (z *erasureServerPools) listAndSave(ctx context.Context, o *listPathOptions
 		xioutil.SafeClose(saveCh)
 	}()
 
-	return filteredResults()
+	entries, err = filteredResults()
+	if err == nil {
+		// Check if listing recorded an error.
+		err = meta.getErr()
+	}
+	return entries, err
 }

cmd/metacache-set.go

@@ -805,6 +805,17 @@ func (m *metaCacheRPC) setErr(err string) {
 	*m.meta = meta
 }
 
+// getErr will return an error if the listing failed.
+// The error is not type safe.
+func (m *metaCacheRPC) getErr() error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.meta.status == scanStateError {
+		return errors.New(m.meta.error)
+	}
+	return nil
+}
+
 func (er *erasureObjects) saveMetaCacheStream(ctx context.Context, mc *metaCacheRPC, entries <-chan metaCacheEntry) (err error) {
 	o := mc.o
 	o.debugf(color.Green("saveMetaCacheStream:")+" with options: %#v", o)

cmd/metacache.go

@@ -24,6 +24,8 @@ import (
 	"path"
 	"strings"
 	"time"
+
+	"github.com/minio/pkg/v3/console"
 )
 
 type scanStatus uint8
@@ -97,6 +99,37 @@ func (m *metacache) worthKeeping() bool {
 	return true
 }
 
+// keepAlive will continuously update lastHandout until ctx is canceled.
+func (m metacache) keepAlive(ctx context.Context, rpc *peerRESTClient) {
+	// we intentionally operate on a copy of m, so we can update without locks.
+	t := time.NewTicker(metacacheMaxClientWait / 10)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			// Request is done, stop updating.
+			return
+		case <-t.C:
+			m.lastHandout = time.Now()
+
+			if m2, err := rpc.UpdateMetacacheListing(ctx, m); err == nil {
+				if m2.status != scanStateStarted {
+					if serverDebugLog {
+						console.Debugln("returning", m.id, "due to scan state", m2.status, time.Now().Format(time.RFC3339))
+					}
+					return
+				}
+				m = m2
+				if serverDebugLog {
+					console.Debugln("refreshed", m.id, time.Now().Format(time.RFC3339))
+				}
+			} else if serverDebugLog {
+				console.Debugln("error refreshing", m.id, time.Now().Format(time.RFC3339))
+			}
+		}
+	}
+}
+
 // baseDirFromPrefix will return the base directory given an object path.
 // For example an object with name prefix/folder/object.ext will return `prefix/folder/`.
 func baseDirFromPrefix(prefix string) string {
@@ -116,13 +149,17 @@ func baseDirFromPrefix(prefix string) string {
 // update cache with new status.
 // The updates are conditional so multiple callers can update with different states.
 func (m *metacache) update(update metacache) {
-	m.lastUpdate = UTCNow()
+	now := UTCNow()
+	m.lastUpdate = now
 
-	if m.lastHandout.After(m.lastHandout) {
-		m.lastHandout = UTCNow()
+	if update.lastHandout.After(m.lastHandout) {
+		m.lastHandout = update.lastUpdate
+		if m.lastHandout.After(now) {
+			m.lastHandout = now
+		}
 	}
 	if m.status == scanStateStarted && update.status == scanStateSuccess {
-		m.ended = UTCNow()
+		m.ended = now
 	}
 
 	if m.status == scanStateStarted && update.status != scanStateStarted {
@@ -138,7 +175,7 @@ func (m *metacache) update(update metacache) {
 	if m.error == "" && update.error != "" {
 		m.error = update.error
 		m.status = scanStateError
-		m.ended = UTCNow()
+		m.ended = now
 	}
 	m.fileNotFound = m.fileNotFound || update.fileNotFound
 }

cmd/metrics-resource.go

@@ -262,7 +262,7 @@ func collectDriveMetrics(m madmin.RealtimeMetrics) {
 	latestDriveStatsMu.Unlock()
 
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 
 	for _, d := range localDrives {

cmd/metrics-router.go

@@ -38,7 +38,8 @@ const (
 
 // Standard env prometheus auth type
 const (
-	EnvPrometheusAuthType = "MINIO_PROMETHEUS_AUTH_TYPE"
+	EnvPrometheusAuthType    = "MINIO_PROMETHEUS_AUTH_TYPE"
+	EnvPrometheusOpenMetrics = "MINIO_PROMETHEUS_OPEN_METRICS"
 )
 
 type prometheusAuthType string
@@ -58,14 +59,15 @@ func registerMetricsRouter(router *mux.Router) {
 	if authType == prometheusPublic {
 		auth = NoAuthMiddleware
 	}
+
 	metricsRouter.Handle(prometheusMetricsPathLegacy, auth(metricsHandler()))
 	metricsRouter.Handle(prometheusMetricsV2ClusterPath, auth(metricsServerHandler()))
 	metricsRouter.Handle(prometheusMetricsV2BucketPath, auth(metricsBucketHandler()))
 	metricsRouter.Handle(prometheusMetricsV2NodePath, auth(metricsNodeHandler()))
 	metricsRouter.Handle(prometheusMetricsV2ResourcePath, auth(metricsResourceHandler()))
 
-	// Metrics v3!
-	metricsV3Server := newMetricsV3Server(authType)
+	// Metrics v3
+	metricsV3Server := newMetricsV3Server(auth)
 
 	// Register metrics v3 handler. It also accepts an optional query
 	// parameter `?list` - see handler for details.

cmd/metrics-v2.go

@@ -1688,7 +1688,7 @@ func getMinioProcMetrics() *MetricsGroupV2 {
 		cacheInterval: 10 * time.Second,
 	}
 	mg.RegisterRead(func(ctx context.Context) (metrics []MetricV2) {
-		if runtime.GOOS == "windows" {
+		if runtime.GOOS == globalWindowsOSName || runtime.GOOS == globalMacOSName {
 			return nil
 		}
 

cmd/metrics-v3-api.go

@@ -19,6 +19,8 @@ package cmd
 
 import (
 	"context"
+
+	"github.com/minio/minio-go/v7/pkg/set"
 )
 
 const (
@@ -127,7 +129,8 @@ func loadAPIRequestsHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsC
 // This is a `MetricsLoaderFn`.
 func loadAPIRequestsTTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error {
 	renameLabels := map[string]string{"api": "name"}
-	m.SetHistogram(apiRequestsTTFBSecondsDistribution, httpRequestsDuration, renameLabels, nil,
+	labelsFilter := map[string]set.StringSet{"api": set.CreateStringSet("GetObject")}
+	m.SetHistogram(apiRequestsTTFBSecondsDistribution, httpRequestsDuration, labelsFilter, renameLabels, nil,
 		"type", "s3")
 	return nil
 }
@@ -214,7 +217,8 @@ func loadBucketAPIHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsCac
 // This is a `MetricsLoaderFn`.
 func loadBucketAPITTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache, buckets []string) error {
 	renameLabels := map[string]string{"api": "name"}
-	m.SetHistogram(apiRequestsTTFBSecondsDistribution, bucketHTTPRequestsDuration, renameLabels,
+	labelsFilter := map[string]set.StringSet{"api": set.CreateStringSet("GetObject")}
+	m.SetHistogram(apiRequestsTTFBSecondsDistribution, bucketHTTPRequestsDuration, labelsFilter, renameLabels,
 		buckets, "type", "s3")
 	return nil
 }

cmd/metrics-v3-cluster-usage.go

@@ -139,7 +139,7 @@ var (
 )
 
 // loadClusterUsageBucketMetrics - `MetricsLoaderFn` to load bucket usage metrics.
-func loadClusterUsageBucketMetrics(ctx context.Context, m MetricValues, c *metricsCache, buckets []string) error {
+func loadClusterUsageBucketMetrics(ctx context.Context, m MetricValues, c *metricsCache) error {
 	dataUsageInfo, err := c.dataUsageInfo.Get()
 	if err != nil {
 		metricsLogIf(ctx, err)
@@ -153,11 +153,7 @@ func loadClusterUsageBucketMetrics(ctx context.Context, m MetricValues, c *metri
 
 	m.Set(usageSinceLastUpdateSeconds, float64(time.Since(dataUsageInfo.LastUpdate)))
 
-	for _, bucket := range buckets {
-		usage, ok := dataUsageInfo.BucketsUsage[bucket]
-		if !ok {
-			continue
-		}
+	for bucket, usage := range dataUsageInfo.BucketsUsage {
 		quota, err := globalBucketQuotaSys.Get(ctx, bucket)
 		if err != nil {
 			// Log and continue if we are unable to retrieve metrics for this

cmd/metrics-v3-handler.go

@@ -23,9 +23,12 @@ import (
 	"net/http"
 	"slices"
 	"strings"
+	"sync"
 
+	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/mcontext"
 	"github.com/minio/mux"
+	"github.com/minio/pkg/v3/env"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
@@ -33,41 +36,39 @@ import (
 type promLogger struct{}
 
 func (p promLogger) Println(v ...interface{}) {
-	s := make([]string, 0, len(v))
-	for _, val := range v {
-		s = append(s, fmt.Sprintf("%v", val))
-	}
-	err := fmt.Errorf("metrics handler error: %v", strings.Join(s, " "))
-	metricsLogIf(GlobalContext, err)
+	metricsLogIf(GlobalContext, fmt.Errorf("metrics handler error: %v", v))
 }
 
 type metricsV3Server struct {
 	registry *prometheus.Registry
 	opts     promhttp.HandlerOpts
-	authFn   func(http.Handler) http.Handler
+	auth     func(http.Handler) http.Handler
 
 	metricsData *metricsV3Collection
 }
 
-func newMetricsV3Server(authType prometheusAuthType) *metricsV3Server {
+var (
+	globalMetricsV3CollectorPaths []collectorPath
+	globalMetricsV3Once           sync.Once
+)
+
+func newMetricsV3Server(auth func(h http.Handler) http.Handler) *metricsV3Server {
 	registry := prometheus.NewRegistry()
-	authFn := AuthMiddleware
-	if authType == prometheusPublic {
-		authFn = NoAuthMiddleware
-	}
-
 	metricGroups := newMetricGroups(registry)
-
+	globalMetricsV3Once.Do(func() {
+		globalMetricsV3CollectorPaths = metricGroups.collectorPaths
+	})
 	return &metricsV3Server{
 		registry: registry,
 		opts: promhttp.HandlerOpts{
 			ErrorLog:            promLogger{},
-			ErrorHandling:       promhttp.HTTPErrorOnError,
+			ErrorHandling:       promhttp.ContinueOnError,
 			Registry:            registry,
 			MaxRequestsInFlight: 2,
+			EnableOpenMetrics:   env.Get(EnvPrometheusOpenMetrics, config.EnableOff) == config.EnableOn,
+			ProcessStartTime:    globalBootTime,
 		},
-		authFn: authFn,
-
+		auth:        auth,
 		metricsData: metricGroups,
 	}
 }
@@ -163,7 +164,7 @@ func (h *metricsV3Server) handle(path string, isListingRequest bool, buckets []s
 		http.Error(w, "Metrics Resource Not found", http.StatusNotFound)
 	})
 
-	// Require that metrics path has at least component.
+	// Require that metrics path has one component at least.
 	if path == "/" {
 		return notFoundHandler
 	}
@@ -221,7 +222,7 @@ func (h *metricsV3Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	pathComponents := mux.Vars(r)["pathComps"]
 	isListingRequest := r.Form.Has("list")
 
-	buckets := []string{}
+	var buckets []string
 	if strings.HasPrefix(pathComponents, "/bucket/") {
 		// bucket specific metrics, extract the bucket name from the path.
 		// it's the last part of the path. e.g. /bucket/api/<bucket-name>
@@ -246,5 +247,5 @@ func (h *metricsV3Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	})
 
 	// Add authentication
-	h.authFn(tracedHandler).ServeHTTP(w, r)
+	h.auth(tracedHandler).ServeHTTP(w, r)
 }

cmd/metrics-v3-system-cpu.go

@@ -55,15 +55,17 @@ func loadCPUMetrics(ctx context.Context, m MetricValues, c *metricsCache) error
 	}
 
 	ts := cpuMetrics.TimesStat
-	tot := ts.User + ts.System + ts.Idle + ts.Iowait + ts.Nice + ts.Steal
-	cpuUserVal := math.Round(ts.User/tot*100*100) / 100
-	m.Set(sysCPUUser, cpuUserVal)
-	cpuSystemVal := math.Round(ts.System/tot*100*100) / 100
-	m.Set(sysCPUSystem, cpuSystemVal)
-	cpuNiceVal := math.Round(ts.Nice/tot*100*100) / 100
-	m.Set(sysCPUNice, cpuNiceVal)
-	cpuStealVal := math.Round(ts.Steal/tot*100*100) / 100
-	m.Set(sysCPUSteal, cpuStealVal)
+	if ts != nil {
+		tot := ts.User + ts.System + ts.Idle + ts.Iowait + ts.Nice + ts.Steal
+		cpuUserVal := math.Round(ts.User/tot*100*100) / 100
+		m.Set(sysCPUUser, cpuUserVal)
+		cpuSystemVal := math.Round(ts.System/tot*100*100) / 100
+		m.Set(sysCPUSystem, cpuSystemVal)
+		cpuNiceVal := math.Round(ts.Nice/tot*100*100) / 100
+		m.Set(sysCPUNice, cpuNiceVal)
+		cpuStealVal := math.Round(ts.Steal/tot*100*100) / 100
+		m.Set(sysCPUSteal, cpuStealVal)
+	}
 
 	// metrics-resource.go runs a job to collect resource metrics including their Avg values and
 	// stores them in resourceMetricsMap. We can use it to get the Avg values of CPU idle and IOWait.

cmd/metrics-v3-system-process.go

@@ -156,11 +156,13 @@ func loadProcessMetrics(ctx context.Context, m MetricValues, c *metricsCache) er
 		m.Set(processUptimeSeconds, time.Since(globalBootTime).Seconds())
 	}
 
-	p, err := procfs.Self()
-	if err != nil {
-		metricsLogIf(ctx, err)
-	} else {
-		loadProcFSMetrics(ctx, p, m)
+	if runtime.GOOS != globalWindowsOSName && runtime.GOOS != globalMacOSName {
+		p, err := procfs.Self()
+		if err != nil {
+			metricsLogIf(ctx, err)
+		} else {
+			loadProcFSMetrics(ctx, p, m)
+		}
 	}
 
 	if globalIsDistErasure && globalLockServer != nil {

cmd/metrics-v3-types.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/logger"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
@@ -35,8 +36,8 @@ type collectorPath string
 // converted to snake-case (by replaced '/' and '-' with '_') and prefixed with
 // `minio_`.
 func (cp collectorPath) metricPrefix() string {
-	s := strings.TrimPrefix(string(cp), "/")
-	s = strings.ReplaceAll(s, "/", "_")
+	s := strings.TrimPrefix(string(cp), SlashSeparator)
+	s = strings.ReplaceAll(s, SlashSeparator, "_")
 	s = strings.ReplaceAll(s, "-", "_")
 	return "minio_" + s
 }
@@ -56,8 +57,8 @@ func (cp collectorPath) isDescendantOf(arg string) bool {
 	if len(arg) >= len(descendant) {
 		return false
 	}
-	if !strings.HasSuffix(arg, "/") {
-		arg += "/"
+	if !strings.HasSuffix(arg, SlashSeparator) {
+		arg += SlashSeparator
 	}
 	return strings.HasPrefix(descendant, arg)
 }
@@ -72,10 +73,11 @@ const (
 	GaugeMT
 	// HistogramMT - represents a histogram metric.
 	HistogramMT
-	// rangeL - represents a range label.
-	rangeL = "range"
 )
 
+// rangeL - represents a range label.
+const rangeL = "range"
+
 func (mt MetricType) String() string {
 	switch mt {
 	case CounterMT:
@@ -246,28 +248,38 @@ func (m *MetricValues) Set(name MetricName, value float64, labels ...string) {
 // SetHistogram - sets values for the given MetricName using the provided
 // histogram.
 //
+// `filterByLabels` is a map of label names to list of allowed label values to
+// filter by. Note that this filtering happens before any renaming of labels.
+//
 // `renameLabels` is a map of label names to rename. The keys are the original
 // label names and the values are the new label names.
 //
-// TODO: bucketFilter doc
+// `bucketFilter` is a list of bucket values to filter. If this is non-empty,
+// only metrics for the given buckets are added.
 //
 // `extraLabels` are additional labels to add to each metric. They are ordered
 // label name and value pairs.
 func (m *MetricValues) SetHistogram(name MetricName, hist *prometheus.HistogramVec,
-	renameLabels map[string]string, bucketFilter []string, extraLabels ...string,
+	filterByLabels map[string]set.StringSet, renameLabels map[string]string, bucketFilter []string,
+	extraLabels ...string,
 ) {
 	if _, ok := m.descriptors[name]; !ok {
 		panic(fmt.Sprintf("metric has no description: %s", name))
 	}
 	dummyDesc := MetricDescription{}
 	metricsV2 := getHistogramMetrics(hist, dummyDesc, false)
+mainLoop:
 	for _, metric := range metricsV2 {
+		for label, allowedValues := range filterByLabels {
+			if !allowedValues.Contains(metric.VariableLabels[label]) {
+				continue mainLoop
+			}
+		}
+
 		// If a bucket filter is provided, only add metrics for the given
 		// buckets.
-		if len(bucketFilter) > 0 {
-			if !slices.Contains(bucketFilter, metric.VariableLabels["bucket"]) {
-				continue
-			}
+		if len(bucketFilter) > 0 && !slices.Contains(bucketFilter, metric.VariableLabels["bucket"]) {
+			continue
 		}
 
 		labels := make([]string, 0, len(metric.VariableLabels)*2)

cmd/metrics-v3.go

@@ -270,7 +270,7 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 		loadClusterUsageObjectMetrics,
 	)
 
-	clusterUsageBucketsMG := NewBucketMetricsGroup(clusterUsageBucketsCollectorPath,
+	clusterUsageBucketsMG := NewMetricsGroup(clusterUsageBucketsCollectorPath,
 		[]MetricDescriptor{
 			usageSinceLastUpdateSecondsMD,
 			usageBucketTotalBytesMD,

cmd/naughty-disk_test.go

@@ -222,6 +222,13 @@ func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string
 	return d.disk.CheckParts(ctx, volume, path, fi)
 }
 
+func (d *naughtyDisk) DeleteBulk(ctx context.Context, volume string, paths ...string) (err error) {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.DeleteBulk(ctx, volume, paths...)
+}
+
 func (d *naughtyDisk) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) {
 	if err := d.calcError(); err != nil {
 		return err

cmd/net.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"errors"
-	"fmt"
 	"net"
 	"net/url"
 	"runtime"
@@ -32,8 +31,16 @@ import (
 	xnet "github.com/minio/pkg/v3/net"
 )
 
-// IPv4 addresses of local host.
-var localIP4 = mustGetLocalIP4()
+var (
+	// IPv4 addresses of localhost.
+	localIP4 = mustGetLocalIP4()
+
+	// IPv6 addresses of localhost.
+	localIP6 = mustGetLocalIP6()
+
+	// List of all local loopback addresses.
+	localLoopbacks = mustGetLocalLoopbacks()
+)
 
 // mustSplitHostPort is a wrapper to net.SplitHostPort() where error is assumed to be a fatal.
 func mustSplitHostPort(hostPort string) (host, port string) {
@@ -74,6 +81,16 @@ func mustGetLocalIPs() (ipList []net.IP) {
 	return ipList
 }
 
+func mustGetLocalLoopbacks() (ipList set.StringSet) {
+	ipList = set.NewStringSet()
+	for _, ip := range mustGetLocalIPs() {
+		if ip != nil && ip.IsLoopback() {
+			ipList.Add(ip.String())
+		}
+	}
+	return
+}
+
 // mustGetLocalIP4 returns IPv4 addresses of localhost.  It panics on error.
 func mustGetLocalIP4() (ipList set.StringSet) {
 	ipList = set.NewStringSet()
@@ -160,15 +177,15 @@ func getConsoleEndpoints() (consoleEndpoints []string) {
 	}
 	var ipList []string
 	if globalMinioConsoleHost == "" {
-		ipList = sortIPs(mustGetLocalIP4().ToSlice())
-		ipList = append(ipList, mustGetLocalIP6().ToSlice()...)
+		ipList = sortIPs(localIP4.ToSlice())
+		ipList = append(ipList, localIP6.ToSlice()...)
 	} else {
 		ipList = []string{globalMinioConsoleHost}
 	}
 
+	consoleEndpoints = make([]string, 0, len(ipList))
 	for _, ip := range ipList {
-		endpoint := fmt.Sprintf("%s://%s", getURLScheme(globalIsTLS), net.JoinHostPort(ip, globalMinioConsolePort))
-		consoleEndpoints = append(consoleEndpoints, endpoint)
+		consoleEndpoints = append(consoleEndpoints, getURLScheme(globalIsTLS)+"://"+net.JoinHostPort(ip, globalMinioConsolePort))
 	}
 
 	return consoleEndpoints
@@ -180,15 +197,15 @@ func getAPIEndpoints() (apiEndpoints []string) {
 	}
 	var ipList []string
 	if globalMinioHost == "" {
-		ipList = sortIPs(mustGetLocalIP4().ToSlice())
-		ipList = append(ipList, mustGetLocalIP6().ToSlice()...)
+		ipList = sortIPs(localIP4.ToSlice())
+		ipList = append(ipList, localIP6.ToSlice()...)
 	} else {
 		ipList = []string{globalMinioHost}
 	}
 
+	apiEndpoints = make([]string, 0, len(ipList))
 	for _, ip := range ipList {
-		endpoint := fmt.Sprintf("%s://%s", getURLScheme(globalIsTLS), net.JoinHostPort(ip, globalMinioPort))
-		apiEndpoints = append(apiEndpoints, endpoint)
+		apiEndpoints = append(apiEndpoints, getURLScheme(globalIsTLS)+"://"+net.JoinHostPort(ip, globalMinioPort))
 	}
 
 	return apiEndpoints

cmd/notification.go

@@ -424,7 +424,7 @@ func (sys *NotificationSys) SignalConfigReload(subSys string) []NotificationPeer
 		}
 		client := client
 		ng.Go(GlobalContext, func() error {
-			return client.SignalService(serviceReloadDynamic, subSys, false)
+			return client.SignalService(serviceReloadDynamic, subSys, false, nil)
 		}, idx, *client.host)
 	}
 	return ng.Wait()
@@ -440,14 +440,14 @@ func (sys *NotificationSys) SignalService(sig serviceSignal) []NotificationPeerE
 		client := client
 		ng.Go(GlobalContext, func() error {
 			// force == true preserves the current behavior
-			return client.SignalService(sig, "", false)
+			return client.SignalService(sig, "", false, nil)
 		}, idx, *client.host)
 	}
 	return ng.Wait()
 }
 
 // SignalServiceV2 - calls signal service RPC call on all peers with v2 API
-func (sys *NotificationSys) SignalServiceV2(sig serviceSignal, dryRun bool) []NotificationPeerErr {
+func (sys *NotificationSys) SignalServiceV2(sig serviceSignal, dryRun bool, execAt *time.Time) []NotificationPeerErr {
 	ng := WithNPeers(len(sys.peerClients))
 	for idx, client := range sys.peerClients {
 		if client == nil {
@@ -455,7 +455,7 @@ func (sys *NotificationSys) SignalServiceV2(sig serviceSignal, dryRun bool) []No
 		}
 		client := client
 		ng.Go(GlobalContext, func() error {
-			return client.SignalService(sig, "", dryRun)
+			return client.SignalService(sig, "", dryRun, execAt)
 		}, idx, *client.host)
 	}
 	return ng.Wait()
@@ -1314,7 +1314,7 @@ func (sys *NotificationSys) ServiceFreeze(ctx context.Context, freeze bool) []No
 		}
 		client := client
 		ng.Go(GlobalContext, func() error {
-			return client.SignalService(serviceSig, "", false)
+			return client.SignalService(serviceSig, "", false, nil)
 		}, idx, *client.host)
 	}
 	nerrs := ng.Wait()

cmd/object-api-interface.go

@@ -179,8 +179,9 @@ type DeleteBucketOptions struct {
 
 // BucketOptions provides options for ListBuckets and GetBucketInfo call.
 type BucketOptions struct {
-	Deleted bool // true only when site replication is enabled
-	Cached  bool // true only when we are requesting a cached response instead of hitting the disk for example ListBuckets() call.
+	Deleted    bool // true only when site replication is enabled
+	Cached     bool // true only when we are requesting a cached response instead of hitting the disk for example ListBuckets() call.
+	NoMetadata bool
 }
 
 // SetReplicaStatus sets replica status and timestamp for delete operations in ObjectOptions