count metrics properly for any failures during drive heal (#20193)

or via `mc admin heal --set 1 --pool 1`

.github/workflows/vulncheck.yml

@@ -21,8 +21,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.22.4
-          check-latest: true
+          go-version: 1.22.5
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest
         shell: bash

.typos.toml

@@ -2,6 +2,9 @@
 extend-exclude = [
     ".git/",
     "docs/",
+    "CREDITS",
+    "go.mod",
+    "go.sum",
 ]
 ignore-hidden = false
 
@@ -16,6 +19,7 @@ extend-ignore-re = [
     "MIIDBTCCAe2gAwIBAgIQWHw7h.*",
     'http\.Header\{"X-Amz-Server-Side-Encryptio":',
     "ZoEoZdLlzVbOlT9rbhD7ZN7TLyiYXSAlB79uGEge",
+    "ERRO:",
 ]
 
 [default.extend-words]

CREDITS

@@ -6334,6 +6334,203 @@ SOFTWARE.
 
 ================================================================
 
+github.com/go-ini/ini
+https://github.com/go-ini/ini
+----------------------------------------------------------------
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction, and
+distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by the copyright
+owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all other entities
+that control, are controlled by, or are under common control with that entity.
+For the purposes of this definition, "control" means (i) the power, direct or
+indirect, to cause the direction or management of such entity, whether by
+contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity exercising
+permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications, including
+but not limited to software source code, documentation source, and configuration
+files.
+
+"Object" form shall mean any form resulting from mechanical transformation or
+translation of a Source form, including but not limited to compiled object code,
+generated documentation, and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or Object form, made
+available under the License, as indicated by a copyright notice that is included
+in or attached to the work (an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object form, that
+is based on (or derived from) the Work and for which the editorial revisions,
+annotations, elaborations, or other modifications represent, as a whole, an
+original work of authorship. For the purposes of this License, Derivative Works
+shall not include works that remain separable from, or merely link (or bind by
+name) to the interfaces of, the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including the original version
+of the Work and any modifications or additions to that Work or Derivative Works
+thereof, that is intentionally submitted to Licensor for inclusion in the Work
+by the copyright owner or by an individual or Legal Entity authorized to submit
+on behalf of the copyright owner. For the purposes of this definition,
+"submitted" means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems, and
+issue tracking systems that are managed by, or on behalf of, the Licensor for
+the purpose of discussing and improving the Work, but excluding communication
+that is conspicuously marked or otherwise designated in writing by the copyright
+owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
+of whom a Contribution has been received by Licensor and subsequently
+incorporated within the Work.
+
+2. Grant of Copyright License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the Work and such
+Derivative Works in Source or Object form.
+
+3. Grant of Patent License.
+
+Subject to the terms and conditions of this License, each Contributor hereby
+grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
+irrevocable (except as stated in this section) patent license to make, have
+made, use, offer to sell, sell, import, and otherwise transfer the Work, where
+such license applies only to those patent claims licensable by such Contributor
+that are necessarily infringed by their Contribution(s) alone or by combination
+of their Contribution(s) with the Work to which such Contribution(s) was
+submitted. If You institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work or a
+Contribution incorporated within the Work constitutes direct or contributory
+patent infringement, then any patent licenses granted to You under this License
+for that Work shall terminate as of the date such litigation is filed.
+
+4. Redistribution.
+
+You may reproduce and distribute copies of the Work or Derivative Works thereof
+in any medium, with or without modifications, and in Source or Object form,
+provided that You meet the following conditions:
+
+You must give any other recipients of the Work or Derivative Works a copy of
+this License; and
+You must cause any modified files to carry prominent notices stating that You
+changed the files; and
+You must retain, in the Source form of any Derivative Works that You distribute,
+all copyright, patent, trademark, and attribution notices from the Source form
+of the Work, excluding those notices that do not pertain to any part of the
+Derivative Works; and
+If the Work includes a "NOTICE" text file as part of its distribution, then any
+Derivative Works that You distribute must include a readable copy of the
+attribution notices contained within such NOTICE file, excluding those notices
+that do not pertain to any part of the Derivative Works, in at least one of the
+following places: within a NOTICE text file distributed as part of the
+Derivative Works; within the Source form or documentation, if provided along
+with the Derivative Works; or, within a display generated by the Derivative
+Works, if and wherever such third-party notices normally appear. The contents of
+the NOTICE file are for informational purposes only and do not modify the
+License. You may add Your own attribution notices within Derivative Works that
+You distribute, alongside or as an addendum to the NOTICE text from the Work,
+provided that such additional attribution notices cannot be construed as
+modifying the License.
+You may add Your own copyright statement to Your modifications and may provide
+additional or different license terms and conditions for use, reproduction, or
+distribution of Your modifications, or for any such Derivative Works as a whole,
+provided Your use, reproduction, and distribution of the Work otherwise complies
+with the conditions stated in this License.
+
+5. Submission of Contributions.
+
+Unless You explicitly state otherwise, any Contribution intentionally submitted
+for inclusion in the Work by You to the Licensor shall be under the terms and
+conditions of this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify the terms of
+any separate license agreement you may have executed with Licensor regarding
+such Contributions.
+
+6. Trademarks.
+
+This License does not grant permission to use the trade names, trademarks,
+service marks, or product names of the Licensor, except as required for
+reasonable and customary use in describing the origin of the Work and
+reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty.
+
+Unless required by applicable law or agreed to in writing, Licensor provides the
+Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
+including, without limitation, any warranties or conditions of TITLE,
+NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
+solely responsible for determining the appropriateness of using or
+redistributing the Work and assume any risks associated with Your exercise of
+permissions under this License.
+
+8. Limitation of Liability.
+
+In no event and under no legal theory, whether in tort (including negligence),
+contract, or otherwise, unless required by applicable law (such as deliberate
+and grossly negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special, incidental,
+or consequential damages of any character arising as a result of this License or
+out of the use or inability to use the Work (including but not limited to
+damages for loss of goodwill, work stoppage, computer failure or malfunction, or
+any and all other commercial damages or losses), even if such Contributor has
+been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability.
+
+While redistributing the Work or Derivative Works thereof, You may choose to
+offer, and charge a fee for, acceptance of support, warranty, indemnity, or
+other liability obligations and/or rights consistent with this License. However,
+in accepting such obligations, You may act only on Your own behalf and on Your
+sole responsibility, not on behalf of any other Contributor, and only if You
+agree to indemnify, defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason of your
+accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work
+
+To apply the Apache License to your work, attach the following boilerplate
+notice, with the fields enclosed by brackets "[]" replaced with your own
+identifying information. (Don't include the brackets!) The text should be
+enclosed in the appropriate comment syntax for the file format. We also
+recommend that a file or class name and description of purpose be included on
+the same "printed page" as the copyright notice for easier identification within
+third-party archives.
+
+   Copyright 2014 Unknwon
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+================================================================
+
 github.com/go-jose/go-jose/v4
 https://github.com/go-jose/go-jose/v4
 ----------------------------------------------------------------
@@ -11223,33 +11420,29 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 github.com/gorilla/websocket
 https://github.com/gorilla/websocket
 ----------------------------------------------------------------
-Copyright (c) 2023 The Gorilla Authors. All rights reserved.
+Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
+modification, are permitted provided that the following conditions are met:
 
-	 * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-	 * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-	 * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
+  Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
 
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 ================================================================
 
 github.com/hashicorp/errwrap
@@ -13082,376 +13275,6 @@ Mozilla Public License, version 2.0
    be used to construe this License against a Contributor.
 
 
-10. Versions of the License
-
-10.1. New Versions
-
-      Mozilla Foundation is the license steward. Except as provided in Section
-      10.3, no one other than the license steward has the right to modify or
-      publish new versions of this License. Each version will be given a
-      distinguishing version number.
-
-10.2. Effect of New Versions
-
-      You may distribute the Covered Software under the terms of the version
-      of the License under which You originally received the Covered Software,
-      or under the terms of any subsequent version published by the license
-      steward.
-
-10.3. Modified Versions
-
-      If you create software not governed by this License, and you want to
-      create a new license for such software, you may create and use a
-      modified version of this License if you rename the license and remove
-      any references to the name of the license steward (except to note that
-      such modified license differs from this License).
-
-10.4. Distributing Source Code Form that is Incompatible With Secondary
-      Licenses If You choose to distribute Source Code Form that is
-      Incompatible With Secondary Licenses under the terms of this version of
-      the License, the notice described in Exhibit B of this License must be
-      attached.
-
-Exhibit A - Source Code Form License Notice
-
-      This Source Code Form is subject to the
-      terms of the Mozilla Public License, v.
-      2.0. If a copy of the MPL was not
-      distributed with this file, You can
-      obtain one at
-      http://mozilla.org/MPL/2.0/.
-
-If it is not possible or desirable to put the notice in a particular file,
-then You may include the notice in a location (such as a LICENSE file in a
-relevant directory) where a recipient would be likely to look for such a
-notice.
-
-You may add additional accurate notices of copyright ownership.
-
-Exhibit B - "Incompatible With Secondary Licenses" Notice
-
-      This Source Code Form is "Incompatible
-      With Secondary Licenses", as defined by
-      the Mozilla Public License, v. 2.0.
-
-================================================================
-
-github.com/hashicorp/golang-lru/v2
-https://github.com/hashicorp/golang-lru/v2
-----------------------------------------------------------------
-Copyright (c) 2014 HashiCorp, Inc.
-
-Mozilla Public License, version 2.0
-
-1. Definitions
-
-1.1. "Contributor"
-
-     means each individual or legal entity that creates, contributes to the
-     creation of, or owns Covered Software.
-
-1.2. "Contributor Version"
-
-     means the combination of the Contributions of others (if any) used by a
-     Contributor and that particular Contributor's Contribution.
-
-1.3. "Contribution"
-
-     means Covered Software of a particular Contributor.
-
-1.4. "Covered Software"
-
-     means Source Code Form to which the initial Contributor has attached the
-     notice in Exhibit A, the Executable Form of such Source Code Form, and
-     Modifications of such Source Code Form, in each case including portions
-     thereof.
-
-1.5. "Incompatible With Secondary Licenses"
-     means
-
-     a. that the initial Contributor has attached the notice described in
-        Exhibit B to the Covered Software; or
-
-     b. that the Covered Software was made available under the terms of
-        version 1.1 or earlier of the License, but not also under the terms of
-        a Secondary License.
-
-1.6. "Executable Form"
-
-     means any form of the work other than Source Code Form.
-
-1.7. "Larger Work"
-
-     means a work that combines Covered Software with other material, in a
-     separate file or files, that is not Covered Software.
-
-1.8. "License"
-
-     means this document.
-
-1.9. "Licensable"
-
-     means having the right to grant, to the maximum extent possible, whether
-     at the time of the initial grant or subsequently, any and all of the
-     rights conveyed by this License.
-
-1.10. "Modifications"
-
-     means any of the following:
-
-     a. any file in Source Code Form that results from an addition to,
-        deletion from, or modification of the contents of Covered Software; or
-
-     b. any new file in Source Code Form that contains any Covered Software.
-
-1.11. "Patent Claims" of a Contributor
-
-      means any patent claim(s), including without limitation, method,
-      process, and apparatus claims, in any patent Licensable by such
-      Contributor that would be infringed, but for the grant of the License,
-      by the making, using, selling, offering for sale, having made, import,
-      or transfer of either its Contributions or its Contributor Version.
-
-1.12. "Secondary License"
-
-      means either the GNU General Public License, Version 2.0, the GNU Lesser
-      General Public License, Version 2.1, the GNU Affero General Public
-      License, Version 3.0, or any later versions of those licenses.
-
-1.13. "Source Code Form"
-
-      means the form of the work preferred for making modifications.
-
-1.14. "You" (or "Your")
-
-      means an individual or a legal entity exercising rights under this
-      License. For legal entities, "You" includes any entity that controls, is
-      controlled by, or is under common control with You. For purposes of this
-      definition, "control" means (a) the power, direct or indirect, to cause
-      the direction or management of such entity, whether by contract or
-      otherwise, or (b) ownership of more than fifty percent (50%) of the
-      outstanding shares or beneficial ownership of such entity.
-
-
-2. License Grants and Conditions
-
-2.1. Grants
-
-     Each Contributor hereby grants You a world-wide, royalty-free,
-     non-exclusive license:
-
-     a. under intellectual property rights (other than patent or trademark)
-        Licensable by such Contributor to use, reproduce, make available,
-        modify, display, perform, distribute, and otherwise exploit its
-        Contributions, either on an unmodified basis, with Modifications, or
-        as part of a Larger Work; and
-
-     b. under Patent Claims of such Contributor to make, use, sell, offer for
-        sale, have made, import, and otherwise transfer either its
-        Contributions or its Contributor Version.
-
-2.2. Effective Date
-
-     The licenses granted in Section 2.1 with respect to any Contribution
-     become effective for each Contribution on the date the Contributor first
-     distributes such Contribution.
-
-2.3. Limitations on Grant Scope
-
-     The licenses granted in this Section 2 are the only rights granted under
-     this License. No additional rights or licenses will be implied from the
-     distribution or licensing of Covered Software under this License.
-     Notwithstanding Section 2.1(b) above, no patent license is granted by a
-     Contributor:
-
-     a. for any code that a Contributor has removed from Covered Software; or
-
-     b. for infringements caused by: (i) Your and any other third party's
-        modifications of Covered Software, or (ii) the combination of its
-        Contributions with other software (except as part of its Contributor
-        Version); or
-
-     c. under Patent Claims infringed by Covered Software in the absence of
-        its Contributions.
-
-     This License does not grant any rights in the trademarks, service marks,
-     or logos of any Contributor (except as may be necessary to comply with
-     the notice requirements in Section 3.4).
-
-2.4. Subsequent Licenses
-
-     No Contributor makes additional grants as a result of Your choice to
-     distribute the Covered Software under a subsequent version of this
-     License (see Section 10.2) or under the terms of a Secondary License (if
-     permitted under the terms of Section 3.3).
-
-2.5. Representation
-
-     Each Contributor represents that the Contributor believes its
-     Contributions are its original creation(s) or it has sufficient rights to
-     grant the rights to its Contributions conveyed by this License.
-
-2.6. Fair Use
-
-     This License is not intended to limit any rights You have under
-     applicable copyright doctrines of fair use, fair dealing, or other
-     equivalents.
-
-2.7. Conditions
-
-     Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
-     Section 2.1.
-
-
-3. Responsibilities
-
-3.1. Distribution of Source Form
-
-     All distribution of Covered Software in Source Code Form, including any
-     Modifications that You create or to which You contribute, must be under
-     the terms of this License. You must inform recipients that the Source
-     Code Form of the Covered Software is governed by the terms of this
-     License, and how they can obtain a copy of this License. You may not
-     attempt to alter or restrict the recipients' rights in the Source Code
-     Form.
-
-3.2. Distribution of Executable Form
-
-     If You distribute Covered Software in Executable Form then:
-
-     a. such Covered Software must also be made available in Source Code Form,
-        as described in Section 3.1, and You must inform recipients of the
-        Executable Form how they can obtain a copy of such Source Code Form by
-        reasonable means in a timely manner, at a charge no more than the cost
-        of distribution to the recipient; and
-
-     b. You may distribute such Executable Form under the terms of this
-        License, or sublicense it under different terms, provided that the
-        license for the Executable Form does not attempt to limit or alter the
-        recipients' rights in the Source Code Form under this License.
-
-3.3. Distribution of a Larger Work
-
-     You may create and distribute a Larger Work under terms of Your choice,
-     provided that You also comply with the requirements of this License for
-     the Covered Software. If the Larger Work is a combination of Covered
-     Software with a work governed by one or more Secondary Licenses, and the
-     Covered Software is not Incompatible With Secondary Licenses, this
-     License permits You to additionally distribute such Covered Software
-     under the terms of such Secondary License(s), so that the recipient of
-     the Larger Work may, at their option, further distribute the Covered
-     Software under the terms of either this License or such Secondary
-     License(s).
-
-3.4. Notices
-
-     You may not remove or alter the substance of any license notices
-     (including copyright notices, patent notices, disclaimers of warranty, or
-     limitations of liability) contained within the Source Code Form of the
-     Covered Software, except that You may alter any license notices to the
-     extent required to remedy known factual inaccuracies.
-
-3.5. Application of Additional Terms
-
-     You may choose to offer, and to charge a fee for, warranty, support,
-     indemnity or liability obligations to one or more recipients of Covered
-     Software. However, You may do so only on Your own behalf, and not on
-     behalf of any Contributor. You must make it absolutely clear that any
-     such warranty, support, indemnity, or liability obligation is offered by
-     You alone, and You hereby agree to indemnify every Contributor for any
-     liability incurred by such Contributor as a result of warranty, support,
-     indemnity or liability terms You offer. You may include additional
-     disclaimers of warranty and limitations of liability specific to any
-     jurisdiction.
-
-4. Inability to Comply Due to Statute or Regulation
-
-   If it is impossible for You to comply with any of the terms of this License
-   with respect to some or all of the Covered Software due to statute,
-   judicial order, or regulation then You must: (a) comply with the terms of
-   this License to the maximum extent possible; and (b) describe the
-   limitations and the code they affect. Such description must be placed in a
-   text file included with all distributions of the Covered Software under
-   this License. Except to the extent prohibited by statute or regulation,
-   such description must be sufficiently detailed for a recipient of ordinary
-   skill to be able to understand it.
-
-5. Termination
-
-5.1. The rights granted under this License will terminate automatically if You
-     fail to comply with any of its terms. However, if You become compliant,
-     then the rights granted under this License from a particular Contributor
-     are reinstated (a) provisionally, unless and until such Contributor
-     explicitly and finally terminates Your grants, and (b) on an ongoing
-     basis, if such Contributor fails to notify You of the non-compliance by
-     some reasonable means prior to 60 days after You have come back into
-     compliance. Moreover, Your grants from a particular Contributor are
-     reinstated on an ongoing basis if such Contributor notifies You of the
-     non-compliance by some reasonable means, this is the first time You have
-     received notice of non-compliance with this License from such
-     Contributor, and You become compliant prior to 30 days after Your receipt
-     of the notice.
-
-5.2. If You initiate litigation against any entity by asserting a patent
-     infringement claim (excluding declaratory judgment actions,
-     counter-claims, and cross-claims) alleging that a Contributor Version
-     directly or indirectly infringes any patent, then the rights granted to
-     You by any and all Contributors for the Covered Software under Section
-     2.1 of this License shall terminate.
-
-5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
-     license agreements (excluding distributors and resellers) which have been
-     validly granted by You or Your distributors under this License prior to
-     termination shall survive termination.
-
-6. Disclaimer of Warranty
-
-   Covered Software is provided under this License on an "as is" basis,
-   without warranty of any kind, either expressed, implied, or statutory,
-   including, without limitation, warranties that the Covered Software is free
-   of defects, merchantable, fit for a particular purpose or non-infringing.
-   The entire risk as to the quality and performance of the Covered Software
-   is with You. Should any Covered Software prove defective in any respect,
-   You (not any Contributor) assume the cost of any necessary servicing,
-   repair, or correction. This disclaimer of warranty constitutes an essential
-   part of this License. No use of  any Covered Software is authorized under
-   this License except under this disclaimer.
-
-7. Limitation of Liability
-
-   Under no circumstances and under no legal theory, whether tort (including
-   negligence), contract, or otherwise, shall any Contributor, or anyone who
-   distributes Covered Software as permitted above, be liable to You for any
-   direct, indirect, special, incidental, or consequential damages of any
-   character including, without limitation, damages for lost profits, loss of
-   goodwill, work stoppage, computer failure or malfunction, or any and all
-   other commercial damages or losses, even if such party shall have been
-   informed of the possibility of such damages. This limitation of liability
-   shall not apply to liability for death or personal injury resulting from
-   such party's negligence to the extent applicable law prohibits such
-   limitation. Some jurisdictions do not allow the exclusion or limitation of
-   incidental or consequential damages, so this exclusion and limitation may
-   not apply to You.
-
-8. Litigation
-
-   Any litigation relating to this License may be brought only in the courts
-   of a jurisdiction where the defendant maintains its principal place of
-   business and such litigation shall be governed by laws of that
-   jurisdiction, without reference to its conflict-of-law provisions. Nothing
-   in this Section shall prevent a party's ability to bring cross-claims or
-   counter-claims.
-
-9. Miscellaneous
-
-   This License represents the complete agreement concerning the subject
-   matter hereof. If any provision of this License is held to be
-   unenforceable, such provision shall be reformed only to the extent
-   necessary to make it enforceable. Any law or regulation which provides that
-   the language of a contract shall be construed against the drafter shall not
-   be used to construe this License against a Contributor.
-
-
 10. Versions of the License
 
 10.1. New Versions
@@ -24119,6 +23942,43 @@ SOFTWARE.
 
 ================================================================
 
+github.com/munnerz/goautoneg
+https://github.com/munnerz/goautoneg
+----------------------------------------------------------------
+Copyright (c) 2011, Open Knowledge Foundation Ltd.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+
+    Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+
+    Neither the name of the Open Knowledge Foundation Ltd. nor the
+    names of its contributors may be used to endorse or promote
+    products derived from this software without specific prior written
+    permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
 github.com/nats-io/jwt/v2
 https://github.com/nats-io/jwt/v2
 ----------------------------------------------------------------
@@ -33171,203 +33031,6 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ================================================================
 
-gopkg.in/ini.v1
-https://gopkg.in/ini.v1
-----------------------------------------------------------------
-Apache License
-Version 2.0, January 2004
-http://www.apache.org/licenses/
-
-TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-1. Definitions.
-
-"License" shall mean the terms and conditions for use, reproduction, and
-distribution as defined by Sections 1 through 9 of this document.
-
-"Licensor" shall mean the copyright owner or entity authorized by the copyright
-owner that is granting the License.
-
-"Legal Entity" shall mean the union of the acting entity and all other entities
-that control, are controlled by, or are under common control with that entity.
-For the purposes of this definition, "control" means (i) the power, direct or
-indirect, to cause the direction or management of such entity, whether by
-contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
-outstanding shares, or (iii) beneficial ownership of such entity.
-
-"You" (or "Your") shall mean an individual or Legal Entity exercising
-permissions granted by this License.
-
-"Source" form shall mean the preferred form for making modifications, including
-but not limited to software source code, documentation source, and configuration
-files.
-
-"Object" form shall mean any form resulting from mechanical transformation or
-translation of a Source form, including but not limited to compiled object code,
-generated documentation, and conversions to other media types.
-
-"Work" shall mean the work of authorship, whether in Source or Object form, made
-available under the License, as indicated by a copyright notice that is included
-in or attached to the work (an example is provided in the Appendix below).
-
-"Derivative Works" shall mean any work, whether in Source or Object form, that
-is based on (or derived from) the Work and for which the editorial revisions,
-annotations, elaborations, or other modifications represent, as a whole, an
-original work of authorship. For the purposes of this License, Derivative Works
-shall not include works that remain separable from, or merely link (or bind by
-name) to the interfaces of, the Work and Derivative Works thereof.
-
-"Contribution" shall mean any work of authorship, including the original version
-of the Work and any modifications or additions to that Work or Derivative Works
-thereof, that is intentionally submitted to Licensor for inclusion in the Work
-by the copyright owner or by an individual or Legal Entity authorized to submit
-on behalf of the copyright owner. For the purposes of this definition,
-"submitted" means any form of electronic, verbal, or written communication sent
-to the Licensor or its representatives, including but not limited to
-communication on electronic mailing lists, source code control systems, and
-issue tracking systems that are managed by, or on behalf of, the Licensor for
-the purpose of discussing and improving the Work, but excluding communication
-that is conspicuously marked or otherwise designated in writing by the copyright
-owner as "Not a Contribution."
-
-"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
-of whom a Contribution has been received by Licensor and subsequently
-incorporated within the Work.
-
-2. Grant of Copyright License.
-
-Subject to the terms and conditions of this License, each Contributor hereby
-grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
-irrevocable copyright license to reproduce, prepare Derivative Works of,
-publicly display, publicly perform, sublicense, and distribute the Work and such
-Derivative Works in Source or Object form.
-
-3. Grant of Patent License.
-
-Subject to the terms and conditions of this License, each Contributor hereby
-grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
-irrevocable (except as stated in this section) patent license to make, have
-made, use, offer to sell, sell, import, and otherwise transfer the Work, where
-such license applies only to those patent claims licensable by such Contributor
-that are necessarily infringed by their Contribution(s) alone or by combination
-of their Contribution(s) with the Work to which such Contribution(s) was
-submitted. If You institute patent litigation against any entity (including a
-cross-claim or counterclaim in a lawsuit) alleging that the Work or a
-Contribution incorporated within the Work constitutes direct or contributory
-patent infringement, then any patent licenses granted to You under this License
-for that Work shall terminate as of the date such litigation is filed.
-
-4. Redistribution.
-
-You may reproduce and distribute copies of the Work or Derivative Works thereof
-in any medium, with or without modifications, and in Source or Object form,
-provided that You meet the following conditions:
-
-You must give any other recipients of the Work or Derivative Works a copy of
-this License; and
-You must cause any modified files to carry prominent notices stating that You
-changed the files; and
-You must retain, in the Source form of any Derivative Works that You distribute,
-all copyright, patent, trademark, and attribution notices from the Source form
-of the Work, excluding those notices that do not pertain to any part of the
-Derivative Works; and
-If the Work includes a "NOTICE" text file as part of its distribution, then any
-Derivative Works that You distribute must include a readable copy of the
-attribution notices contained within such NOTICE file, excluding those notices
-that do not pertain to any part of the Derivative Works, in at least one of the
-following places: within a NOTICE text file distributed as part of the
-Derivative Works; within the Source form or documentation, if provided along
-with the Derivative Works; or, within a display generated by the Derivative
-Works, if and wherever such third-party notices normally appear. The contents of
-the NOTICE file are for informational purposes only and do not modify the
-License. You may add Your own attribution notices within Derivative Works that
-You distribute, alongside or as an addendum to the NOTICE text from the Work,
-provided that such additional attribution notices cannot be construed as
-modifying the License.
-You may add Your own copyright statement to Your modifications and may provide
-additional or different license terms and conditions for use, reproduction, or
-distribution of Your modifications, or for any such Derivative Works as a whole,
-provided Your use, reproduction, and distribution of the Work otherwise complies
-with the conditions stated in this License.
-
-5. Submission of Contributions.
-
-Unless You explicitly state otherwise, any Contribution intentionally submitted
-for inclusion in the Work by You to the Licensor shall be under the terms and
-conditions of this License, without any additional terms or conditions.
-Notwithstanding the above, nothing herein shall supersede or modify the terms of
-any separate license agreement you may have executed with Licensor regarding
-such Contributions.
-
-6. Trademarks.
-
-This License does not grant permission to use the trade names, trademarks,
-service marks, or product names of the Licensor, except as required for
-reasonable and customary use in describing the origin of the Work and
-reproducing the content of the NOTICE file.
-
-7. Disclaimer of Warranty.
-
-Unless required by applicable law or agreed to in writing, Licensor provides the
-Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
-including, without limitation, any warranties or conditions of TITLE,
-NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
-solely responsible for determining the appropriateness of using or
-redistributing the Work and assume any risks associated with Your exercise of
-permissions under this License.
-
-8. Limitation of Liability.
-
-In no event and under no legal theory, whether in tort (including negligence),
-contract, or otherwise, unless required by applicable law (such as deliberate
-and grossly negligent acts) or agreed to in writing, shall any Contributor be
-liable to You for damages, including any direct, indirect, special, incidental,
-or consequential damages of any character arising as a result of this License or
-out of the use or inability to use the Work (including but not limited to
-damages for loss of goodwill, work stoppage, computer failure or malfunction, or
-any and all other commercial damages or losses), even if such Contributor has
-been advised of the possibility of such damages.
-
-9. Accepting Warranty or Additional Liability.
-
-While redistributing the Work or Derivative Works thereof, You may choose to
-offer, and charge a fee for, acceptance of support, warranty, indemnity, or
-other liability obligations and/or rights consistent with this License. However,
-in accepting such obligations, You may act only on Your own behalf and on Your
-sole responsibility, not on behalf of any other Contributor, and only if You
-agree to indemnify, defend, and hold each Contributor harmless for any liability
-incurred by, or claims asserted against, such Contributor by reason of your
-accepting any such warranty or additional liability.
-
-END OF TERMS AND CONDITIONS
-
-APPENDIX: How to apply the Apache License to your work
-
-To apply the Apache License to your work, attach the following boilerplate
-notice, with the fields enclosed by brackets "[]" replaced with your own
-identifying information. (Don't include the brackets!) The text should be
-enclosed in the appropriate comment syntax for the file format. We also
-recommend that a file or class name and description of purpose be included on
-the same "printed page" as the copyright notice for easier identification within
-third-party archives.
-
-   Copyright 2014 Unknwon
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-================================================================
-
 gopkg.in/yaml.v2
 https://gopkg.in/yaml.v2
 ----------------------------------------------------------------

Makefile

@@ -34,11 +34,14 @@ verifiers: lint check-gen
 
 check-gen: ## check for updated autogenerated files
 	@go generate ./... >/dev/null
+	@go mod tidy -compat=1.21
 	@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
+	@(! git diff --name-only | grep 'go.sum') || (echo "Non-committed changes in auto-generated go.sum is detected, please commit them to proceed." && false)
 
 lint: getdeps ## runs golangci-lint suite of linters
 	@echo "Running $@ check"
 	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml
+	@command typos && typos ./ || echo "typos binary is not found.. skipping.."
 
 lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
 	@echo "Running $@ check"

buildscripts/verify-healing-empty-erasure-set.sh

@@ -101,7 +101,7 @@ function fail() {
 }
 
 function check_online() {
-	if ! grep -q 'Status:' ${WORK_DIR}/dist-minio-*.log; then
+	if ! grep -q 'API:' ${WORK_DIR}/dist-minio-*.log; then
 		echo "1"
 	fi
 }

buildscripts/verify-healing.sh

@@ -78,7 +78,7 @@ function start_minio_3_node() {
 }
 
 function check_heal() {
-	if ! grep -q 'Status:' ${WORK_DIR}/dist-minio-*.log; then
+	if ! grep -q 'API:' ${WORK_DIR}/dist-minio-*.log; then
 		return 1
 	fi
 

cmd/admin-handler-utils.go

@@ -216,6 +216,12 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
+		case errors.Is(err, errTierInvalidConfig):
+			apiErr = APIError{
+				Code:           "XMinioAdminTierInvalidConfig",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		default:
 			apiErr = errorCodes.ToAPIErrWithErr(toAdminAPIErrCode(ctx, err), err)
 		}

cmd/admin-handlers-idp-ldap.go

@@ -552,7 +552,7 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 		dnList = append(dnList, selfDN)
 	}
 
-	accessKeyMap := make(map[string]madmin.ListAccessKeysLDAPResp)
+	var ldapUserList []string
 	if isAll {
 		ldapUsers, err := globalIAMSys.ListLDAPUsers(ctx)
 		if err != nil {
@@ -560,7 +560,7 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 			return
 		}
 		for user := range ldapUsers {
-			accessKeyMap[user] = madmin.ListAccessKeysLDAPResp{}
+			ldapUserList = append(ldapUserList, user)
 		}
 	} else {
 		for _, userDN := range dnList {
@@ -573,7 +573,7 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 			if foundResult == nil {
 				continue
 			}
-			accessKeyMap[foundResult.NormDN] = madmin.ListAccessKeysLDAPResp{}
+			ldapUserList = append(ldapUserList, foundResult.NormDN)
 		}
 	}
 
@@ -598,9 +598,12 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 		return
 	}
 
-	for dn, accessKeys := range accessKeyMap {
+	accessKeyMap := make(map[string]madmin.ListAccessKeysLDAPResp)
+	for _, internalDN := range ldapUserList {
+		externalDN := globalIAMSys.LDAPConfig.DecodeDN(internalDN)
+		accessKeys := madmin.ListAccessKeysLDAPResp{}
 		if listSTSKeys {
-			stsKeys, err := globalIAMSys.ListSTSAccounts(ctx, dn)
+			stsKeys, err := globalIAMSys.ListSTSAccounts(ctx, internalDN)
 			if err != nil {
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 				return
@@ -614,13 +617,12 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 			}
 			// if only STS keys, skip if user has no STS keys
 			if !listServiceAccounts && len(stsKeys) == 0 {
-				delete(accessKeyMap, dn)
 				continue
 			}
 		}
 
 		if listServiceAccounts {
-			serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, dn)
+			serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, internalDN)
 			if err != nil {
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 				return
@@ -634,11 +636,10 @@ func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.
 			}
 			// if only service accounts, skip if user has no service accounts
 			if !listSTSKeys && len(serviceAccounts) == 0 {
-				delete(accessKeyMap, dn)
 				continue
 			}
 		}
-		accessKeyMap[dn] = accessKeys
+		accessKeyMap[externalDN] = accessKeys
 	}
 
 	data, err := json.Marshal(accessKeyMap)

cmd/admin-handlers-users.go

@@ -2308,7 +2308,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					// clean import.
 					err := globalIAMSys.DeleteServiceAccount(ctx, svcAcctReq.AccessKey, true)
 					if err != nil {
-						delErr := fmt.Errorf("failed to delete existing service account(%s) before importing it: %w", svcAcctReq.AccessKey, err)
+						delErr := fmt.Errorf("failed to delete existing service account (%s) before importing it: %w", svcAcctReq.AccessKey, err)
 						writeErrorResponseJSON(ctx, w, importError(ctx, delErr, allSvcAcctsFile, user), r.URL)
 						return
 					}

cmd/admin-handlers.go

@@ -848,9 +848,10 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
 }
 
 func lriToLockEntry(l lockRequesterInfo, now time.Time, resource, server string) *madmin.LockEntry {
+	t := time.Unix(0, l.Timestamp)
 	entry := &madmin.LockEntry{
-		Timestamp:  l.Timestamp,
-		Elapsed:    now.Sub(l.Timestamp),
+		Timestamp:  t,
+		Elapsed:    now.Sub(t),
 		Resource:   resource,
 		ServerList: []string{server},
 		Source:     l.Source,
@@ -2186,7 +2187,7 @@ func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseHeadersOnly(w)
 }
 
-// KMSKeyStatusHandler - GET /minio/admin/v3/kms/status
+// KMSStatusHandler - GET /minio/admin/v3/kms/status
 func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 

cmd/admin-handlers_test.go

@@ -463,6 +463,7 @@ func TestTopLockEntries(t *testing.T) {
 			Owner:      lri.Owner,
 			ID:         lri.UID,
 			Quorum:     lri.Quorum,
+			Timestamp:  time.Unix(0, lri.Timestamp),
 		})
 	}
 

cmd/admin-heal-ops.go

@@ -761,6 +761,15 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		return nil
 	}
 
+	countOKDrives := func(drives []madmin.HealDriveInfo) (count int) {
+		for _, drive := range drives {
+			if drive.State == madmin.DriveStateOk {
+				count++
+			}
+		}
+		return count
+	}
+
 	// task queued, now wait for the response.
 	select {
 	case res := <-task.respCh:
@@ -781,6 +790,11 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		if res.err != nil {
 			res.result.Detail = res.err.Error()
 		}
+		if res.result.ParityBlocks > 0 && res.result.DataBlocks > 0 && res.result.DataBlocks > res.result.ParityBlocks {
+			if got := countOKDrives(res.result.After.Drives); got < res.result.ParityBlocks {
+				res.result.Detail = fmt.Sprintf("quorum loss - expected %d minimum, got drive states in OK %d", res.result.ParityBlocks, got)
+			}
+		}
 		return h.pushHealResultItem(res.result)
 	case <-h.ctx.Done():
 		return nil

cmd/admin-router.go

@@ -341,6 +341,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-jobs").HandlerFunc(
 			adminMiddleware(adminAPI.ListBatchJobs))
 
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/status-job").HandlerFunc(
+			adminMiddleware(adminAPI.BatchJobStatus))
+
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/describe-job").HandlerFunc(
 			adminMiddleware(adminAPI.DescribeBatchJob))
 		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/cancel-job").HandlerFunc(

cmd/background-newdisks-heal-ops.go

@@ -362,10 +362,10 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 
 func getLocalDisksToHeal() (disksToHeal Endpoints) {
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 	for _, disk := range localDrives {
-		_, err := disk.GetDiskID()
+		_, err := disk.DiskInfo(context.Background(), DiskInfoOptions{})
 		if errors.Is(err, errUnformattedDisk) {
 			disksToHeal = append(disksToHeal, disk.Endpoint())
 			continue
@@ -393,6 +393,17 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		return fmt.Errorf("Unexpected error disk must be initialized by now after formatting: %s", endpoint)
 	}
 
+	_, err := disk.DiskInfo(ctx, DiskInfoOptions{})
+	if err != nil {
+		if errors.Is(err, errDriveIsRoot) {
+			// This is a root drive, ignore and move on
+			return nil
+		}
+		if !errors.Is(err, errUnformattedDisk) {
+			return err
+		}
+	}
+
 	// Prevent parallel erasure set healing
 	locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%d/%d", poolIdx, setIdx))
 	lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)

cmd/batch-expire.go

@@ -36,6 +36,7 @@ import (
 	"github.com/minio/pkg/v3/env"
 	"github.com/minio/pkg/v3/wildcard"
 	"github.com/minio/pkg/v3/workers"
+	"github.com/minio/pkg/v3/xtime"
 	"gopkg.in/yaml.v3"
 )
 
@@ -116,7 +117,7 @@ func (p BatchJobExpirePurge) Validate() error {
 // BatchJobExpireFilter holds all the filters currently supported for batch replication
 type BatchJobExpireFilter struct {
 	line, col     int
-	OlderThan     time.Duration       `yaml:"olderThan,omitempty" json:"olderThan"`
+	OlderThan     xtime.Duration      `yaml:"olderThan,omitempty" json:"olderThan"`
 	CreatedBefore *time.Time          `yaml:"createdBefore,omitempty" json:"createdBefore"`
 	Tags          []BatchJobKV        `yaml:"tags,omitempty" json:"tags"`
 	Metadata      []BatchJobKV        `yaml:"metadata,omitempty" json:"metadata"`
@@ -162,7 +163,7 @@ func (ef BatchJobExpireFilter) Matches(obj ObjectInfo, now time.Time) bool {
 	if len(ef.Name) > 0 && !wildcard.Match(ef.Name, obj.Name) {
 		return false
 	}
-	if ef.OlderThan > 0 && now.Sub(obj.ModTime) <= ef.OlderThan {
+	if ef.OlderThan > 0 && now.Sub(obj.ModTime) <= ef.OlderThan.D() {
 		return false
 	}
 
@@ -514,7 +515,7 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 		JobType:   string(job.Type()),
 		StartTime: job.Started,
 	}
-	if err := ri.load(ctx, api, job); err != nil {
+	if err := ri.loadOrInit(ctx, api, job); err != nil {
 		return err
 	}
 
@@ -552,22 +553,25 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 	go func() {
 		saveTicker := time.NewTicker(10 * time.Second)
 		defer saveTicker.Stop()
-		for {
+		quit := false
+		after := time.Minute
+		for !quit {
 			select {
 			case <-saveTicker.C:
-				// persist in-memory state to disk after every 10secs.
-				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
-
 			case <-ctx.Done():
-				// persist in-memory state immediately before exiting due to context cancellation.
-				batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
-				return
-
+				quit = true
 			case <-saverQuitCh:
-				// persist in-memory state immediately to disk.
-				batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
-				return
+				quit = true
 			}
+
+			if quit {
+				// save immediately if we are quitting
+				after = 0
+			}
+
+			ctx, cancel := context.WithTimeout(GlobalContext, 30*time.Second) // independent context
+			batchLogIf(ctx, ri.updateAfter(ctx, api, after, job))
+			cancel()
 		}
 	}()
 
@@ -584,7 +588,7 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 		versionsCount int
 		toDel         []expireObjInfo
 	)
-	failed := true
+	failed := false
 	for result := range results {
 		if result.Err != nil {
 			failed = true

cmd/batch-expire_gen.go

@@ -306,7 +306,7 @@ func (z *BatchJobExpireFilter) DecodeMsg(dc *msgp.Reader) (err error) {
 		}
 		switch msgp.UnsafeString(field) {
 		case "OlderThan":
-			z.OlderThan, err = dc.ReadDuration()
+			err = z.OlderThan.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "OlderThan")
 				return
@@ -433,7 +433,7 @@ func (z *BatchJobExpireFilter) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteDuration(z.OlderThan)
+	err = z.OlderThan.EncodeMsg(en)
 	if err != nil {
 		err = msgp.WrapError(err, "OlderThan")
 		return
@@ -544,7 +544,11 @@ func (z *BatchJobExpireFilter) MarshalMsg(b []byte) (o []byte, err error) {
 	// map header, size 8
 	// string "OlderThan"
 	o = append(o, 0x88, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
-	o = msgp.AppendDuration(o, z.OlderThan)
+	o, err = z.OlderThan.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "OlderThan")
+		return
+	}
 	// string "CreatedBefore"
 	o = append(o, 0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65)
 	if z.CreatedBefore == nil {
@@ -613,7 +617,7 @@ func (z *BatchJobExpireFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		}
 		switch msgp.UnsafeString(field) {
 		case "OlderThan":
-			z.OlderThan, bts, err = msgp.ReadDurationBytes(bts)
+			bts, err = z.OlderThan.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "OlderThan")
 				return
@@ -734,7 +738,7 @@ func (z *BatchJobExpireFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobExpireFilter) Msgsize() (s int) {
-	s = 1 + 10 + msgp.DurationSize + 14
+	s = 1 + 10 + z.OlderThan.Msgsize() + 14
 	if z.CreatedBefore == nil {
 		s += msgp.NilSize
 	} else {

cmd/batch-expire_test.go

@@ -20,7 +20,7 @@ package cmd
 import (
 	"testing"
 
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 func TestParseBatchJobExpire(t *testing.T) {
@@ -32,7 +32,7 @@ expire: # Expire objects that match a condition
   rules:
     - type: object  # regular objects with zero or more older versions
       name: NAME # match object names that satisfy the wildcard expression.
-      olderThan: 70h # match objects older than this value
+      olderThan: 7d10h # match objects older than this value
       createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
       tags:
         - key: name
@@ -64,7 +64,7 @@ expire: # Expire objects that match a condition
     delay: 500ms # least amount of delay between each retry
 `
 	var job BatchJobRequest
-	err := yaml.UnmarshalStrict([]byte(expireYaml), &job)
+	err := yaml.Unmarshal([]byte(expireYaml), &job)
 	if err != nil {
 		t.Fatal("Failed to parse batch-job-expire yaml", err)
 	}

cmd/batch-handlers.go

@@ -28,6 +28,7 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
+	"path/filepath"
 	"runtime"
 	"strconv"
 	"strings"
@@ -57,6 +58,11 @@ import (
 
 var globalBatchConfig batch.Config
 
+const (
+	// Keep the completed/failed job stats 3 days before removing it
+	oldJobsExpiration = 3 * 24 * time.Hour
+)
+
 // BatchJobRequest this is an internal data structure not for external consumption.
 type BatchJobRequest struct {
 	ID        string               `yaml:"-" json:"name"`
@@ -262,7 +268,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 		JobType:   string(job.Type()),
 		StartTime: job.Started,
 	}
-	if err := ri.load(ctx, api, job); err != nil {
+	if err := ri.loadOrInit(ctx, api, job); err != nil {
 		return err
 	}
 	if ri.Complete {
@@ -281,12 +287,12 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	isStorageClassOnly := len(r.Flags.Filter.Metadata) == 1 && strings.EqualFold(r.Flags.Filter.Metadata[0].Key, xhttp.AmzStorageClass)
 
 	skip := func(oi ObjectInfo) (ok bool) {
-		if r.Flags.Filter.OlderThan > 0 && time.Since(oi.ModTime) < r.Flags.Filter.OlderThan {
+		if r.Flags.Filter.OlderThan > 0 && time.Since(oi.ModTime) < r.Flags.Filter.OlderThan.D() {
 			// skip all objects that are newer than specified older duration
 			return true
 		}
 
-		if r.Flags.Filter.NewerThan > 0 && time.Since(oi.ModTime) >= r.Flags.Filter.NewerThan {
+		if r.Flags.Filter.NewerThan > 0 && time.Since(oi.ModTime) >= r.Flags.Filter.NewerThan.D() {
 			// skip all objects that are older than specified newer duration
 			return true
 		}
@@ -534,7 +540,7 @@ func toObjectInfo(bucket, object string, objInfo miniogo.ObjectInfo) ObjectInfo
 	return oi
 }
 
-func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo) error {
+func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo, prefix string) error {
 	input := make(chan minio.SnowballObject, 1)
 	opts := minio.SnowballOptions{
 		Opts:     minio.PutObjectOptions{},
@@ -556,6 +562,10 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
 				continue
 			}
 
+			if prefix != "" {
+				entry.Name = pathJoin(prefix, entry.Name)
+			}
+
 			snowballObj := minio.SnowballObject{
 				// Create path to store objects within the bucket.
 				Key:       entry.Name,
@@ -719,63 +729,85 @@ const (
 
 	batchReplJobAPIVersion        = "v1"
 	batchReplJobDefaultRetries    = 3
-	batchReplJobDefaultRetryDelay = 250 * time.Millisecond
+	batchReplJobDefaultRetryDelay = time.Second
 )
 
-func getJobReportPath(job BatchJobRequest) string {
-	var fileName string
-	switch {
-	case job.Replicate != nil:
-		fileName = batchReplName
-	case job.KeyRotate != nil:
-		fileName = batchKeyRotationName
-	case job.Expire != nil:
-		fileName = batchExpireName
-	}
-	return pathJoin(batchJobReportsPrefix, job.ID, fileName)
-}
-
 func getJobPath(job BatchJobRequest) string {
 	return pathJoin(batchJobPrefix, job.ID)
 }
 
+func (ri *batchJobInfo) getJobReportPath() (string, error) {
+	var fileName string
+	switch madmin.BatchJobType(ri.JobType) {
+	case madmin.BatchJobReplicate:
+		fileName = batchReplName
+	case madmin.BatchJobKeyRotate:
+		fileName = batchKeyRotationName
+	case madmin.BatchJobExpire:
+		fileName = batchExpireName
+	default:
+		return "", fmt.Errorf("unknown job type: %v", ri.JobType)
+	}
+	return pathJoin(batchJobReportsPrefix, ri.JobID, fileName), nil
+}
+
+func (ri *batchJobInfo) loadOrInit(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+	err := ri.load(ctx, api, job)
+	if errors.Is(err, errNoSuchJob) {
+		switch {
+		case job.Replicate != nil:
+			ri.Version = batchReplVersionV1
+			ri.RetryAttempts = batchReplJobDefaultRetries
+			if job.Replicate.Flags.Retry.Attempts > 0 {
+				ri.RetryAttempts = job.Replicate.Flags.Retry.Attempts
+			}
+		case job.KeyRotate != nil:
+			ri.Version = batchKeyRotateVersionV1
+			ri.RetryAttempts = batchKeyRotateJobDefaultRetries
+			if job.KeyRotate.Flags.Retry.Attempts > 0 {
+				ri.RetryAttempts = job.KeyRotate.Flags.Retry.Attempts
+			}
+		case job.Expire != nil:
+			ri.Version = batchExpireVersionV1
+			ri.RetryAttempts = batchExpireJobDefaultRetries
+			if job.Expire.Retry.Attempts > 0 {
+				ri.RetryAttempts = job.Expire.Retry.Attempts
+			}
+		}
+		return nil
+	}
+	return err
+}
+
 func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+	path, err := job.getJobReportPath()
+	if err != nil {
+		batchLogIf(ctx, err)
+		return err
+	}
+	return ri.loadByPath(ctx, api, path)
+}
+
+func (ri *batchJobInfo) loadByPath(ctx context.Context, api ObjectLayer, path string) error {
 	var format, version uint16
-	switch {
-	case job.Replicate != nil:
+	switch filepath.Base(path) {
+	case batchReplName:
 		version = batchReplVersionV1
 		format = batchReplFormat
-	case job.KeyRotate != nil:
+	case batchKeyRotationName:
 		version = batchKeyRotateVersionV1
 		format = batchKeyRotationFormat
-	case job.Expire != nil:
+	case batchExpireName:
 		version = batchExpireVersionV1
 		format = batchExpireFormat
 	default:
 		return errors.New("no supported batch job request specified")
 	}
-	data, err := readConfig(ctx, api, getJobReportPath(job))
+
+	data, err := readConfig(ctx, api, path)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) || isErrObjectNotFound(err) {
-			ri.Version = int(version)
-			switch {
-			case job.Replicate != nil:
-				ri.RetryAttempts = batchReplJobDefaultRetries
-				if job.Replicate.Flags.Retry.Attempts > 0 {
-					ri.RetryAttempts = job.Replicate.Flags.Retry.Attempts
-				}
-			case job.KeyRotate != nil:
-				ri.RetryAttempts = batchKeyRotateJobDefaultRetries
-				if job.KeyRotate.Flags.Retry.Attempts > 0 {
-					ri.RetryAttempts = job.KeyRotate.Flags.Retry.Attempts
-				}
-			case job.Expire != nil:
-				ri.RetryAttempts = batchExpireJobDefaultRetries
-				if job.Expire.Retry.Attempts > 0 {
-					ri.RetryAttempts = job.Expire.Retry.Attempts
-				}
-			}
-			return nil
+			return errNoSuchJob
 		}
 		return err
 	}
@@ -919,7 +951,12 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 		if err != nil {
 			return err
 		}
-		return saveConfig(ctx, api, getJobReportPath(job), buf)
+		path, err := ri.getJobReportPath()
+		if err != nil {
+			batchLogIf(ctx, err)
+			return err
+		}
+		return saveConfig(ctx, api, path, buf)
 	}
 	ri.mu.Unlock()
 	return nil
@@ -944,8 +981,10 @@ func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo,
 	ri.mu.Lock()
 	defer ri.mu.Unlock()
 
-	ri.Bucket = bucket
-	ri.Object = info.Name
+	if success {
+		ri.Bucket = bucket
+		ri.Object = info.Name
+	}
 	ri.countItem(info.Size, info.DeleteMarker, success, attempt)
 }
 
@@ -971,7 +1010,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		JobType:   string(job.Type()),
 		StartTime: job.Started,
 	}
-	if err := ri.load(ctx, api, job); err != nil {
+	if err := ri.loadOrInit(ctx, api, job); err != nil {
 		return err
 	}
 	if ri.Complete {
@@ -981,18 +1020,18 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	lastObject := ri.Object
 
 	delay := job.Replicate.Flags.Retry.Delay
-	if delay == 0 {
+	if delay < time.Second {
 		delay = batchReplJobDefaultRetryDelay
 	}
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	selectObj := func(info FileInfo) (ok bool) {
-		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
+		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan.D() {
 			// skip all objects that are newer than specified older duration
 			return false
 		}
 
-		if r.Flags.Filter.NewerThan > 0 && time.Since(info.ModTime) >= r.Flags.Filter.NewerThan {
+		if r.Flags.Filter.NewerThan > 0 && time.Since(info.ModTime) >= r.Flags.Filter.NewerThan.D() {
 			// skip all objects that are older than specified newer duration
 			return false
 		}
@@ -1071,86 +1110,84 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 	c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
 
-	var (
-		walkCh = make(chan itemOrErr[ObjectInfo], 100)
-		slowCh = make(chan itemOrErr[ObjectInfo], 100)
-	)
-
-	if !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
-		go func() {
-			// Snowball currently needs the high level minio-go Client, not the Core one
-			cl, err := miniogo.New(u.Host, &miniogo.Options{
-				Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
-				Secure:       u.Scheme == "https",
-				Transport:    getRemoteInstanceTransport(),
-				BucketLookup: lookupStyle(r.Target.Path),
-			})
-			if err != nil {
-				batchLogIf(ctx, err)
-				return
-			}
-
-			// Already validated before arriving here
-			smallerThan, _ := humanize.ParseBytes(*r.Source.Snowball.SmallerThan)
-
-			batch := make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
-			writeFn := func(batch []ObjectInfo) {
-				if len(batch) > 0 {
-					if err := r.writeAsArchive(ctx, api, cl, batch); err != nil {
-						batchLogIf(ctx, err)
-						for _, b := range batch {
-							slowCh <- itemOrErr[ObjectInfo]{Item: b}
-						}
-					} else {
-						ri.trackCurrentBucketBatch(r.Source.Bucket, batch)
-						globalBatchJobsMetrics.save(job.ID, ri)
-						// persist in-memory state to disk after every 10secs.
-						batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
-					}
-				}
-			}
-			for obj := range walkCh {
-				if obj.Item.DeleteMarker || !obj.Item.VersionPurgeStatus.Empty() || obj.Item.Size >= int64(smallerThan) {
-					slowCh <- obj
-					continue
-				}
-
-				batch = append(batch, obj.Item)
-
-				if len(batch) < *r.Source.Snowball.Batch {
-					continue
-				}
-				writeFn(batch)
-				batch = batch[:0]
-			}
-			writeFn(batch)
-			xioutil.SafeClose(slowCh)
-		}()
-	} else {
-		slowCh = walkCh
-	}
-
-	workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_REPLICATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
-	if err != nil {
-		return err
-	}
-
-	wk, err := workers.New(workerSize)
-	if err != nil {
-		// invalid worker size.
-		return err
-	}
-
-	walkQuorum := env.Get("_MINIO_BATCH_REPLICATION_WALK_QUORUM", "strict")
-	if walkQuorum == "" {
-		walkQuorum = "strict"
-	}
-
 	retryAttempts := ri.RetryAttempts
 	retry := false
 	for attempts := 1; attempts <= retryAttempts; attempts++ {
 		attempts := attempts
+		var (
+			walkCh = make(chan itemOrErr[ObjectInfo], 100)
+			slowCh = make(chan itemOrErr[ObjectInfo], 100)
+		)
 
+		if r.Source.Snowball.Disable != nil && !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
+			go func() {
+				// Snowball currently needs the high level minio-go Client, not the Core one
+				cl, err := miniogo.New(u.Host, &miniogo.Options{
+					Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
+					Secure:       u.Scheme == "https",
+					Transport:    getRemoteInstanceTransport(),
+					BucketLookup: lookupStyle(r.Target.Path),
+				})
+				if err != nil {
+					batchLogOnceIf(ctx, err, job.ID+"miniogo.New")
+					return
+				}
+
+				// Already validated before arriving here
+				smallerThan, _ := humanize.ParseBytes(*r.Source.Snowball.SmallerThan)
+
+				batch := make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
+				writeFn := func(batch []ObjectInfo) {
+					if len(batch) > 0 {
+						if err := r.writeAsArchive(ctx, api, cl, batch, r.Target.Prefix); err != nil {
+							batchLogOnceIf(ctx, err, job.ID+"writeAsArchive")
+							for _, b := range batch {
+								slowCh <- itemOrErr[ObjectInfo]{Item: b}
+							}
+						} else {
+							ri.trackCurrentBucketBatch(r.Source.Bucket, batch)
+							globalBatchJobsMetrics.save(job.ID, ri)
+							// persist in-memory state to disk after every 10secs.
+							batchLogOnceIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job), job.ID+"updateAfter")
+						}
+					}
+				}
+				for obj := range walkCh {
+					if obj.Item.DeleteMarker || !obj.Item.VersionPurgeStatus.Empty() || obj.Item.Size >= int64(smallerThan) {
+						slowCh <- obj
+						continue
+					}
+
+					batch = append(batch, obj.Item)
+
+					if len(batch) < *r.Source.Snowball.Batch {
+						continue
+					}
+					writeFn(batch)
+					batch = batch[:0]
+				}
+				writeFn(batch)
+				xioutil.SafeClose(slowCh)
+			}()
+		} else {
+			slowCh = walkCh
+		}
+
+		workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_REPLICATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
+		if err != nil {
+			return err
+		}
+
+		wk, err := workers.New(workerSize)
+		if err != nil {
+			// invalid worker size.
+			return err
+		}
+
+		walkQuorum := env.Get("_MINIO_BATCH_REPLICATION_WALK_QUORUM", "strict")
+		if walkQuorum == "" {
+			walkQuorum = "strict"
+		}
 		ctx, cancel := context.WithCancel(ctx)
 		// one of source/target is s3, skip delete marker and all versions under the same object name.
 		s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
@@ -1171,7 +1208,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		for res := range slowCh {
 			if res.Err != nil {
 				ri.Failed = true
-				batchLogIf(ctx, res.Err)
+				batchLogOnceIf(ctx, res.Err, job.ID+"res.Err")
 				continue
 			}
 			result := res.Item
@@ -1198,7 +1235,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 						return
 					}
 					stopFn(result, err)
-					batchLogIf(ctx, err)
+					batchLogOnceIf(ctx, err, job.ID+"ReplicateToTarget")
 					success = false
 				} else {
 					stopFn(result, nil)
@@ -1206,7 +1243,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				ri.trackCurrentBucketObject(r.Source.Bucket, result, success, attempts)
 				globalBatchJobsMetrics.save(job.ID, ri)
 				// persist in-memory state to disk after every 10secs.
-				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+				batchLogOnceIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job), job.ID+"updateAfter2")
 
 				if wait := globalBatchConfig.ReplicationWait(); wait > 0 {
 					time.Sleep(wait)
@@ -1221,10 +1258,10 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 		globalBatchJobsMetrics.save(job.ID, ri)
 		// persist in-memory state to disk.
-		batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+		batchLogOnceIf(ctx, ri.updateAfter(ctx, api, 0, job), job.ID+"updateAfter3")
 
 		if err := r.Notify(ctx, ri); err != nil {
-			batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
+			batchLogOnceIf(ctx, fmt.Errorf("unable to notify %v", err), job.ID+"notify")
 		}
 
 		cancel()
@@ -1436,10 +1473,24 @@ func (j BatchJobRequest) Validate(ctx context.Context, o ObjectLayer) error {
 }
 
 func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) {
-	deleteConfig(ctx, api, getJobReportPath(j))
 	deleteConfig(ctx, api, getJobPath(j))
 }
 
+func (j BatchJobRequest) getJobReportPath() (string, error) {
+	var fileName string
+	switch {
+	case j.Replicate != nil:
+		fileName = batchReplName
+	case j.KeyRotate != nil:
+		fileName = batchKeyRotationName
+	case j.Expire != nil:
+		fileName = batchExpireName
+	default:
+		return "", errors.New("unknown job type")
+	}
+	return pathJoin(batchJobReportsPrefix, j.ID, fileName), nil
+}
+
 func (j *BatchJobRequest) save(ctx context.Context, api ObjectLayer) error {
 	if j.Replicate == nil && j.KeyRotate == nil && j.Expire == nil {
 		return errInvalidArgument
@@ -1520,6 +1571,9 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, result.Err), r.URL)
 			return
 		}
+		if strings.HasPrefix(result.Item.Name, batchJobReportsPrefix+slashSeparator) {
+			continue
+		}
 		req := &BatchJobRequest{}
 		if err := req.load(ctx, objectAPI, result.Item.Name); err != nil {
 			if !errors.Is(err, errNoSuchJob) {
@@ -1542,6 +1596,55 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
 	batchLogIf(ctx, json.NewEncoder(w).Encode(&listResult))
 }
 
+// BatchJobStatus - returns the status of a batch job saved in the disk
+func (a adminAPIHandlers) BatchJobStatus(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListBatchJobsAction)
+	if objectAPI == nil {
+		return
+	}
+
+	jobID := r.Form.Get("jobId")
+	if jobID == "" {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL)
+		return
+	}
+
+	req := BatchJobRequest{ID: jobID}
+	if i := strings.Index(jobID, "-"); i > 0 {
+		switch madmin.BatchJobType(jobID[:i]) {
+		case madmin.BatchJobReplicate:
+			req.Replicate = &BatchJobReplicateV1{}
+		case madmin.BatchJobKeyRotate:
+			req.KeyRotate = &BatchJobKeyRotateV1{}
+		case madmin.BatchJobExpire:
+			req.Expire = &BatchJobExpire{}
+		default:
+			writeErrorResponseJSON(ctx, w, toAPIError(ctx, errors.New("job ID format unrecognized")), r.URL)
+			return
+		}
+	}
+
+	ri := &batchJobInfo{}
+	if err := ri.load(ctx, objectAPI, req); err != nil {
+		if !errors.Is(err, errNoSuchJob) {
+			batchLogIf(ctx, err)
+		}
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	buf, err := json.Marshal(madmin.BatchJobStatus{LastMetric: ri.metric()})
+	if err != nil {
+		batchLogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	w.Write(buf)
+}
+
 var errNoSuchJob = errors.New("no such job")
 
 // DescribeBatchJob returns the currently active batch job definition
@@ -1633,7 +1736,7 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	job.ID = fmt.Sprintf("%s%s%d", shortuuid.New(), getKeySeparator(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
+	job.ID = fmt.Sprintf("%s-%s%s%d", job.Type(), shortuuid.New(), getKeySeparator(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
 	job.User = user
 	job.Started = time.Now()
 
@@ -1721,11 +1824,60 @@ func newBatchJobPool(ctx context.Context, o ObjectLayer, workers int) *BatchJobP
 		jobCancelers: make(map[string]context.CancelFunc),
 	}
 	jpool.ResizeWorkers(workers)
-	jpool.resume()
+
+	randomWait := func() time.Duration {
+		// randomWait depends on the number of nodes to avoid triggering resume and cleanups at the same time.
+		return time.Duration(rand.Float64() * float64(time.Duration(globalEndpoints.NEndpoints())*time.Hour))
+	}
+
+	go func() {
+		jpool.resume(randomWait)
+		jpool.cleanupReports(randomWait)
+	}()
+
 	return jpool
 }
 
-func (j *BatchJobPool) resume() {
+func (j *BatchJobPool) cleanupReports(randomWait func() time.Duration) {
+	t := time.NewTimer(randomWait())
+	defer t.Stop()
+
+	for {
+		select {
+		case <-GlobalContext.Done():
+			return
+		case <-t.C:
+			results := make(chan itemOrErr[ObjectInfo], 100)
+			ctx, cancel := context.WithCancel(j.ctx)
+			defer cancel()
+			if err := j.objLayer.Walk(ctx, minioMetaBucket, batchJobReportsPrefix, results, WalkOptions{}); err != nil {
+				batchLogIf(j.ctx, err)
+				t.Reset(randomWait())
+				continue
+			}
+			for result := range results {
+				if result.Err != nil {
+					batchLogIf(j.ctx, result.Err)
+					continue
+				}
+				ri := &batchJobInfo{}
+				if err := ri.loadByPath(ctx, j.objLayer, result.Item.Name); err != nil {
+					batchLogIf(ctx, err)
+					continue
+				}
+				if (ri.Complete || ri.Failed) && time.Since(ri.LastUpdate) > oldJobsExpiration {
+					deleteConfig(ctx, j.objLayer, result.Item.Name)
+				}
+			}
+
+			t.Reset(randomWait())
+		}
+	}
+}
+
+func (j *BatchJobPool) resume(randomWait func() time.Duration) {
+	time.Sleep(randomWait())
+
 	results := make(chan itemOrErr[ObjectInfo], 100)
 	ctx, cancel := context.WithCancel(j.ctx)
 	defer cancel()
@@ -1738,6 +1890,9 @@ func (j *BatchJobPool) resume() {
 			batchLogIf(j.ctx, result.Err)
 			continue
 		}
+		if strings.HasPrefix(result.Item.Name, batchJobReportsPrefix+slashSeparator) {
+			continue
+		}
 		// ignore batch-replicate.bin and batch-rotate.bin entries
 		if strings.HasSuffix(result.Item.Name, slashSeparator) {
 			continue
@@ -1988,7 +2143,7 @@ func (m *batchJobMetrics) purgeJobMetrics() {
 			var toDeleteJobMetrics []string
 			m.RLock()
 			for id, metrics := range m.metrics {
-				if time.Since(metrics.LastUpdate) > 24*time.Hour && (metrics.Complete || metrics.Failed) {
+				if time.Since(metrics.LastUpdate) > oldJobsExpiration && (metrics.Complete || metrics.Failed) {
 					toDeleteJobMetrics = append(toDeleteJobMetrics, id)
 				}
 			}

cmd/batch-replicate.go

@@ -21,8 +21,8 @@ import (
 	"time"
 
 	miniogo "github.com/minio/minio-go/v7"
-
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/pkg/v3/xtime"
 )
 
 //go:generate msgp -file $GOFILE
@@ -65,12 +65,12 @@ import (
 
 // BatchReplicateFilter holds all the filters currently supported for batch replication
 type BatchReplicateFilter struct {
-	NewerThan     time.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
-	OlderThan     time.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
-	CreatedAfter  time.Time     `yaml:"createdAfter,omitempty" json:"createdAfter"`
-	CreatedBefore time.Time     `yaml:"createdBefore,omitempty" json:"createdBefore"`
-	Tags          []BatchJobKV  `yaml:"tags,omitempty" json:"tags"`
-	Metadata      []BatchJobKV  `yaml:"metadata,omitempty" json:"metadata"`
+	NewerThan     xtime.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
+	OlderThan     xtime.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
+	CreatedAfter  time.Time      `yaml:"createdAfter,omitempty" json:"createdAfter"`
+	CreatedBefore time.Time      `yaml:"createdBefore,omitempty" json:"createdBefore"`
+	Tags          []BatchJobKV   `yaml:"tags,omitempty" json:"tags"`
+	Metadata      []BatchJobKV   `yaml:"metadata,omitempty" json:"metadata"`
 }
 
 // BatchJobReplicateFlags various configurations for replication job definition currently includes

cmd/batch-replicate_gen.go

@@ -1409,13 +1409,13 @@ func (z *BatchReplicateFilter) DecodeMsg(dc *msgp.Reader) (err error) {
 		}
 		switch msgp.UnsafeString(field) {
 		case "NewerThan":
-			z.NewerThan, err = dc.ReadDuration()
+			err = z.NewerThan.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "NewerThan")
 				return
 			}
 		case "OlderThan":
-			z.OlderThan, err = dc.ReadDuration()
+			err = z.OlderThan.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "OlderThan")
 				return
@@ -1489,7 +1489,7 @@ func (z *BatchReplicateFilter) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteDuration(z.NewerThan)
+	err = z.NewerThan.EncodeMsg(en)
 	if err != nil {
 		err = msgp.WrapError(err, "NewerThan")
 		return
@@ -1499,7 +1499,7 @@ func (z *BatchReplicateFilter) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteDuration(z.OlderThan)
+	err = z.OlderThan.EncodeMsg(en)
 	if err != nil {
 		err = msgp.WrapError(err, "OlderThan")
 		return
@@ -1567,10 +1567,18 @@ func (z *BatchReplicateFilter) MarshalMsg(b []byte) (o []byte, err error) {
 	// map header, size 6
 	// string "NewerThan"
 	o = append(o, 0x86, 0xa9, 0x4e, 0x65, 0x77, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
-	o = msgp.AppendDuration(o, z.NewerThan)
+	o, err = z.NewerThan.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "NewerThan")
+		return
+	}
 	// string "OlderThan"
 	o = append(o, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
-	o = msgp.AppendDuration(o, z.OlderThan)
+	o, err = z.OlderThan.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "OlderThan")
+		return
+	}
 	// string "CreatedAfter"
 	o = append(o, 0xac, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x66, 0x74, 0x65, 0x72)
 	o = msgp.AppendTime(o, z.CreatedAfter)
@@ -1619,13 +1627,13 @@ func (z *BatchReplicateFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		}
 		switch msgp.UnsafeString(field) {
 		case "NewerThan":
-			z.NewerThan, bts, err = msgp.ReadDurationBytes(bts)
+			bts, err = z.NewerThan.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "NewerThan")
 				return
 			}
 		case "OlderThan":
-			z.OlderThan, bts, err = msgp.ReadDurationBytes(bts)
+			bts, err = z.OlderThan.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "OlderThan")
 				return
@@ -1694,7 +1702,7 @@ func (z *BatchReplicateFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchReplicateFilter) Msgsize() (s int) {
-	s = 1 + 10 + msgp.DurationSize + 10 + msgp.DurationSize + 13 + msgp.TimeSize + 14 + msgp.TimeSize + 5 + msgp.ArrayHeaderSize
+	s = 1 + 10 + z.NewerThan.Msgsize() + 10 + z.OlderThan.Msgsize() + 13 + msgp.TimeSize + 14 + msgp.TimeSize + 5 + msgp.ArrayHeaderSize
 	for za0001 := range z.Tags {
 		s += z.Tags[za0001].Msgsize()
 	}

cmd/batch-replicate_test.go

@@ -0,0 +1,100 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+func TestParseBatchJobReplicate(t *testing.T) {
+	replicateYaml := `
+replicate:
+  apiVersion: v1
+  # source of the objects to be replicated
+  source:
+    type: minio # valid values are "s3" or "minio"
+    bucket: mytest
+    prefix: object-prefix1 # 'PREFIX' is optional
+    # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted
+    # Either the 'source' or 'remote' *must* be the "local" deployment
+#    endpoint: "http://127.0.0.1:9000"
+#    # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto"
+#    credentials:
+#      accessKey: minioadmin # Required
+#      secretKey: minioadmin # Required
+#    # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used
+    snowball: # automatically activated if the source is local
+      disable: true # optionally turn-off snowball archive transfer
+#      batch: 100 # upto this many objects per archive
+#      inmemory: true # indicates if the archive must be staged locally or in-memory
+#      compress: false # S2/Snappy compressed archive
+#      smallerThan: 5MiB # create archive for all objects smaller than 5MiB
+#      skipErrs: false # skips any source side read() errors
+
+  # target where the objects must be replicated
+  target:
+    type: minio # valid values are "s3" or "minio"
+    bucket: mytest
+    prefix: stage # 'PREFIX' is optional
+    # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted
+
+    # Either the 'source' or 'remote' *must* be the "local" deployment
+    endpoint: "http://127.0.0.1:9001"
+    # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto"
+    credentials:
+      accessKey: minioadmin
+      secretKey: minioadmin
+    # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used
+
+  # NOTE: All flags are optional
+  # - filtering criteria only applies for all source objects match the criteria
+  # - configurable notification endpoints
+  # - configurable retries for the job (each retry skips successfully previously replaced objects)
+  flags:
+    filter:
+      newerThan: "7d10h31s" # match objects newer than this value (e.g. 7d10h31s)
+      olderThan: "7d" # match objects older than this value (e.g. 7d10h31s)
+#      createdAfter: "date" # match objects created after "date"
+#      createdBefore: "date" # match objects created before "date"
+
+      ## NOTE: tags are not supported when "source" is remote.
+      tags:
+         - key: "name"
+           value: "pick*" # match objects with tag 'name', with all values starting with 'pick'
+
+      metadata:
+         - key: "content-type"
+           value: "image/*" # match objects with 'content-type', with all values starting with 'image/'
+
+#    notify:
+#      endpoint: "https://notify.endpoint" # notification endpoint to receive job status events
+#      token: "Bearer xxxxx" # optional authentication token for the notification endpoint
+#
+#    retry:
+#      attempts: 10 # number of retries for the job before giving up
+#      delay: "500ms" # least amount of delay between each retry
+
+`
+	var job BatchJobRequest
+	err := yaml.Unmarshal([]byte(replicateYaml), &job)
+	if err != nil {
+		t.Fatal("Failed to parse batch-job-replicate yaml", err)
+	}
+}

cmd/batch-rotate.go

@@ -257,7 +257,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		JobType:   string(job.Type()),
 		StartTime: job.Started,
 	}
-	if err := ri.load(ctx, api, job); err != nil {
+	if err := ri.loadOrInit(ctx, api, job); err != nil {
 		return err
 	}
 	if ri.Complete {
@@ -389,6 +389,17 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 					stopFn(result, err)
 					batchLogIf(ctx, err)
 					success = false
+					if attempts >= retryAttempts {
+						auditOptions := AuditLogOptions{
+							Event:     "KeyRotate",
+							APIName:   "StartBatchJob",
+							Bucket:    result.Bucket,
+							Object:    result.Name,
+							VersionID: result.VersionID,
+							Error:     err.Error(),
+						}
+						auditLogInternal(ctx, auditOptions)
+					}
 				} else {
 					stopFn(result, nil)
 				}

cmd/bucket-replication-handlers.go

@@ -75,7 +75,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
-	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, true)
+	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, &validateReplicationDestinationOptions{CheckRemoteBucket: true})
 	if apiErr != noError {
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
@@ -559,7 +559,7 @@ func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.Respon
 		lockEnabled = lcfg.Enabled()
 	}
 
-	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, true)
+	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, &validateReplicationDestinationOptions{CheckRemoteBucket: true})
 	if apiErr != noError {
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return

cmd/bucket-replication.go

@@ -91,7 +91,10 @@ func getReplicationConfig(ctx context.Context, bucketName string) (rc *replicati
 
 // validateReplicationDestination returns error if replication destination bucket missing or not configured
 // It also returns true if replication destination is same as this server.
-func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config, checkRemote bool) (bool, APIError) {
+func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config, opts *validateReplicationDestinationOptions) (bool, APIError) {
+	if opts == nil {
+		opts = &validateReplicationDestinationOptions{}
+	}
 	var arns []string
 	if rCfg.RoleArn != "" {
 		arns = append(arns, rCfg.RoleArn)
@@ -113,7 +116,7 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 		if clnt == nil {
 			return sameTarget, toAPIError(ctx, BucketRemoteTargetNotFound{Bucket: bucket})
 		}
-		if checkRemote { // validate remote bucket
+		if opts.CheckRemoteBucket { // validate remote bucket
 			found, err := clnt.BucketExists(ctx, arn.Bucket)
 			if err != nil {
 				return sameTarget, errorCodes.ToAPIErrWithErr(ErrRemoteDestinationNotFoundError, err)
@@ -133,23 +136,29 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 				}
 			}
 		}
+		// if checked bucket, then check the ready is unnecessary
+		if !opts.CheckRemoteBucket && opts.CheckReady {
+			endpoint := clnt.EndpointURL().String()
+			if errInt, ok := opts.checkReadyErr.Load(endpoint); !ok {
+				err = checkRemoteEndpoint(ctx, clnt.EndpointURL())
+				opts.checkReadyErr.Store(endpoint, err)
+			} else {
+				if errInt == nil {
+					err = nil
+				} else {
+					err = errInt.(error)
+				}
+			}
+			switch err.(type) {
+			case BucketRemoteIdenticalToSource:
+				return true, errorCodes.ToAPIErrWithErr(ErrBucketRemoteIdenticalToSource, fmt.Errorf("remote target endpoint %s is self referential", clnt.EndpointURL().String()))
+			default:
+			}
+		}
 		// validate replication ARN against target endpoint
-		c := globalBucketTargetSys.GetRemoteTargetClient(bucket, arnStr)
-		if c != nil {
-			if err := checkRemoteEndpoint(ctx, c.EndpointURL()); err != nil {
-				switch err.(type) {
-				case BucketRemoteIdenticalToSource:
-					return true, errorCodes.ToAPIErrWithErr(ErrBucketRemoteIdenticalToSource, fmt.Errorf("remote target endpoint %s is self referential", c.EndpointURL().String()))
-				default:
-				}
-			}
-			if c.EndpointURL().String() == clnt.EndpointURL().String() {
-				selfTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
-				if !sameTarget {
-					sameTarget = selfTarget
-				}
-				continue
-			}
+		selfTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
+		if !sameTarget {
+			sameTarget = selfTarget
 		}
 	}
 
@@ -765,21 +774,6 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
 	return "", false
 }
 
-func getCRCMeta(oi ObjectInfo, partNum int, h http.Header) map[string]string {
-	meta := make(map[string]string)
-	cs := oi.decryptChecksums(partNum, h)
-	for k, v := range cs {
-		cksum := hash.NewChecksumString(k, v)
-		if cksum == nil {
-			continue
-		}
-		if cksum.Valid() {
-			meta[cksum.Type.Key()] = v
-		}
-	}
-	return meta
-}
-
 func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, partNum int) (putOpts minio.PutObjectOptions, err error) {
 	meta := make(map[string]string)
 	isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined)
@@ -788,11 +782,6 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
 		// In case of SSE-C objects copy the allowed internal headers as well
 		if !isSSEC || !slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
 			if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
-				if strings.EqualFold(k, ReservedMetadataPrefixLower+"crc") {
-					for k, v := range getCRCMeta(objInfo, partNum, nil) {
-						meta[k] = v
-					}
-				}
 				continue
 			}
 			if isStandardHeader(k) {
@@ -811,8 +800,12 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
 		if isSSEC {
 			meta[ReplicationSsecChecksumHeader] = base64.StdEncoding.EncodeToString(objInfo.Checksum)
 		} else {
-			for k, v := range getCRCMeta(objInfo, 0, nil) {
-				meta[k] = v
+			for _, pi := range objInfo.Parts {
+				if pi.Number == partNum {
+					for k, v := range pi.Checksums {
+						meta[k] = v
+					}
+				}
 			}
 		}
 	}
@@ -1306,7 +1299,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 		HeaderSize: headerSize,
 	}
 	newCtx := ctx
-	if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) {
+	if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) && objInfo.Size < minLargeObjSize {
 		var cancel context.CancelFunc
 		newCtx, cancel = context.WithTimeout(ctx, throttleDeadline)
 		defer cancel()
@@ -1581,7 +1574,7 @@ applyAction:
 			HeaderSize: headerSize,
 		}
 		newCtx := ctx
-		if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) {
+		if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) && objInfo.Size < minLargeObjSize {
 			var cancel context.CancelFunc
 			newCtx, cancel = context.WithTimeout(ctx, throttleDeadline)
 			defer cancel()
@@ -1666,8 +1659,7 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 		cHeader := http.Header{}
 		cHeader.Add(xhttp.MinIOSourceReplicationRequest, "true")
 		if !isSSEC {
-			crc := getCRCMeta(objInfo, partInfo.Number, nil) // No SSE-C keys here.
-			for k, v := range crc {
+			for k, v := range partInfo.Checksums {
 				cHeader.Add(k, v)
 			}
 		}
@@ -1803,15 +1795,18 @@ var (
 type ReplicationPool struct {
 	// atomic ops:
 	activeWorkers    int32
+	activeLrgWorkers int32
 	activeMRFWorkers int32
 
-	objLayer   ObjectLayer
-	ctx        context.Context
-	priority   string
-	maxWorkers int
-	mu         sync.RWMutex
-	mrfMU      sync.Mutex
-	resyncer   *replicationResyncer
+	objLayer    ObjectLayer
+	ctx         context.Context
+	priority    string
+	maxWorkers  int
+	maxLWorkers int
+
+	mu       sync.RWMutex
+	mrfMU    sync.Mutex
+	resyncer *replicationResyncer
 
 	// workers:
 	workers    []chan ReplicationWorkerOperation
@@ -1882,9 +1877,13 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 	if maxWorkers > 0 && failedWorkers > maxWorkers {
 		failedWorkers = maxWorkers
 	}
+	maxLWorkers := LargeWorkerCount
+	if opts.MaxLWorkers > 0 {
+		maxLWorkers = opts.MaxLWorkers
+	}
 	pool := &ReplicationPool{
 		workers:         make([]chan ReplicationWorkerOperation, 0, workers),
-		lrgworkers:      make([]chan ReplicationWorkerOperation, 0, LargeWorkerCount),
+		lrgworkers:      make([]chan ReplicationWorkerOperation, 0, maxLWorkers),
 		mrfReplicaCh:    make(chan ReplicationWorkerOperation, 100000),
 		mrfWorkerKillCh: make(chan struct{}, failedWorkers),
 		resyncer:        newresyncer(),
@@ -1894,9 +1893,10 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 		objLayer:        o,
 		priority:        priority,
 		maxWorkers:      maxWorkers,
+		maxLWorkers:     maxLWorkers,
 	}
 
-	pool.AddLargeWorkers()
+	pool.ResizeLrgWorkers(maxLWorkers, 0)
 	pool.ResizeWorkers(workers, 0)
 	pool.ResizeFailedWorkers(failedWorkers)
 	go pool.resyncer.PersistToDisk(ctx, o)
@@ -1975,23 +1975,8 @@ func (p *ReplicationPool) AddWorker(input <-chan ReplicationWorkerOperation, opT
 	}
 }
 
-// AddLargeWorkers adds a static number of workers to handle large uploads
-func (p *ReplicationPool) AddLargeWorkers() {
-	for i := 0; i < LargeWorkerCount; i++ {
-		p.lrgworkers = append(p.lrgworkers, make(chan ReplicationWorkerOperation, 100000))
-		i := i
-		go p.AddLargeWorker(p.lrgworkers[i])
-	}
-	go func() {
-		<-p.ctx.Done()
-		for i := 0; i < LargeWorkerCount; i++ {
-			xioutil.SafeClose(p.lrgworkers[i])
-		}
-	}()
-}
-
 // AddLargeWorker adds a replication worker to the static pool for large uploads.
-func (p *ReplicationPool) AddLargeWorker(input <-chan ReplicationWorkerOperation) {
+func (p *ReplicationPool) AddLargeWorker(input <-chan ReplicationWorkerOperation, opTracker *int32) {
 	for {
 		select {
 		case <-p.ctx.Done():
@@ -2002,11 +1987,23 @@ func (p *ReplicationPool) AddLargeWorker(input <-chan ReplicationWorkerOperation
 			}
 			switch v := oi.(type) {
 			case ReplicateObjectInfo:
+				if opTracker != nil {
+					atomic.AddInt32(opTracker, 1)
+				}
 				globalReplicationStats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 				replicateObject(p.ctx, v, p.objLayer)
 				globalReplicationStats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
+				if opTracker != nil {
+					atomic.AddInt32(opTracker, -1)
+				}
 			case DeletedObjectReplicationInfo:
+				if opTracker != nil {
+					atomic.AddInt32(opTracker, 1)
+				}
 				replicateDelete(p.ctx, v, p.objLayer)
+				if opTracker != nil {
+					atomic.AddInt32(opTracker, -1)
+				}
 			default:
 				bugLogIf(p.ctx, fmt.Errorf("unknown replication type: %T", oi), "unknown-replicate-type")
 			}
@@ -2014,6 +2011,30 @@ func (p *ReplicationPool) AddLargeWorker(input <-chan ReplicationWorkerOperation
 	}
 }
 
+// ResizeLrgWorkers sets replication workers pool for large transfers(>=128MiB) to new size.
+// checkOld can be set to an expected value.
+// If the worker count changed
+func (p *ReplicationPool) ResizeLrgWorkers(n, checkOld int) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if (checkOld > 0 && len(p.lrgworkers) != checkOld) || n == len(p.lrgworkers) || n < 1 {
+		// Either already satisfied or worker count changed while we waited for the lock.
+		return
+	}
+	for len(p.lrgworkers) < n {
+		input := make(chan ReplicationWorkerOperation, 100000)
+		p.lrgworkers = append(p.lrgworkers, input)
+
+		go p.AddLargeWorker(input, &p.activeLrgWorkers)
+	}
+	for len(p.lrgworkers) > n {
+		worker := p.lrgworkers[len(p.lrgworkers)-1]
+		p.lrgworkers = p.lrgworkers[:len(p.lrgworkers)-1]
+		xioutil.SafeClose(worker)
+	}
+}
+
 // ActiveWorkers returns the number of active workers handling replication traffic.
 func (p *ReplicationPool) ActiveWorkers() int {
 	return int(atomic.LoadInt32(&p.activeWorkers))
@@ -2024,6 +2045,11 @@ func (p *ReplicationPool) ActiveMRFWorkers() int {
 	return int(atomic.LoadInt32(&p.activeMRFWorkers))
 }
 
+// ActiveLrgWorkers returns the number of active workers handling traffic > 128MiB object size.
+func (p *ReplicationPool) ActiveLrgWorkers() int {
+	return int(atomic.LoadInt32(&p.activeLrgWorkers))
+}
+
 // ResizeWorkers sets replication workers pool to new size.
 // checkOld can be set to an expected value.
 // If the worker count changed
@@ -2049,7 +2075,7 @@ func (p *ReplicationPool) ResizeWorkers(n, checkOld int) {
 }
 
 // ResizeWorkerPriority sets replication failed workers pool size
-func (p *ReplicationPool) ResizeWorkerPriority(pri string, maxWorkers int) {
+func (p *ReplicationPool) ResizeWorkerPriority(pri string, maxWorkers, maxLWorkers int) {
 	var workers, mrfWorkers int
 	p.mu.Lock()
 	switch pri {
@@ -2076,11 +2102,15 @@ func (p *ReplicationPool) ResizeWorkerPriority(pri string, maxWorkers int) {
 	if maxWorkers > 0 && mrfWorkers > maxWorkers {
 		mrfWorkers = maxWorkers
 	}
+	if maxLWorkers <= 0 {
+		maxLWorkers = LargeWorkerCount
+	}
 	p.priority = pri
 	p.maxWorkers = maxWorkers
 	p.mu.Unlock()
 	p.ResizeWorkers(workers, 0)
 	p.ResizeFailedWorkers(mrfWorkers)
+	p.ResizeLrgWorkers(maxLWorkers, 0)
 }
 
 // ResizeFailedWorkers sets replication failed workers pool size
@@ -2124,9 +2154,18 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		h := xxh3.HashString(ri.Bucket + ri.Name)
 		select {
 		case <-p.ctx.Done():
-		case p.lrgworkers[h%LargeWorkerCount] <- ri:
+		case p.lrgworkers[h%uint64(len(p.lrgworkers))] <- ri:
 		default:
 			globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
+			p.mu.RLock()
+			maxLWorkers := p.maxLWorkers
+			existing := len(p.lrgworkers)
+			p.mu.RUnlock()
+			maxLWorkers = min(maxLWorkers, LargeWorkerCount)
+			if p.ActiveLrgWorkers() < maxLWorkers {
+				workers := min(existing+1, maxLWorkers)
+				p.ResizeLrgWorkers(workers, existing)
+			}
 		}
 		return
 	}
@@ -2229,8 +2268,9 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 }
 
 type replicationPoolOpts struct {
-	Priority   string
-	MaxWorkers int
+	Priority    string
+	MaxWorkers  int
+	MaxLWorkers int
 }
 
 func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
@@ -3505,7 +3545,7 @@ func (p *ReplicationPool) persistToDrive(ctx context.Context, v MRFReplicateEntr
 	}
 
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 
 	for _, localDrive := range localDrives {
@@ -3572,7 +3612,7 @@ func (p *ReplicationPool) loadMRF() (mrfRec MRFReplicateEntries, err error) {
 	}
 
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 
 	for _, localDrive := range localDrives {
@@ -3693,3 +3733,12 @@ func (p *ReplicationPool) getMRF(ctx context.Context, bucket string) (ch <-chan
 
 	return mrfCh, nil
 }
+
+// validateReplicationDestinationOptions is used to configure the validation of the replication destination.
+// validateReplicationDestination uses this to configure the validation.
+type validateReplicationDestinationOptions struct {
+	CheckRemoteBucket bool
+	CheckReady        bool
+
+	checkReadyErr sync.Map
+}

cmd/common-main.go

@@ -685,16 +685,6 @@ func loadEnvVarsFromFiles() {
 		}
 	}
 
-	if env.IsSet(kms.EnvKMSSecretKeyFile) {
-		kmsSecret, err := readFromSecret(env.Get(kms.EnvKMSSecretKeyFile, ""))
-		if err != nil {
-			logger.Fatal(err, "Unable to read the KMS secret key inherited from secret file")
-		}
-		if kmsSecret != "" {
-			os.Setenv(kms.EnvKMSSecretKey, kmsSecret)
-		}
-	}
-
 	if env.IsSet(config.EnvConfigEnvFile) {
 		ekvs, err := minioEnvironFromFile(env.Get(config.EnvConfigEnvFile, ""))
 		if err != nil && !os.IsNotExist(err) {
@@ -843,6 +833,7 @@ func loadRootCredentials() {
 	// Check both cases and authenticate them if correctly defined
 	var user, password string
 	var hasCredentials bool
+	var legacyCredentials bool
 	//nolint:gocritic
 	if env.IsSet(config.EnvRootUser) && env.IsSet(config.EnvRootPassword) {
 		user = env.Get(config.EnvRootUser, "")
@@ -851,6 +842,7 @@ func loadRootCredentials() {
 	} else if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
 		user = env.Get(config.EnvAccessKey, "")
 		password = env.Get(config.EnvSecretKey, "")
+		legacyCredentials = true
 		hasCredentials = true
 	} else if globalServerCtxt.RootUser != "" && globalServerCtxt.RootPwd != "" {
 		user, password = globalServerCtxt.RootUser, globalServerCtxt.RootPwd
@@ -859,8 +851,13 @@ func loadRootCredentials() {
 	if hasCredentials {
 		cred, err := auth.CreateCredentials(user, password)
 		if err != nil {
-			logger.Fatal(config.ErrInvalidCredentials(err),
-				"Unable to validate credentials inherited from the shell environment")
+			if legacyCredentials {
+				logger.Fatal(config.ErrInvalidCredentials(err),
+					"Unable to validate credentials inherited from the shell environment")
+			} else {
+				logger.Fatal(config.ErrInvalidRootUserCredentials(err),
+					"Unable to validate credentials inherited from the shell environment")
+			}
 		}
 		if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
 			msg := fmt.Sprintf("WARNING: %s and %s are deprecated.\n"+
@@ -874,6 +871,12 @@ func loadRootCredentials() {
 	} else {
 		globalActiveCred = auth.DefaultCredentials
 	}
+
+	var err error
+	globalNodeAuthToken, err = authenticateNode(globalActiveCred.AccessKey, globalActiveCred.SecretKey)
+	if err != nil {
+		logger.Fatal(err, "Unable to generate internode credentials")
+	}
 }
 
 // Initialize KMS global variable after valiadating and loading the configuration.

cmd/config-current.go

@@ -101,7 +101,7 @@ func initHelp() {
 		config.HelpKV{
 			Key:         config.SubnetSubSys,
 			Type:        "string",
-			Description: "register the cluster to MinIO SUBNET",
+			Description: "register Enterprise license for the cluster",
 			Optional:    true,
 		},
 		config.HelpKV{

cmd/data-scanner.go

@@ -1275,7 +1275,7 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return false
 		}
-		ilmLogIf(ctx, err)
+		ilmLogIf(ctx, fmt.Errorf("expireTransitionedObject(%s, %s): %w", obj.Bucket, obj.Name, err))
 		return false
 	}
 	timeILM(1)
@@ -1328,7 +1328,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 			return false
 		}
 		// Assume it is still there.
-		ilmLogOnceIf(ctx, err, "non-transition-expiry")
+		ilmLogOnceIf(ctx, fmt.Errorf("DeleteObject(%s, %s): %w", obj.Bucket, obj.Name, err), "non-transition-expiry"+obj.Name)
 		return false
 	}
 	if dobj.Name == "" {

cmd/dummy-data-generator_test.go

@@ -61,10 +61,9 @@ func NewDummyDataGen(totalLength, skipOffset int64) io.ReadSeeker {
 	}
 
 	skipOffset %= int64(len(alphabets))
-	as := make([]byte, 2*len(alphabets))
-	copy(as, alphabets)
-	copy(as[len(alphabets):], alphabets)
-	b := as[skipOffset : skipOffset+int64(len(alphabets))]
+	const multiply = 100
+	as := bytes.Repeat(alphabets, multiply)
+	b := as[skipOffset : skipOffset+int64(len(alphabets)*(multiply-1))]
 	return &DummyDataGen{
 		length: totalLength,
 		b:      b,

cmd/encryption-v1.go

@@ -1172,6 +1172,14 @@ func (o *ObjectInfo) decryptChecksums(part int, h http.Header) map[string]string
 	if len(data) == 0 {
 		return nil
 	}
+	if part > 0 && !crypto.SSEC.IsEncrypted(o.UserDefined) {
+		// already decrypted in ToObjectInfo for multipart objects
+		for _, pi := range o.Parts {
+			if pi.Number == part {
+				return pi.Checksums
+			}
+		}
+	}
 	if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted {
 		decrypted, err := o.metadataDecrypter(h)("object-checksum", data)
 		if err != nil {

cmd/erasure-healing-common_test.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 	"time"
 
@@ -147,6 +148,10 @@ func TestCommonTime(t *testing.T) {
 // TestListOnlineDisks - checks if listOnlineDisks and outDatedDisks
 // are consistent with each other.
 func TestListOnlineDisks(t *testing.T) {
+	if runtime.GOOS == globalWindowsOSName {
+		t.Skip()
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -230,7 +235,7 @@ func TestListOnlineDisks(t *testing.T) {
 	}
 
 	object := "object"
-	data := bytes.Repeat([]byte("a"), smallFileThreshold*16)
+	data := bytes.Repeat([]byte("a"), smallFileThreshold*32)
 	z := obj.(*erasureServerPools)
 
 	erasureDisks, err := z.GetDisks(0, 0)

cmd/erasure-healing.go

@@ -629,7 +629,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		}
 
 		for i, v := range result.Before.Drives {
-			if v.Endpoint == disk.String() {
+			if v.Endpoint == disk.Endpoint().String() {
 				result.After.Drives[i].State = madmin.DriveStateOk
 			}
 		}

cmd/erasure-metadata.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/amztime"
+	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash/sha256"
@@ -173,6 +174,7 @@ func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInf
 		}
 	}
 	objInfo.Checksum = fi.Checksum
+	objInfo.decryptPartsChecksums(nil)
 	objInfo.Inlined = fi.InlineData()
 	// Success.
 	return objInfo
@@ -456,6 +458,7 @@ func commonParity(parities []int, defaultParityCount int) int {
 }
 
 func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int) {
+	totalShards := len(partsMetadata)
 	parities = make([]int, len(partsMetadata))
 	for index, metadata := range partsMetadata {
 		if errs[index] != nil {
@@ -466,9 +469,13 @@ func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int)
 			parities[index] = -1
 			continue
 		}
+		//nolint:gocritic
 		// Delete marker or zero byte objects take highest parity.
 		if metadata.Deleted || metadata.Size == 0 {
-			parities[index] = len(partsMetadata) / 2
+			parities[index] = totalShards / 2
+		} else if metadata.TransitionStatus == lifecycle.TransitionComplete {
+			// For tiered objects, read quorum is N/2+1 to ensure simple majority on xl.meta. It is not equal to EcM because the data integrity is entrusted with the warm tier.
+			parities[index] = totalShards - (totalShards/2 + 1)
 		} else {
 			parities[index] = metadata.Erasure.ParityBlocks
 		}

cmd/erasure-metadata_test.go

@@ -19,6 +19,8 @@ package cmd
 
 import (
 	"context"
+	"fmt"
+	"slices"
 	"strconv"
 	"testing"
 	"time"
@@ -359,3 +361,124 @@ func TestSkipTierFreeVersion(t *testing.T) {
 		t.Fatal("Expected SkipTierFreeVersion to be set on FileInfo but wasn't")
 	}
 }
+
+func TestListObjectParities(t *testing.T) {
+	mkMetaArr := func(N, parity, agree int) []FileInfo {
+		fi := newFileInfo("obj-1", N-parity, parity)
+		fi.TransitionTier = "WARM-TIER"
+		fi.TransitionedObjName = mustGetUUID()
+		fi.TransitionStatus = "complete"
+		fi.Size = 1 << 20
+
+		metaArr := make([]FileInfo, N)
+		for i := range N {
+			fi.Erasure.Index = i + 1
+			metaArr[i] = fi
+			if i < agree {
+				continue
+			}
+			metaArr[i].TransitionTier, metaArr[i].TransitionedObjName = "", ""
+			metaArr[i].TransitionStatus = ""
+		}
+		return metaArr
+	}
+	mkParities := func(N, agreedParity, disagreedParity, agree int) []int {
+		ps := make([]int, N)
+		for i := range N {
+			if i < agree {
+				ps[i] = agreedParity
+				continue
+			}
+			ps[i] = disagreedParity // disagree
+		}
+		return ps
+	}
+
+	mkTest := func(N, parity, agree int) (res struct {
+		metaArr  []FileInfo
+		errs     []error
+		parities []int
+		parity   int
+	},
+	) {
+		res.metaArr = mkMetaArr(N, parity, agree)
+		res.parities = mkParities(N, N-(N/2+1), parity, agree)
+		res.errs = make([]error, N)
+		if agree >= N/2+1 { // simple majority consensus
+			res.parity = N - (N/2 + 1)
+		} else {
+			res.parity = -1
+		}
+		return res
+	}
+
+	nonTieredTest := func(N, parity, agree int) (res struct {
+		metaArr  []FileInfo
+		errs     []error
+		parities []int
+		parity   int
+	},
+	) {
+		fi := newFileInfo("obj-1", N-parity, parity)
+		fi.Size = 1 << 20
+		metaArr := make([]FileInfo, N)
+		parities := make([]int, N)
+		for i := range N {
+			fi.Erasure.Index = i + 1
+			metaArr[i] = fi
+			parities[i] = parity
+			if i < agree {
+				continue
+			}
+			metaArr[i].Erasure.Index = 0 // creates invalid fi on remaining drives
+			parities[i] = -1             // invalid fi are assigned parity -1
+		}
+		res.metaArr = metaArr
+		res.parities = parities
+		res.errs = make([]error, N)
+		if agree >= N-parity {
+			res.parity = parity
+		} else {
+			res.parity = -1
+		}
+
+		return res
+	}
+	tests := []struct {
+		metaArr  []FileInfo
+		errs     []error
+		parities []int
+		parity   int
+	}{
+		// More than simple majority consensus
+		mkTest(15, 3, 11),
+		// No simple majority consensus
+		mkTest(15, 3, 7),
+		// Exact simple majority consensus
+		mkTest(15, 3, 8),
+		// More than simple majority consensus
+		mkTest(16, 4, 11),
+		// No simple majority consensus
+		mkTest(16, 4, 8),
+		// Exact simple majority consensus
+		mkTest(16, 4, 9),
+		// non-tiered object require read quorum of EcM
+		nonTieredTest(15, 3, 12),
+		// non-tiered object with fewer than EcM in consensus
+		nonTieredTest(15, 3, 11),
+		// non-tiered object require read quorum of EcM
+		nonTieredTest(16, 4, 12),
+		// non-tiered object with fewer than EcM in consensus
+		nonTieredTest(16, 4, 11),
+	}
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
+			if got := listObjectParities(test.metaArr, test.errs); !slices.Equal(got, test.parities) {
+				t.Fatalf("Expected parities %v but got %v", test.parities, got)
+			}
+			if got := commonParity(test.parities, len(test.metaArr)/2); got != test.parity {
+				t.Fatalf("Expected common parity %v but got %v", test.parity, got)
+			}
+		})
+	}
+}

cmd/erasure-multipart.go

@@ -103,10 +103,9 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 	return fi, partsMetadata, err
 }
 
-// Removes part.meta given by partName belonging to a multipart upload from minioMetaBucket
-func (er erasureObjects) removePartMeta(bucket, object, uploadID, dataDir string, partNumber int) {
-	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
-	curpartPath := pathJoin(uploadIDPath, dataDir, fmt.Sprintf("part.%d", partNumber))
+// cleanMultipartPath removes all extraneous files and parts from the multipart folder, this is used per CompleteMultipart.
+// do not use this function outside of completeMultipartUpload()
+func (er erasureObjects) cleanupMultipartPath(ctx context.Context, paths ...string) {
 	storageDisks := er.getDisks()
 
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -116,42 +115,7 @@ func (er erasureObjects) removePartMeta(bucket, object, uploadID, dataDir string
 		}
 		index := index
 		g.Go(func() error {
-			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath+".meta", DeleteOptions{
-				Recursive: false,
-				Immediate: false,
-			})
-
-			return nil
-		}, index)
-	}
-	g.Wait()
-}
-
-// Removes part given by partName belonging to a multipart upload from minioMetaBucket
-func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir string, partNumber int) {
-	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
-	curpartPath := pathJoin(uploadIDPath, dataDir, fmt.Sprintf("part.%d", partNumber))
-	storageDisks := er.getDisks()
-
-	g := errgroup.WithNErrs(len(storageDisks))
-	for index, disk := range storageDisks {
-		if disk == nil {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			// Ignoring failure to remove parts that weren't present in CompleteMultipartUpload
-			// requests. xl.meta is the authoritative source of truth on which parts constitute
-			// the object. The presence of parts that don't belong in the object doesn't affect correctness.
-			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath, DeleteOptions{
-				Recursive: false,
-				Immediate: false,
-			})
-			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath+".meta", DeleteOptions{
-				Recursive: false,
-				Immediate: false,
-			})
-
+			_ = storageDisks[index].DeleteBulk(ctx, minioMetaMultipartBucket, paths...)
 			return nil
 		}, index)
 	}
@@ -159,7 +123,7 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
 }
 
 // Clean-up the old multipart uploads. Should be run in a Go routine.
-func (er erasureObjects) cleanupStaleUploads(ctx context.Context, expiry time.Duration) {
+func (er erasureObjects) cleanupStaleUploads(ctx context.Context) {
 	// run multiple cleanup's local to this server.
 	var wg sync.WaitGroup
 	for _, disk := range er.getLocalDisks() {
@@ -167,7 +131,7 @@ func (er erasureObjects) cleanupStaleUploads(ctx context.Context, expiry time.Du
 			wg.Add(1)
 			go func(disk StorageAPI) {
 				defer wg.Done()
-				er.cleanupStaleUploadsOnDisk(ctx, disk, expiry)
+				er.cleanupStaleUploadsOnDisk(ctx, disk)
 			}(disk)
 		}
 	}
@@ -193,7 +157,7 @@ func (er erasureObjects) deleteAll(ctx context.Context, bucket, prefix string) {
 }
 
 // Remove the old multipart uploads on the given disk.
-func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk StorageAPI, expiry time.Duration) {
+func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk StorageAPI) {
 	drivePath := disk.Endpoint().Path
 
 	readDirFn(pathJoin(drivePath, minioMetaMultipartBucket), func(shaDir string, typ os.FileMode) error {
@@ -219,7 +183,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 				modTime = fi.ModTime
 				wait()
 			}
-			if time.Since(modTime) < expiry {
+			if time.Since(modTime) < globalAPIConfig.getStaleUploadsExpiry() {
 				return nil
 			}
 			w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
@@ -238,7 +202,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 			return nil
 		}
 		// Modtime is returned in the Created field. See (*xlStorage).StatVol
-		if time.Since(vi.Created) < expiry {
+		if time.Since(vi.Created) < globalAPIConfig.getStaleUploadsExpiry() {
 			return nil
 		}
 		w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
@@ -267,7 +231,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 		return w.Run(func() error {
 			wait := deleteMultipartCleanupSleeper.Timer(ctx)
-			if time.Since(vi.Created) > expiry {
+			if time.Since(vi.Created) > globalAPIConfig.getStaleUploadsExpiry() {
 				pathUUID := mustGetUUID()
 				targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID)
 
@@ -1359,10 +1323,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 
+	paths := make([]string, 0, len(currentFI.Parts))
 	// Remove parts that weren't present in CompleteMultipartUpload request.
 	for _, curpart := range currentFI.Parts {
-		// Remove part.meta which is not needed anymore.
-		er.removePartMeta(bucket, object, uploadID, currentFI.DataDir, curpart.Number)
+		paths = append(paths, pathJoin(uploadIDPath, currentFI.DataDir, fmt.Sprintf("part.%d.meta", curpart.Number)))
 
 		if objectPartIndex(fi.Parts, curpart.Number) == -1 {
 			// Delete the missing part files. e.g,
@@ -1371,10 +1335,12 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			// Request 3: PutObjectPart 2
 			// Request 4: CompleteMultipartUpload --part 2
 			// N.B. 1st part is not present. This part should be removed from the storage.
-			er.removeObjectPart(bucket, object, uploadID, currentFI.DataDir, curpart.Number)
+			paths = append(paths, pathJoin(uploadIDPath, currentFI.DataDir, fmt.Sprintf("part.%d", curpart.Number)))
 		}
 	}
 
+	er.cleanupMultipartPath(ctx, paths...) // cleanup all part.N.meta, and skipped part.N's before final rename().
+
 	defer func() {
 		if err == nil {
 			er.deleteAll(context.Background(), minioMetaMultipartBucket, uploadIDPath)

cmd/erasure-object.go

@@ -283,7 +283,7 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 	}
 
 	if unlockOnDefer {
-		unlockOnDefer = fi.InlineData()
+		unlockOnDefer = fi.InlineData() || len(fi.Data) > 0
 	}
 
 	pr, pw := xioutil.WaitPipe()
@@ -908,6 +908,8 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		}
 
 		rw.Lock()
+		// when its a versioned bucket and empty versionID - at totalResp == setDriveCount
+		// we must use rawFileInfo to resolve versions to figure out the latest version.
 		if opts.VersionID == "" && totalResp == er.setDriveCount {
 			fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(pickLatestQuorumFilesInfo(ctx,
 				rawArr, errs, bucket, object, readData, opts.InclFreeVersions, true))
@@ -915,7 +917,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 			fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(metaArr, errs)
 		}
 		rw.Unlock()
-		if err == nil && fi.InlineData() {
+		if err == nil && (fi.InlineData() || len(fi.Data) > 0) {
 			break
 		}
 	}
@@ -1399,7 +1401,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	writers := make([]io.Writer, len(onlineDisks))
 	var inlineBuffers []*bytes.Buffer
 	if shardFileSize >= 0 {
-		if !opts.Versioned && shardFileSize < inlineBlock {
+		if !opts.Versioned && shardFileSize <= inlineBlock {
 			inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
 		} else if shardFileSize < inlineBlock/8 {
 			inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
@@ -1407,7 +1409,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	} else {
 		// If compressed, use actual size to determine.
 		if sz := erasure.ShardFileSize(data.ActualSize()); sz > 0 {
-			if !opts.Versioned && sz < inlineBlock {
+			if !opts.Versioned && sz <= inlineBlock {
 				inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
 			} else if sz < inlineBlock/8 {
 				inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))

cmd/erasure-server-pool-decom.go

@@ -358,7 +358,7 @@ func (p *poolMeta) validate(pools []*erasureSets) (bool, error) {
 			update = true
 		}
 		if ok && pi.completed {
-			return false, fmt.Errorf("pool(%s) = %s is decommissioned, please remove from server command line", humanize.Ordinal(pi.position+1), k)
+			logger.LogIf(GlobalContext, "decommission", fmt.Errorf("pool(%s) = %s is decommissioned, please remove from server command line", humanize.Ordinal(pi.position+1), k))
 		}
 	}
 

cmd/erasure-server-pool-decom_test.go

@@ -134,7 +134,7 @@ func TestPoolMetaValidate(t *testing.T) {
 			meta:           nmeta1,
 			pools:          pools,
 			name:           "Invalid-Completed-Pool-Not-Removed",
-			expectedErr:    true,
+			expectedErr:    false,
 			expectedUpdate: false,
 		},
 		{

cmd/erasure-server-pool.go

@@ -168,7 +168,7 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 
 	if !globalIsDistErasure {
 		globalLocalDrivesMu.Lock()
-		globalLocalDrives = localDrives
+		globalLocalDrivesMap = make(map[string]StorageAPI, len(localDrives))
 		for _, drive := range localDrives {
 			globalLocalDrivesMap[drive.Endpoint().String()] = drive
 		}
@@ -1526,14 +1526,12 @@ func (z *erasureServerPools) listObjectsGeneric(ctx context.Context, bucket, pre
 			loi.NextMarker = last.Name
 		}
 
-		if merged.lastSkippedEntry != "" {
-			if merged.lastSkippedEntry > loi.NextMarker {
-				// An object hidden by ILM was found during listing. Since the number of entries
-				// fetched from drives is limited, set IsTruncated to true to ask the s3 client
-				// to continue listing if it wishes in order to find if there is more objects.
-				loi.IsTruncated = true
-				loi.NextMarker = merged.lastSkippedEntry
-			}
+		if loi.IsTruncated && merged.lastSkippedEntry > loi.NextMarker {
+			// An object hidden by ILM was found during a truncated listing. Since the number of entries
+			// fetched from drives is limited by max-keys, we should use the last ILM filtered entry
+			// as a continuation token if it is lexially higher than the last visible object so that the
+			// next call of WalkDir() with the max-keys can reach new objects not seen previously.
+			loi.NextMarker = merged.lastSkippedEntry
 		}
 
 		if loi.NextMarker != "" {
@@ -2343,12 +2341,18 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 
 	var poolErrs [][]error
 	for idx, erasureSet := range z.serverPools {
+		if opts.Pool != nil && *opts.Pool != idx {
+			continue
+		}
 		if z.IsSuspended(idx) {
 			continue
 		}
 		errs := make([]error, len(erasureSet.sets))
 		var wg sync.WaitGroup
 		for idx, set := range erasureSet.sets {
+			if opts.Set != nil && *opts.Set != idx {
+				continue
+			}
 			wg.Add(1)
 			go func(idx int, set *erasureObjects) {
 				defer wg.Done()
@@ -2441,6 +2445,7 @@ const (
 type HealthOptions struct {
 	Maintenance    bool
 	DeploymentType string
+	NoLogging      bool
 }
 
 // HealthResult returns the current state of the system, also
@@ -2477,7 +2482,7 @@ func (hr HealthResult) String() string {
 		if i == 0 {
 			str.WriteString(")")
 		} else {
-			str.WriteString("), ")
+			str.WriteString(") | ")
 		}
 	}
 	return str.String()
@@ -2600,7 +2605,7 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 			})
 
 			healthy := erasureSetUpCount[poolIdx][setIdx].online >= poolWriteQuorums[poolIdx]
-			if !healthy {
+			if !healthy && !opts.NoLogging {
 				storageLogIf(logger.SetReqInfo(ctx, reqInfo),
 					fmt.Errorf("Write quorum could not be established on pool: %d, set: %d, expected write quorum: %d, drives-online: %d",
 						poolIdx, setIdx, poolWriteQuorums[poolIdx], erasureSetUpCount[poolIdx][setIdx].online), logger.FatalKind)
@@ -2608,7 +2613,7 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 			result.Healthy = result.Healthy && healthy
 
 			healthyRead := erasureSetUpCount[poolIdx][setIdx].online >= poolReadQuorums[poolIdx]
-			if !healthyRead {
+			if !healthyRead && !opts.NoLogging {
 				storageLogIf(logger.SetReqInfo(ctx, reqInfo),
 					fmt.Errorf("Read quorum could not be established on pool: %d, set: %d, expected read quorum: %d, drives-online: %d",
 						poolIdx, setIdx, poolReadQuorums[poolIdx], erasureSetUpCount[poolIdx][setIdx].online))

cmd/erasure-sets.go

@@ -120,13 +120,6 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 
 	format, err := loadFormatErasure(disk, false)
 	if err != nil {
-		if errors.Is(err, errUnformattedDisk) {
-			info, derr := disk.DiskInfo(context.TODO(), DiskInfoOptions{})
-			if derr != nil && info.RootDisk {
-				disk.Close()
-				return nil, nil, fmt.Errorf("Drive: %s is a root drive", disk)
-			}
-		}
 		disk.Close()
 		return nil, nil, fmt.Errorf("Drive: %s returned %w", disk, err) // make sure to '%w' to wrap the error
 	}
@@ -196,7 +189,7 @@ func (s *erasureSets) Legacy() (ok bool) {
 
 // connectDisks - attempt to connect all the endpoints, loads format
 // and re-arranges the disks in proper position.
-func (s *erasureSets) connectDisks() {
+func (s *erasureSets) connectDisks(log bool) {
 	defer func() {
 		s.lastConnectDisksOpTime = time.Now()
 	}()
@@ -230,8 +223,10 @@ func (s *erasureSets) connectDisks() {
 			if err != nil {
 				if endpoint.IsLocal && errors.Is(err, errUnformattedDisk) {
 					globalBackgroundHealState.pushHealLocalDisks(endpoint)
-				} else {
-					printEndpointError(endpoint, err, true)
+				} else if !errors.Is(err, errDriveIsRoot) {
+					if log {
+						printEndpointError(endpoint, err, true)
+					}
 				}
 				return
 			}
@@ -267,13 +262,7 @@ func (s *erasureSets) connectDisks() {
 				if globalIsDistErasure {
 					globalLocalSetDrives[s.poolIndex][setIndex][diskIndex] = disk
 				}
-				for i, ldisk := range globalLocalDrives {
-					_, k, l := ldisk.GetDiskLoc()
-					if k == setIndex && l == diskIndex {
-						globalLocalDrives[i] = disk
-						break
-					}
-				}
+				globalLocalDrivesMap[disk.Endpoint().String()] = disk
 				globalLocalDrivesMu.Unlock()
 			}
 			s.erasureDisksMu.Unlock()
@@ -292,7 +281,7 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 	time.Sleep(time.Duration(r.Float64() * float64(time.Second)))
 
 	// Pre-emptively connect the disks if possible.
-	s.connectDisks()
+	s.connectDisks(false)
 
 	monitor := time.NewTimer(monitorInterval)
 	defer monitor.Stop()
@@ -306,7 +295,7 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 				console.Debugln("running drive monitoring")
 			}
 
-			s.connectDisks()
+			s.connectDisks(true)
 
 			// Reset the timer for next interval
 			monitor.Reset(monitorInterval)
@@ -536,7 +525,7 @@ func (s *erasureSets) cleanupStaleUploads(ctx context.Context) {
 					if set == nil {
 						return
 					}
-					set.cleanupStaleUploads(ctx, globalAPIConfig.getStaleUploadsExpiry())
+					set.cleanupStaleUploads(ctx)
 				}(set)
 			}
 			wg.Wait()
@@ -1140,13 +1129,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 					if globalIsDistErasure {
 						globalLocalSetDrives[s.poolIndex][m][n] = disk
 					}
-					for i, ldisk := range globalLocalDrives {
-						_, k, l := ldisk.GetDiskLoc()
-						if k == m && l == n {
-							globalLocalDrives[i] = disk
-							break
-						}
-					}
+					globalLocalDrivesMap[disk.Endpoint().String()] = disk
 					globalLocalDrivesMu.Unlock()
 				}
 			}

cmd/erasure.go

@@ -102,6 +102,8 @@ func diskErrToDriveState(err error) (state string) {
 		state = madmin.DriveStatePermission
 	case errors.Is(err, errFaultyDisk):
 		state = madmin.DriveStateFaulty
+	case errors.Is(err, errDriveIsRoot):
+		state = madmin.DriveStateRootMount
 	case err == nil:
 		state = madmin.DriveStateOk
 	default:

cmd/fmt-gen.go

@@ -0,0 +1,120 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build fmtgen
+
+package cmd
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/klauspost/compress/zip"
+	"github.com/minio/cli"
+)
+
+var fmtGenFlags = []cli.Flag{
+	cli.IntFlag{
+		Name:  "parity",
+		Usage: "specify erasure code parity",
+	},
+	cli.StringFlag{
+		Name:  "deployment-id",
+		Usage: "deployment-id of the MinIO cluster for which format.json is needed",
+	},
+	cli.StringFlag{
+		Name:   "address",
+		Value:  ":" + GlobalMinioDefaultPort,
+		Usage:  "bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname",
+		EnvVar: "MINIO_ADDRESS",
+	},
+}
+
+var fmtGenCmd = cli.Command{
+	Name:   "fmt-gen",
+	Usage:  "Generate format.json files for an erasure server pool",
+	Flags:  append(fmtGenFlags, GlobalFlags...),
+	Action: fmtGenMain,
+	CustomHelpTemplate: `NAME:
+  {{.HelpName}} - {{.Usage}}
+
+USAGE:
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR1 [DIR2..]
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR{1...64}
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR{1...64} DIR{65...128}
+
+DIR:
+  DIR points to a directory on a filesystem. When you want to combine
+  multiple drives into a single large system, pass one directory per
+  filesystem separated by space. You may also use a '...' convention
+  to abbreviate the directory arguments. Remote directories in a
+  distributed setup are encoded as HTTP(s) URIs.
+{{if .VisibleFlags}}
+FLAGS:
+  {{range .VisibleFlags}}{{.}}
+  {{end}}{{end}}
+EXAMPLES:
+  1. Generate format.json.zip containing format.json files for all drives in a distributed MinIO server pool of 32 nodes with 32 drives each.
+     {{.Prompt}} {{.HelpName}} http://node{1...32}.example.com/mnt/export{1...32}
+
+`,
+}
+
+func fmtGenMain(ctxt *cli.Context) {
+	deploymentID := ctxt.String("deployment-id")
+	err := buildServerCtxt(ctxt, &globalServerCtxt)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	handleCommonArgs(globalServerCtxt)
+	pools, _, err := createServerEndpoints(globalMinioAddr, globalServerCtxt.Layout.pools, globalServerCtxt.Layout.legacy)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	zipFile, err := os.Create("format.json.zip")
+	if err != nil {
+		log.Fatalf("failed to create format.json.zip: %v", err)
+	}
+	defer zipFile.Close()
+	fmtZipW := zip.NewWriter(zipFile)
+	defer fmtZipW.Close()
+	for _, pool := range pools { // for each pool
+		setCount, setDriveCount := pool.SetCount, pool.DrivesPerSet
+		format := newFormatErasureV3(setCount, setDriveCount)
+		format.ID = deploymentID
+		for i := 0; i < setCount; i++ { // for each erasure set
+			for j := 0; j < setDriveCount; j++ {
+				newFormat := format.Clone()
+				newFormat.Erasure.This = format.Erasure.Sets[i][j]
+				if deploymentID != "" {
+					newFormat.ID = deploymentID
+				}
+				drive := pool.Endpoints[i*setDriveCount+j]
+				fmtBytes, err := json.Marshal(newFormat)
+				if err != nil {
+					//nolint:gocritic
+					log.Fatalf("failed to marshal format.json for %s: %v", drive.String(), err)
+				}
+				fmtJSON := filepath.Join(drive.Host, drive.Path, minioMetaBucket, "format.json")
+				embedFileInZip(fmtZipW, fmtJSON, fmtBytes, 0o600)
+			}
+		}
+	}
+}

cmd/fmt-gen_notsupported.go

@@ -1,6 +1,6 @@
-// Copyright (c) 2015-2022 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
-// This file is part of MinIO Object Storage stack
+// # This file is part of MinIO Object Storage stack
 //
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
@@ -15,9 +15,10 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-//go:build go1.19
-// +build go1.19
+//go:build !fmtgen
 
-package fips
+package cmd
 
-const go19 = true
+import "github.com/minio/cli"
+
+var fmtGenCmd cli.Command

cmd/generic-handlers.go

@@ -247,11 +247,14 @@ func guessIsRPCReq(req *http.Request) bool {
 	if req == nil {
 		return false
 	}
-	if req.Method == http.MethodGet && req.URL != nil && req.URL.Path == grid.RoutePath {
-		return true
+	if req.Method == http.MethodGet && req.URL != nil {
+		switch req.URL.Path {
+		case grid.RoutePath, grid.RouteLockPath:
+			return true
+		}
 	}
 
-	return req.Method == http.MethodPost &&
+	return (req.Method == http.MethodPost || req.Method == http.MethodGet) &&
 		strings.HasPrefix(req.URL.Path, minioReservedBucketPath+SlashSeparator)
 }
 
@@ -469,7 +472,7 @@ func setBucketForwardingMiddleware(h http.Handler) http.Handler {
 		}
 		if globalDNSConfig == nil || !globalBucketFederation ||
 			guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
-			guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) {
+			guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) || isKMSReq(r) {
 			h.ServeHTTP(w, r)
 			return
 		}

cmd/generic-handlers_test.go

@@ -51,9 +51,10 @@ func TestGuessIsRPC(t *testing.T) {
 	r = &http.Request{
 		Proto:  "HTTP/1.1",
 		Method: http.MethodGet,
+		URL:    u,
 	}
-	if guessIsRPCReq(r) {
-		t.Fatal("Test shouldn't report as net/rpc for a non net/rpc request.")
+	if !guessIsRPCReq(r) {
+		t.Fatal("Test shouldn't fail for a possible net/rpc request.")
 	}
 	r = &http.Request{
 		Proto:  "HTTP/1.1",
@@ -63,6 +64,14 @@ func TestGuessIsRPC(t *testing.T) {
 	if !guessIsRPCReq(r) {
 		t.Fatal("Grid RPC path not detected")
 	}
+	r = &http.Request{
+		Proto:  "HTTP/1.1",
+		Method: http.MethodGet,
+		URL:    &url.URL{Path: grid.RouteLockPath},
+	}
+	if !guessIsRPCReq(r) {
+		t.Fatal("Grid RPC path not detected")
+	}
 }
 
 var isHTTPHeaderSizeTooLargeTests = []struct {

cmd/global-heal.go

@@ -150,6 +150,11 @@ type healEntryResult struct {
 
 // healErasureSet lists and heals all objects in a specific erasure set
 func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string, tracker *healingTracker) error {
+	bgSeq, found := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
+	if !found {
+		return errors.New("no local healing sequence initialized, unable to heal the drive")
+	}
+
 	scanMode := madmin.HealNormalScan
 
 	// Make sure to copy since `buckets slice`
@@ -163,11 +168,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 	}
 
 	for _, bucket := range healBuckets {
-		_, err := objAPI.HealBucket(ctx, bucket, madmin.HealOpts{
-			Recreate: true,
-			ScanMode: scanMode,
-		})
-		if err != nil {
+		if err := bgSeq.healBucket(objAPI, bucket, true); err != nil {
 			// Log bucket healing error if any, we shall retry again.
 			healingLogIf(ctx, err)
 		}
@@ -264,10 +265,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		tracker.setBucket(bucket)
 		// Heal current bucket again in case if it is failed
 		// in the beginning of erasure set healing
-		if _, err := objAPI.HealBucket(ctx, bucket, madmin.HealOpts{
-			Recreate: true,
-			ScanMode: scanMode,
-		}); err != nil {
+		if err := bgSeq.healBucket(objAPI, bucket, true); err != nil {
 			// Set this such that when we return this function
 			// we let the caller retry this disk again for the
 			// buckets that failed healing.
@@ -366,6 +364,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				}
 				return false
 			case results <- result:
+				bgSeq.countScanned(madmin.HealItemObject)
 				return true
 			}
 		}
@@ -416,8 +415,10 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 						return
 					}
 					result = healEntryFailure(0)
+					bgSeq.countFailed(madmin.HealItemObject)
 					healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, entry.name, err))
 				} else {
+					bgSeq.countHealed(madmin.HealItemObject)
 					result = healEntrySuccess(uint64(res.ObjectSize))
 				}
 
@@ -441,6 +442,8 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					continue
 				}
 
+				var versionHealed bool
+
 				res, err := er.HealObject(ctx, bucket, encodedEntryName,
 					version.VersionID, madmin.HealOpts{
 						ScanMode: scanMode,
@@ -453,15 +456,24 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 						versionNotFound++
 						continue
 					}
-					// If not deleted, assume they failed.
+				} else {
+					// Look for the healing results
+					if res.After.Drives[tracker.DiskIndex].State == madmin.DriveStateOk {
+						versionHealed = true
+					}
+				}
+
+				if versionHealed {
+					bgSeq.countHealed(madmin.HealItemObject)
+					result = healEntrySuccess(uint64(version.Size))
+				} else {
+					bgSeq.countFailed(madmin.HealItemObject)
 					result = healEntryFailure(uint64(version.Size))
 					if version.VersionID != "" {
 						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
 					} else {
 						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, version.Name, err))
 					}
-				} else {
-					result = healEntrySuccess(uint64(res.ObjectSize))
 				}
 
 				if !send(result) {
@@ -509,7 +521,11 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				jt.Take()
 				go healEntry(bucket, *entry)
 			},
-			finished: nil,
+			finished: func(errs []error) {
+				if countErrs(errs, nil) != len(errs) {
+					retErr = fmt.Errorf("one or more errors reported during listing: %v", errors.Join(errs...))
+				}
+			},
 		})
 		jt.Wait() // synchronize all the concurrent heal jobs
 		if err != nil {
@@ -517,7 +533,10 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			// we let the caller retry this disk again for the
 			// buckets it failed to list.
 			retErr = err
-			healingLogIf(ctx, fmt.Errorf("listing failed with: %v on bucket: %v", err, bucket))
+		}
+
+		if retErr != nil {
+			healingLogIf(ctx, fmt.Errorf("listing failed with: %v on bucket: %v", retErr, bucket))
 			continue
 		}
 

cmd/globals.go

@@ -310,6 +310,7 @@ var (
 	globalBootTime = UTCNow()
 
 	globalActiveCred         auth.Credentials
+	globalNodeAuthToken      string
 	globalSiteReplicatorCred siteReplicatorCred
 
 	// Captures if root credentials are set via ENV.
@@ -413,10 +414,9 @@ var (
 	globalServiceFreezeCnt int32
 	globalServiceFreezeMu  sync.Mutex // Updates.
 
-	// List of local drives to this node, this is only set during server startup,
-	// and is only mutated by HealFormat. Hold globalLocalDrivesMu to access.
-	globalLocalDrives    []StorageAPI
-	globalLocalDrivesMap = make(map[string]StorageAPI)
+	// Map of local drives to this node, this is set during server startup,
+	// disk reconnect and mutated by HealFormat. Hold globalLocalDrivesMu to access.
+	globalLocalDrivesMap map[string]StorageAPI
 	globalLocalDrivesMu  sync.RWMutex
 
 	globalDriveMonitoring = env.Get("_MINIO_DRIVE_ACTIVE_MONITORING", config.EnableOn) == config.EnableOn

cmd/grid.go

@@ -31,9 +31,15 @@ import (
 // globalGrid is the global grid manager.
 var globalGrid atomic.Pointer[grid.Manager]
 
+// globalLockGrid is the global lock grid manager.
+var globalLockGrid atomic.Pointer[grid.Manager]
+
 // globalGridStart is a channel that will block startup of grid connections until closed.
 var globalGridStart = make(chan struct{})
 
+// globalLockGridStart is a channel that will block startup of lock grid connections until closed.
+var globalLockGridStart = make(chan struct{})
+
 func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error {
 	hosts, local := eps.GridHosts()
 	lookupHost := globalDNSCache.LookupHost
@@ -41,21 +47,24 @@ func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error {
 		// Pass Dialer for websocket grid, make sure we do not
 		// provide any DriveOPTimeout() function, as that is not
 		// useful over persistent connections.
-		Dialer:       grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions.ForWebsocket()))),
+		Dialer: grid.ConnectWS(
+			grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions.ForWebsocket()))),
+			newCachedAuthToken(),
+			&tls.Config{
+				RootCAs:          globalRootCAs,
+				CipherSuites:     fips.TLSCiphers(),
+				CurvePreferences: fips.TLSCurveIDs(),
+			}),
 		Local:        local,
 		Hosts:        hosts,
-		AddAuth:      newCachedAuthToken(),
-		AuthRequest:  storageServerRequestValidate,
+		AuthToken:    validateStorageRequestToken,
+		AuthFn:       newCachedAuthToken(),
 		BlockConnect: globalGridStart,
-		TLSConfig: &tls.Config{
-			RootCAs:          globalRootCAs,
-			CipherSuites:     fips.TLSCiphers(),
-			CurvePreferences: fips.TLSCurveIDs(),
-		},
 		// Record incoming and outgoing bytes.
-		Incoming: globalConnStats.incInternodeInputBytes,
-		Outgoing: globalConnStats.incInternodeOutputBytes,
-		TraceTo:  globalTrace,
+		Incoming:  globalConnStats.incInternodeInputBytes,
+		Outgoing:  globalConnStats.incInternodeOutputBytes,
+		TraceTo:   globalTrace,
+		RoutePath: grid.RoutePath,
 	})
 	if err != nil {
 		return err
@@ -63,3 +72,36 @@ func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error {
 	globalGrid.Store(g)
 	return nil
 }
+
+func initGlobalLockGrid(ctx context.Context, eps EndpointServerPools) error {
+	hosts, local := eps.GridHosts()
+	lookupHost := globalDNSCache.LookupHost
+	g, err := grid.NewManager(ctx, grid.ManagerOptions{
+		// Pass Dialer for websocket grid, make sure we do not
+		// provide any DriveOPTimeout() function, as that is not
+		// useful over persistent connections.
+		Dialer: grid.ConnectWSWithRoutePath(
+			grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions.ForWebsocket()))),
+			newCachedAuthToken(),
+			&tls.Config{
+				RootCAs:          globalRootCAs,
+				CipherSuites:     fips.TLSCiphers(),
+				CurvePreferences: fips.TLSCurveIDs(),
+			}, grid.RouteLockPath),
+		Local:        local,
+		Hosts:        hosts,
+		AuthToken:    validateStorageRequestToken,
+		AuthFn:       newCachedAuthToken(),
+		BlockConnect: globalGridStart,
+		// Record incoming and outgoing bytes.
+		Incoming:  globalConnStats.incInternodeInputBytes,
+		Outgoing:  globalConnStats.incInternodeOutputBytes,
+		TraceTo:   globalTrace,
+		RoutePath: grid.RouteLockPath,
+	})
+	if err != nil {
+		return err
+	}
+	globalLockGrid.Store(g)
+	return nil
+}

cmd/handler-api.go

@@ -39,14 +39,15 @@ import (
 type apiConfig struct {
 	mu sync.RWMutex
 
-	requestsDeadline      time.Duration
-	requestsPool          chan struct{}
-	clusterDeadline       time.Duration
-	listQuorum            string
-	corsAllowOrigins      []string
-	replicationPriority   string
-	replicationMaxWorkers int
-	transitionWorkers     int
+	requestsDeadline       time.Duration
+	requestsPool           chan struct{}
+	clusterDeadline        time.Duration
+	listQuorum             string
+	corsAllowOrigins       []string
+	replicationPriority    string
+	replicationMaxWorkers  int
+	replicationMaxLWorkers int
+	transitionWorkers      int
 
 	staleUploadsExpiry          time.Duration
 	staleUploadsCleanupInterval time.Duration
@@ -170,11 +171,12 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) {
 	}
 	t.listQuorum = listQuorum
 	if globalReplicationPool != nil &&
-		(cfg.ReplicationPriority != t.replicationPriority || cfg.ReplicationMaxWorkers != t.replicationMaxWorkers) {
-		globalReplicationPool.ResizeWorkerPriority(cfg.ReplicationPriority, cfg.ReplicationMaxWorkers)
+		(cfg.ReplicationPriority != t.replicationPriority || cfg.ReplicationMaxWorkers != t.replicationMaxWorkers || cfg.ReplicationMaxLWorkers != t.replicationMaxLWorkers) {
+		globalReplicationPool.ResizeWorkerPriority(cfg.ReplicationPriority, cfg.ReplicationMaxWorkers, cfg.ReplicationMaxLWorkers)
 	}
 	t.replicationPriority = cfg.ReplicationPriority
 	t.replicationMaxWorkers = cfg.ReplicationMaxWorkers
+	t.replicationMaxLWorkers = cfg.ReplicationMaxLWorkers
 
 	// N B api.transition_workers will be deprecated
 	if globalTransitionState != nil {
@@ -381,14 +383,16 @@ func (t *apiConfig) getReplicationOpts() replicationPoolOpts {
 
 	if t.replicationPriority == "" {
 		return replicationPoolOpts{
-			Priority:   "auto",
-			MaxWorkers: WorkerMaxLimit,
+			Priority:    "auto",
+			MaxWorkers:  WorkerMaxLimit,
+			MaxLWorkers: LargeWorkerCount,
 		}
 	}
 
 	return replicationPoolOpts{
-		Priority:   t.replicationPriority,
-		MaxWorkers: t.replicationMaxWorkers,
+		Priority:    t.replicationPriority,
+		MaxWorkers:  t.replicationMaxWorkers,
+		MaxLWorkers: t.replicationMaxLWorkers,
 	}
 }
 

cmd/http-stats.go

@@ -27,10 +27,6 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-const (
-	apiGetObject = "GetObject"
-)
-
 // connStats - Network statistics
 // Count total input/output transferred bytes during
 // the server's life.
@@ -132,7 +128,7 @@ func (bh *bucketHTTPStats) updateHTTPStats(bucket, api string, w *xhttp.Response
 		return
 	}
 
-	if w != nil && api == apiGetObject {
+	if w != nil {
 		// Increment the prometheus http request response histogram with API, Bucket
 		bucketHTTPRequestsDuration.With(prometheus.Labels{
 			"api":    api,
@@ -437,9 +433,7 @@ func (st *HTTPStats) updateStats(api string, w *xhttp.ResponseRecorder) {
 	st.totalS3Requests.Inc(api)
 
 	// Increment the prometheus http request response histogram with appropriate label
-	if api == apiGetObject {
-		httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(w.TimeToFirstByte.Seconds())
-	}
+	httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(w.TimeToFirstByte.Seconds())
 
 	code := w.StatusCode
 

cmd/iam-object-store.go

@@ -495,7 +495,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 		for _, item := range regUsersList {
 			userName := path.Dir(item)
 			if err := iamOS.loadUser(ctx, userName, regUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
-				return fmt.Errorf("unable to load the user `%s`: %w", userName, err)
+				return fmt.Errorf("unable to load the user: %w", err)
 			}
 		}
 		if took := time.Since(regUsersLoadStartTime); took > maxIAMLoadOpTime {
@@ -510,7 +510,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 		for _, item := range groupsList {
 			group := path.Dir(item)
 			if err := iamOS.loadGroup(ctx, group, cache.iamGroupsMap); err != nil && err != errNoSuchGroup {
-				return fmt.Errorf("unable to load the group `%s`: %w", group, err)
+				return fmt.Errorf("unable to load the group: %w", err)
 			}
 		}
 		if took := time.Since(groupsLoadStartTime); took > maxIAMLoadOpTime {
@@ -524,7 +524,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	for _, item := range userPolicyMappingsList {
 		userName := strings.TrimSuffix(item, ".json")
 		if err := iamOS.loadMappedPolicy(ctx, userName, regUser, false, cache.iamUserPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return fmt.Errorf("unable to load the policy mapping for the user `%s`: %w", userName, err)
+			return fmt.Errorf("unable to load the policy mapping for the user: %w", err)
 		}
 	}
 	if took := time.Since(userPolicyMappingLoadStartTime); took > maxIAMLoadOpTime {
@@ -537,7 +537,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	for _, item := range groupPolicyMappingsList {
 		groupName := strings.TrimSuffix(item, ".json")
 		if err := iamOS.loadMappedPolicy(ctx, groupName, regUser, true, cache.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return fmt.Errorf("unable to load the policy mapping for the group `%s`: %w", groupName, err)
+			return fmt.Errorf("unable to load the policy mapping for the group: %w", err)
 		}
 	}
 	if took := time.Since(groupPolicyMappingLoadStartTime); took > maxIAMLoadOpTime {
@@ -551,7 +551,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	for _, item := range svcAccList {
 		userName := path.Dir(item)
 		if err := iamOS.loadUser(ctx, userName, svcUser, svcUsersMap); err != nil && err != errNoSuchUser {
-			return fmt.Errorf("unable to load the service account `%s`: %w", userName, err)
+			return fmt.Errorf("unable to load the service account: %w", err)
 		}
 	}
 	if took := time.Since(svcAccLoadStartTime); took > maxIAMLoadOpTime {
@@ -580,7 +580,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 			// OIDC/AssumeRoleWithCustomToken/AssumeRoleWithCertificate).
 			err := iamOS.loadMappedPolicy(ctx, svcParent, stsUser, false, cache.iamSTSPolicyMap)
 			if err != nil && !errors.Is(err, errNoSuchPolicy) {
-				return fmt.Errorf("unable to load the policy mapping for the STS user `%s`: %w", svcParent, err)
+				return fmt.Errorf("unable to load the policy mapping for the STS user: %w", err)
 			}
 		}
 	}

cmd/iam-store.go

@@ -2088,50 +2088,72 @@ func (store *IAMStoreSys) GetAllSTSUserMappings(userPredicate func(string) bool)
 	return stsMap, nil
 }
 
-// Assumes store is locked by caller. If users is empty, returns all user mappings.
-func (store *IAMStoreSys) listUserPolicyMappings(cache *iamCache, users []string,
-	userPredicate func(string) bool,
+// Assumes store is locked by caller. If userMap is empty, returns all user mappings.
+func (store *IAMStoreSys) listUserPolicyMappings(cache *iamCache, userMap map[string]set.StringSet,
+	userPredicate func(string) bool, decodeFunc func(string) string,
 ) []madmin.UserPolicyEntities {
+	stsMap := xsync.NewMapOf[string, MappedPolicy]()
+	resMap := make(map[string]madmin.UserPolicyEntities, len(userMap))
+
+	for user, groupSet := range userMap {
+		// Attempt to load parent user mapping for STS accounts
+		store.loadMappedPolicy(context.TODO(), user, stsUser, false, stsMap)
+		decodeUser := user
+		if decodeFunc != nil {
+			decodeUser = decodeFunc(user)
+		}
+		blankEntities := madmin.UserPolicyEntities{User: decodeUser}
+		if !groupSet.IsEmpty() {
+			blankEntities.MemberOfMappings = store.listGroupPolicyMappings(cache, groupSet, nil, decodeFunc)
+		}
+		resMap[user] = blankEntities
+	}
+
 	var r []madmin.UserPolicyEntities
-	usersSet := set.CreateStringSet(users...)
 	cache.iamUserPolicyMap.Range(func(user string, mappedPolicy MappedPolicy) bool {
 		if userPredicate != nil && !userPredicate(user) {
 			return true
 		}
 
-		if !usersSet.IsEmpty() && !usersSet.Contains(user) {
-			return true
+		entitiesWithMemberOf, ok := resMap[user]
+		if !ok {
+			if len(userMap) > 0 {
+				return true
+			}
+			decodeUser := user
+			if decodeFunc != nil {
+				decodeUser = decodeFunc(user)
+			}
+			entitiesWithMemberOf = madmin.UserPolicyEntities{User: decodeUser}
 		}
 
 		ps := mappedPolicy.toSlice()
 		sort.Strings(ps)
-		r = append(r, madmin.UserPolicyEntities{
-			User:     user,
-			Policies: ps,
-		})
+		entitiesWithMemberOf.Policies = ps
+		resMap[user] = entitiesWithMemberOf
 		return true
 	})
 
-	stsMap := xsync.NewMapOf[string, MappedPolicy]()
-	for _, user := range users {
-		// Attempt to load parent user mapping for STS accounts
-		store.loadMappedPolicy(context.TODO(), user, stsUser, false, stsMap)
-	}
-
 	stsMap.Range(func(user string, mappedPolicy MappedPolicy) bool {
 		if userPredicate != nil && !userPredicate(user) {
 			return true
 		}
 
+		entitiesWithMemberOf := resMap[user]
+
 		ps := mappedPolicy.toSlice()
 		sort.Strings(ps)
-		r = append(r, madmin.UserPolicyEntities{
-			User:     user,
-			Policies: ps,
-		})
+		entitiesWithMemberOf.Policies = ps
+		resMap[user] = entitiesWithMemberOf
 		return true
 	})
 
+	for _, v := range resMap {
+		if v.Policies != nil || v.MemberOfMappings != nil {
+			r = append(r, v)
+		}
+	}
+
 	sort.Slice(r, func(i, j int) bool {
 		return r[i].User < r[j].User
 	})
@@ -2140,11 +2162,11 @@ func (store *IAMStoreSys) listUserPolicyMappings(cache *iamCache, users []string
 }
 
 // Assumes store is locked by caller. If groups is empty, returns all group mappings.
-func (store *IAMStoreSys) listGroupPolicyMappings(cache *iamCache, groups []string,
-	groupPredicate func(string) bool,
+func (store *IAMStoreSys) listGroupPolicyMappings(cache *iamCache, groupsSet set.StringSet,
+	groupPredicate func(string) bool, decodeFunc func(string) string,
 ) []madmin.GroupPolicyEntities {
 	var r []madmin.GroupPolicyEntities
-	groupsSet := set.CreateStringSet(groups...)
+
 	cache.iamGroupPolicyMap.Range(func(group string, mappedPolicy MappedPolicy) bool {
 		if groupPredicate != nil && !groupPredicate(group) {
 			return true
@@ -2154,10 +2176,15 @@ func (store *IAMStoreSys) listGroupPolicyMappings(cache *iamCache, groups []stri
 			return true
 		}
 
+		decodeGroup := group
+		if decodeFunc != nil {
+			decodeGroup = decodeFunc(group)
+		}
+
 		ps := mappedPolicy.toSlice()
 		sort.Strings(ps)
 		r = append(r, madmin.GroupPolicyEntities{
-			Group:    group,
+			Group:    decodeGroup,
 			Policies: ps,
 		})
 		return true
@@ -2171,17 +2198,20 @@ func (store *IAMStoreSys) listGroupPolicyMappings(cache *iamCache, groups []stri
 }
 
 // Assumes store is locked by caller. If policies is empty, returns all policy mappings.
-func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
-	userPredicate, groupPredicate func(string) bool,
+func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, queryPolSet set.StringSet,
+	userPredicate, groupPredicate func(string) bool, decodeFunc func(string) string,
 ) []madmin.PolicyEntities {
-	queryPolSet := set.CreateStringSet(policies...)
-
 	policyToUsersMap := make(map[string]set.StringSet)
 	cache.iamUserPolicyMap.Range(func(user string, mappedPolicy MappedPolicy) bool {
 		if userPredicate != nil && !userPredicate(user) {
 			return true
 		}
 
+		decodeUser := user
+		if decodeFunc != nil {
+			decodeUser = decodeFunc(user)
+		}
+
 		commonPolicySet := mappedPolicy.policySet()
 		if !queryPolSet.IsEmpty() {
 			commonPolicySet = commonPolicySet.Intersection(queryPolSet)
@@ -2189,9 +2219,9 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 		for _, policy := range commonPolicySet.ToSlice() {
 			s, ok := policyToUsersMap[policy]
 			if !ok {
-				policyToUsersMap[policy] = set.CreateStringSet(user)
+				policyToUsersMap[policy] = set.CreateStringSet(decodeUser)
 			} else {
-				s.Add(user)
+				s.Add(decodeUser)
 				policyToUsersMap[policy] = s
 			}
 		}
@@ -2205,6 +2235,11 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 				continue
 			}
 
+			decodeUser := user
+			if decodeFunc != nil {
+				decodeUser = decodeFunc(user)
+			}
+
 			var mappedPolicy MappedPolicy
 			store.loadIAMConfig(context.Background(), &mappedPolicy, getMappedPolicyPath(user, stsUser, false))
 
@@ -2215,9 +2250,9 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 			for _, policy := range commonPolicySet.ToSlice() {
 				s, ok := policyToUsersMap[policy]
 				if !ok {
-					policyToUsersMap[policy] = set.CreateStringSet(user)
+					policyToUsersMap[policy] = set.CreateStringSet(decodeUser)
 				} else {
-					s.Add(user)
+					s.Add(decodeUser)
 					policyToUsersMap[policy] = s
 				}
 			}
@@ -2232,6 +2267,11 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 					return true
 				}
 
+				decodeUser := user
+				if decodeFunc != nil {
+					decodeUser = decodeFunc(user)
+				}
+
 				commonPolicySet := mappedPolicy.policySet()
 				if !queryPolSet.IsEmpty() {
 					commonPolicySet = commonPolicySet.Intersection(queryPolSet)
@@ -2239,9 +2279,9 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 				for _, policy := range commonPolicySet.ToSlice() {
 					s, ok := policyToUsersMap[policy]
 					if !ok {
-						policyToUsersMap[policy] = set.CreateStringSet(user)
+						policyToUsersMap[policy] = set.CreateStringSet(decodeUser)
 					} else {
-						s.Add(user)
+						s.Add(decodeUser)
 						policyToUsersMap[policy] = s
 					}
 				}
@@ -2256,6 +2296,11 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 			return true
 		}
 
+		decodeGroup := group
+		if decodeFunc != nil {
+			decodeGroup = decodeFunc(group)
+		}
+
 		commonPolicySet := mappedPolicy.policySet()
 		if !queryPolSet.IsEmpty() {
 			commonPolicySet = commonPolicySet.Intersection(queryPolSet)
@@ -2263,9 +2308,9 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 		for _, policy := range commonPolicySet.ToSlice() {
 			s, ok := policyToGroupsMap[policy]
 			if !ok {
-				policyToGroupsMap[policy] = set.CreateStringSet(group)
+				policyToGroupsMap[policy] = set.CreateStringSet(decodeGroup)
 			} else {
-				s.Add(group)
+				s.Add(decodeGroup)
 				policyToGroupsMap[policy] = s
 			}
 		}
@@ -2305,24 +2350,24 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 }
 
 // ListPolicyMappings - return users/groups mapped to policies.
-func (store *IAMStoreSys) ListPolicyMappings(q madmin.PolicyEntitiesQuery,
-	userPredicate, groupPredicate func(string) bool,
+func (store *IAMStoreSys) ListPolicyMappings(q cleanEntitiesQuery,
+	userPredicate, groupPredicate func(string) bool, decodeFunc func(string) string,
 ) madmin.PolicyEntitiesResult {
 	cache := store.rlock()
 	defer store.runlock()
 
 	var result madmin.PolicyEntitiesResult
 
-	isAllPoliciesQuery := len(q.Users) == 0 && len(q.Groups) == 0 && len(q.Policy) == 0
+	isAllPoliciesQuery := len(q.Users) == 0 && len(q.Groups) == 0 && len(q.Policies) == 0
 
 	if len(q.Users) > 0 {
-		result.UserMappings = store.listUserPolicyMappings(cache, q.Users, userPredicate)
+		result.UserMappings = store.listUserPolicyMappings(cache, q.Users, userPredicate, decodeFunc)
 	}
 	if len(q.Groups) > 0 {
-		result.GroupMappings = store.listGroupPolicyMappings(cache, q.Groups, groupPredicate)
+		result.GroupMappings = store.listGroupPolicyMappings(cache, q.Groups, groupPredicate, decodeFunc)
 	}
-	if len(q.Policy) > 0 || isAllPoliciesQuery {
-		result.PolicyMappings = store.listPolicyMappings(cache, q.Policy, userPredicate, groupPredicate)
+	if len(q.Policies) > 0 || isAllPoliciesQuery {
+		result.PolicyMappings = store.listPolicyMappings(cache, q.Policies, userPredicate, groupPredicate, decodeFunc)
 	}
 	return result
 }

cmd/iam.go

@@ -217,6 +217,9 @@ func (sys *IAMSys) Load(ctx context.Context, firstTime bool) error {
 
 	if firstTime {
 		bootstrapTraceMsg(fmt.Sprintf("globalIAMSys.Load(): (duration: %s)", loadDuration))
+		if globalIsDistErasure {
+			logger.Info("IAM load(startup) finished. (duration: %s)", loadDuration)
+		}
 	}
 
 	select {
@@ -315,6 +318,24 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 		break
 	}
 
+	cache := sys.store.lock()
+	setDefaultCannedPolicies(cache.iamPolicyDocsMap)
+	sys.store.unlock()
+
+	// Load RoleARNs
+	sys.rolesMap = make(map[arn.ARN]string)
+
+	// From OpenID
+	if riMap := sys.OpenIDConfig.GetRoleInfo(); riMap != nil {
+		sys.validateAndAddRolePolicyMappings(ctx, riMap)
+	}
+
+	// From AuthN plugin if enabled.
+	if authn := newGlobalAuthNPluginFn(); authn != nil {
+		riMap := authn.GetRoleInfo()
+		sys.validateAndAddRolePolicyMappings(ctx, riMap)
+	}
+
 	// Load IAM data from storage.
 	for {
 		if err := sys.Load(retryCtx, true); err != nil {
@@ -334,20 +355,6 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 
 	go sys.periodicRoutines(ctx, refreshInterval)
 
-	// Load RoleARNs
-	sys.rolesMap = make(map[arn.ARN]string)
-
-	// From OpenID
-	if riMap := sys.OpenIDConfig.GetRoleInfo(); riMap != nil {
-		sys.validateAndAddRolePolicyMappings(ctx, riMap)
-	}
-
-	// From AuthN plugin if enabled.
-	if authn := newGlobalAuthNPluginFn(); authn != nil {
-		riMap := authn.GetRoleInfo()
-		sys.validateAndAddRolePolicyMappings(ctx, riMap)
-	}
-
 	sys.printIAMRoles()
 
 	bootstrapTraceMsg("finishing IAM loading")
@@ -396,12 +403,12 @@ func (sys *IAMSys) periodicRoutines(ctx context.Context, baseInterval time.Durat
 			// Load all IAM items (except STS creds) periodically.
 			refreshStart := time.Now()
 			if err := sys.Load(ctx, false); err != nil {
-				iamLogIf(ctx, fmt.Errorf("Failure in periodic refresh for IAM (took %.2fs): %v", time.Since(refreshStart).Seconds(), err), logger.WarningKind)
+				iamLogIf(ctx, fmt.Errorf("Failure in periodic refresh for IAM (duration: %s): %v", time.Since(refreshStart), err), logger.WarningKind)
 			} else {
 				took := time.Since(refreshStart).Seconds()
 				if took > maxDurationSecondsForLog {
 					// Log if we took a lot of time to load.
-					logger.Info("IAM refresh took %.2fs", took)
+					logger.Info("IAM refresh took (duration: %s)", took)
 				}
 			}
 
@@ -803,6 +810,57 @@ func (sys *IAMSys) ListLDAPUsers(ctx context.Context) (map[string]madmin.UserInf
 	}
 }
 
+type cleanEntitiesQuery struct {
+	Users    map[string]set.StringSet
+	Groups   set.StringSet
+	Policies set.StringSet
+}
+
+// createCleanEntitiesQuery - maps users to their groups and normalizes user or group DNs if ldap.
+func (sys *IAMSys) createCleanEntitiesQuery(q madmin.PolicyEntitiesQuery, ldap bool) cleanEntitiesQuery {
+	cleanQ := cleanEntitiesQuery{
+		Users:    make(map[string]set.StringSet),
+		Groups:   set.CreateStringSet(q.Groups...),
+		Policies: set.CreateStringSet(q.Policy...),
+	}
+
+	if ldap {
+		// Validate and normalize users, then fetch and normalize their groups
+		// Also include unvalidated users for backward compatibility.
+		for _, user := range q.Users {
+			lookupRes, actualGroups, _ := sys.LDAPConfig.GetValidatedDNWithGroups(user)
+			if lookupRes != nil {
+				groupSet := set.CreateStringSet(actualGroups...)
+
+				// duplicates can be overwritten, fetched groups should be identical.
+				cleanQ.Users[lookupRes.NormDN] = groupSet
+			}
+			// Search for non-normalized DN as well for backward compatibility.
+			if _, ok := cleanQ.Users[user]; !ok {
+				cleanQ.Users[user] = nil
+			}
+		}
+
+		// Validate and normalize groups.
+		for _, group := range q.Groups {
+			lookupRes, underDN, _ := sys.LDAPConfig.GetValidatedGroupDN(nil, group)
+			if lookupRes != nil && underDN {
+				cleanQ.Groups.Add(lookupRes.NormDN)
+			}
+		}
+	} else {
+		for _, user := range q.Users {
+			info, err := sys.store.GetUserInfo(user)
+			var groupSet set.StringSet
+			if err == nil {
+				groupSet = set.CreateStringSet(info.MemberOf...)
+			}
+			cleanQ.Users[user] = groupSet
+		}
+	}
+	return cleanQ
+}
+
 // QueryLDAPPolicyEntities - queries policy associations for LDAP users/groups/policies.
 func (sys *IAMSys) QueryLDAPPolicyEntities(ctx context.Context, q madmin.PolicyEntitiesQuery) (*madmin.PolicyEntitiesResult, error) {
 	if !sys.Initialized() {
@@ -815,7 +873,8 @@ func (sys *IAMSys) QueryLDAPPolicyEntities(ctx context.Context, q madmin.PolicyE
 
 	select {
 	case <-sys.configLoaded:
-		pe := sys.store.ListPolicyMappings(q, sys.LDAPConfig.IsLDAPUserDN, sys.LDAPConfig.IsLDAPGroupDN)
+		cleanQuery := sys.createCleanEntitiesQuery(q, true)
+		pe := sys.store.ListPolicyMappings(cleanQuery, sys.LDAPConfig.IsLDAPUserDN, sys.LDAPConfig.IsLDAPGroupDN, sys.LDAPConfig.DecodeDN)
 		pe.Timestamp = UTCNow()
 		return &pe, nil
 	case <-ctx.Done():
@@ -889,6 +948,7 @@ func (sys *IAMSys) QueryPolicyEntities(ctx context.Context, q madmin.PolicyEntit
 
 	select {
 	case <-sys.configLoaded:
+		cleanQuery := sys.createCleanEntitiesQuery(q, false)
 		var userPredicate, groupPredicate func(string) bool
 		if sys.LDAPConfig.Enabled() {
 			userPredicate = func(s string) bool {
@@ -898,7 +958,7 @@ func (sys *IAMSys) QueryPolicyEntities(ctx context.Context, q madmin.PolicyEntit
 				return !sys.LDAPConfig.IsLDAPGroupDN(s)
 			}
 		}
-		pe := sys.store.ListPolicyMappings(q, userPredicate, groupPredicate)
+		pe := sys.store.ListPolicyMappings(cleanQuery, userPredicate, groupPredicate, nil)
 		pe.Timestamp = UTCNow()
 		return &pe, nil
 	case <-ctx.Done():
@@ -1382,11 +1442,13 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 	// 1. Collect all LDAP users with active creds.
 	allCreds := sys.store.GetSTSAndServiceAccounts()
 	// List of unique LDAP (parent) user DNs that have active creds
-	var parentUsers []string
-	// Map of LDAP user to list of active credential objects
+	var parentUserActualDNList []string
+	// Map of LDAP user (internal representation) to list of active credential objects
 	parentUserToCredsMap := make(map[string][]auth.Credentials)
 	// DN to ldap username mapping for each LDAP user
-	parentUserToLDAPUsernameMap := make(map[string]string)
+	actualDNToLDAPUsernameMap := make(map[string]string)
+	// External (actual) LDAP DN to internal normalized representation
+	actualDNToParentUserMap := make(map[string]string)
 	for _, cred := range allCreds {
 		// Expired credentials don't need parent user updates.
 		if cred.IsExpired() {
@@ -1429,25 +1491,28 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 				continue
 			}
 
-			ldapUsername, ok := jwtClaims.Lookup(ldapUserN)
-			if !ok {
+			ldapUsername, okUserN := jwtClaims.Lookup(ldapUserN)
+			ldapActualDN, okDN := jwtClaims.Lookup(ldapActualUser)
+			if !okUserN || !okDN {
 				// skip this cred - we dont have the
 				// username info needed
 				continue
 			}
 
 			// Collect each new cred.ParentUser into parentUsers
-			parentUsers = append(parentUsers, cred.ParentUser)
+			parentUserActualDNList = append(parentUserActualDNList, ldapActualDN)
 
 			// Update the ldapUsernameMap
-			parentUserToLDAPUsernameMap[cred.ParentUser] = ldapUsername
+			actualDNToLDAPUsernameMap[ldapActualDN] = ldapUsername
+
+			// Update the actualDNToParentUserMap
+			actualDNToParentUserMap[ldapActualDN] = cred.ParentUser
 		}
 		parentUserToCredsMap[cred.ParentUser] = append(parentUserToCredsMap[cred.ParentUser], cred)
-
 	}
 
 	// 2. Query LDAP server for groups of the LDAP users collected.
-	updatedGroups, err := sys.LDAPConfig.LookupGroupMemberships(parentUsers, parentUserToLDAPUsernameMap)
+	updatedGroups, err := sys.LDAPConfig.LookupGroupMemberships(parentUserActualDNList, actualDNToLDAPUsernameMap)
 	if err != nil {
 		// Log and return on error - perhaps it'll work the next time.
 		iamLogIf(GlobalContext, err)
@@ -1455,8 +1520,9 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 	}
 
 	// 3. Update creds for those users whose groups are changed
-	for _, parentUser := range parentUsers {
-		currGroupsSet := updatedGroups[parentUser]
+	for _, parentActualDN := range parentUserActualDNList {
+		currGroupsSet := updatedGroups[parentActualDN]
+		parentUser := actualDNToParentUserMap[parentActualDN]
 		currGroups := currGroupsSet.ToSlice()
 		for _, cred := range parentUserToCredsMap[parentUser] {
 			gSet := set.CreateStringSet(cred.Groups...)
@@ -1514,11 +1580,11 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap
 		// server and is under a configured base DN.
 		validatedParent, isUnderBaseDN, err := sys.LDAPConfig.GetValidatedUserDN(conn, parent)
 		if err != nil {
-			collectedErrors = append(collectedErrors, fmt.Errorf("could not validate `%s` exists in LDAP directory: %w", parent, err))
+			collectedErrors = append(collectedErrors, fmt.Errorf("could not validate parent exists in LDAP directory: %w", err))
 			continue
 		}
 		if validatedParent == nil || !isUnderBaseDN {
-			err := fmt.Errorf("DN `%s` was not found in the LDAP directory", parent)
+			err := fmt.Errorf("DN parent was not found in the LDAP directory")
 			collectedErrors = append(collectedErrors, err)
 			continue
 		}
@@ -1533,11 +1599,11 @@ func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap
 			// configured base DN.
 			validatedGroup, _, err := sys.LDAPConfig.GetValidatedGroupDN(conn, group)
 			if err != nil {
-				collectedErrors = append(collectedErrors, fmt.Errorf("could not validate `%s` exists in LDAP directory: %w", group, err))
+				collectedErrors = append(collectedErrors, fmt.Errorf("could not validate group exists in LDAP directory: %w", err))
 				continue
 			}
 			if validatedGroup == nil {
-				err := fmt.Errorf("DN `%s` was not found in the LDAP directory", group)
+				err := fmt.Errorf("DN group was not found in the LDAP directory")
 				collectedErrors = append(collectedErrors, err)
 				continue
 			}
@@ -1627,7 +1693,7 @@ func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
 			continue
 		}
 		if validatedDN == nil || !underBaseDN {
-			err := fmt.Errorf("DN `%s` was not found in the LDAP directory", k)
+			err := fmt.Errorf("DN was not found in the LDAP directory")
 			collectedErrors = append(collectedErrors, err)
 			continue
 		}
@@ -1970,7 +2036,7 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
 		if dnResult == nil {
 			// dn not found - still attempt to detach if provided user is a DN.
 			if !isAttach && sys.LDAPConfig.IsLDAPUserDN(r.User) {
-				dn = r.User
+				dn = sys.LDAPConfig.QuickNormalizeDN(r.User)
 			} else {
 				err = errNoSuchUser
 				return
@@ -1987,7 +2053,7 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
 		}
 		if dnResult == nil || !underBaseDN {
 			if !isAttach {
-				dn = r.Group
+				dn = sys.LDAPConfig.QuickNormalizeDN(r.Group)
 			} else {
 				err = errNoSuchGroup
 				return
@@ -2214,22 +2280,16 @@ func (sys *IAMSys) IsAllowedSTS(args policy.Args, parentUser string) bool {
 	// 2. Combine the mapped policies into a single combined policy.
 
 	var combinedPolicy policy.Policy
+	// Policies were found, evaluate all of them.
 	if !isOwnerDerived {
-		var err error
-		combinedPolicy, err = sys.store.GetPolicy(strings.Join(policies, ","))
-		if errors.Is(err, errNoSuchPolicy) {
-			for _, pname := range policies {
-				_, err := sys.store.GetPolicy(pname)
-				if errors.Is(err, errNoSuchPolicy) {
-					// all policies presented in the claim should exist
-					iamLogIf(GlobalContext, fmt.Errorf("expected policy (%s) missing from the JWT claim %s, rejecting the request", pname, iamPolicyClaimNameOpenID()))
-					return false
-				}
-			}
-			iamLogIf(GlobalContext, fmt.Errorf("all policies were unexpectedly present!"))
+		availablePoliciesStr, c := sys.store.MergePolicies(strings.Join(policies, ","))
+		if availablePoliciesStr == "" {
+			// all policies presented in the claim should exist
+			iamLogIf(GlobalContext, fmt.Errorf("expected policy (%s) missing from the JWT claim %s, rejecting the request", policies, iamPolicyClaimNameOpenID()))
+
 			return false
 		}
-
+		combinedPolicy = c
 	}
 
 	// 3. If an inline session-policy is present, evaluate it.

cmd/jwt.go

@@ -24,10 +24,8 @@ import (
 
 	jwtgo "github.com/golang-jwt/jwt/v4"
 	jwtreq "github.com/golang-jwt/jwt/v4/request"
-	"github.com/hashicorp/golang-lru/v2/expirable"
 	"github.com/minio/minio/internal/auth"
 	xjwt "github.com/minio/minio/internal/jwt"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/policy"
 )
 
@@ -37,8 +35,8 @@ const (
 	// Default JWT token for web handlers is one day.
 	defaultJWTExpiry = 24 * time.Hour
 
-	// Inter-node JWT token expiry is 15 minutes.
-	defaultInterNodeJWTExpiry = 15 * time.Minute
+	// Inter-node JWT token expiry is 100 years approx.
+	defaultInterNodeJWTExpiry = 100 * 365 * 24 * time.Hour
 )
 
 var (
@@ -50,17 +48,10 @@ var (
 	errMalformedAuth      = errors.New("Malformed authentication input")
 )
 
-type cacheKey struct {
-	accessKey, secretKey, audience string
-}
-
-var cacheLRU = expirable.NewLRU[cacheKey, string](1000, nil, 15*time.Second)
-
-func authenticateNode(accessKey, secretKey, audience string) (string, error) {
+func authenticateNode(accessKey, secretKey string) (string, error) {
 	claims := xjwt.NewStandardClaims()
 	claims.SetExpiry(UTCNow().Add(defaultInterNodeJWTExpiry))
 	claims.SetAccessKey(accessKey)
-	claims.SetAudience(audience)
 
 	jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, claims)
 	return jwt.SignedString([]byte(secretKey))
@@ -141,27 +132,9 @@ func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, b
 	return claims, groups, owner, nil
 }
 
-// newCachedAuthToken returns a token that is cached up to 15 seconds.
-// If globalActiveCred is updated it is reflected at once.
-func newCachedAuthToken() func(audience string) string {
-	fn := func(accessKey, secretKey, audience string) (s string, err error) {
-		k := cacheKey{accessKey: accessKey, secretKey: secretKey, audience: audience}
-
-		var ok bool
-		s, ok = cacheLRU.Get(k)
-		if !ok {
-			s, err = authenticateNode(accessKey, secretKey, audience)
-			if err != nil {
-				return "", err
-			}
-			cacheLRU.Add(k, s)
-		}
-		return s, nil
-	}
-	return func(audience string) string {
-		cred := globalActiveCred
-		token, err := fn(cred.AccessKey, cred.SecretKey, audience)
-		logger.CriticalIf(GlobalContext, err)
-		return token
+// newCachedAuthToken returns the cached token.
+func newCachedAuthToken() func() string {
+	return func() string {
+		return globalNodeAuthToken
 	}
 }

cmd/jwt_test.go

@@ -107,7 +107,7 @@ func BenchmarkParseJWTStandardClaims(b *testing.B) {
 	}
 
 	creds := globalActiveCred
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey, "")
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -138,7 +138,7 @@ func BenchmarkParseJWTMapClaims(b *testing.B) {
 	}
 
 	creds := globalActiveCred
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey, "")
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -176,7 +176,7 @@ func BenchmarkAuthenticateNode(b *testing.B) {
 		b.ResetTimer()
 		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
-			fn(creds.AccessKey, creds.SecretKey, "aud")
+			fn(creds.AccessKey, creds.SecretKey)
 		}
 	})
 	b.Run("cached", func(b *testing.B) {
@@ -184,7 +184,7 @@ func BenchmarkAuthenticateNode(b *testing.B) {
 		b.ResetTimer()
 		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
-			fn("aud")
+			fn()
 		}
 	})
 }

cmd/kms-handlers.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/minio/kms-go/kes"
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/policy"
@@ -56,7 +57,7 @@ func (a kmsAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Request)
 	writeSuccessResponseJSON(w, resp)
 }
 
-// KMSMetricsHandler - POST /minio/kms/v1/metrics
+// KMSMetricsHandler - GET /minio/kms/v1/metrics
 func (a kmsAPIHandlers) KMSMetricsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSMetrics")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -83,7 +84,7 @@ func (a kmsAPIHandlers) KMSMetricsHandler(w http.ResponseWriter, r *http.Request
 	}
 }
 
-// KMSAPIsHandler - POST /minio/kms/v1/apis
+// KMSAPIsHandler - GET /minio/kms/v1/apis
 func (a kmsAPIHandlers) KMSAPIsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSAPIs")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -114,7 +115,7 @@ type versionResponse struct {
 	Version string `json:"version"`
 }
 
-// KMSVersionHandler - POST /minio/kms/v1/version
+// KMSVersionHandler - GET /minio/kms/v1/version
 func (a kmsAPIHandlers) KMSVersionHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSVersion")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -159,7 +160,20 @@ func (a kmsAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	if err := GlobalKMS.CreateKey(ctx, &kms.CreateKeyRequest{Name: r.Form.Get("key-id")}); err != nil {
+	keyID := r.Form.Get("key-id")
+
+	// Ensure policy allows the user to create this key name
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+	if !checkKMSActionAllowed(r, owner, cred, policy.KMSCreateKeyAction, keyID) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
+	if err := GlobalKMS.CreateKey(ctx, &kms.CreateKeyRequest{Name: keyID}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -171,6 +185,9 @@ func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Reques
 	ctx := newContext(r, w, "KMSListKeys")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
+	// This only checks if the action (kms:ListKeys) is allowed, it does not check
+	// each key name against the policy's Resources. We check that below, once
+	// we have the list of key names from the KMS.
 	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSListKeysAction)
 	if objectAPI == nil {
 		return
@@ -180,7 +197,7 @@ func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Reques
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
 		return
 	}
-	names, _, err := GlobalKMS.ListKeyNames(ctx, &kms.ListRequest{
+	allKeyNames, _, err := GlobalKMS.ListKeyNames(ctx, &kms.ListRequest{
 		Prefix: r.Form.Get("pattern"),
 	})
 	if err != nil {
@@ -188,8 +205,24 @@ func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	values := make([]kes.KeyInfo, 0, len(names))
-	for _, name := range names {
+	// Get the cred and owner for checking authz below.
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	// Now we have all the key names, for each of them, check whether the policy grants permission for
+	// the user to list it.
+	keyNames := []string{}
+	for _, name := range allKeyNames {
+		if checkKMSActionAllowed(r, owner, cred, policy.KMSListKeysAction, name) {
+			keyNames = append(keyNames, name)
+		}
+	}
+
+	values := make([]kes.KeyInfo, 0, len(keyNames))
+	for _, name := range keyNames {
 		values = append(values, kes.KeyInfo{
 			Name: name,
 		})
@@ -224,6 +257,17 @@ func (a kmsAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Reque
 		KeyID: keyID,
 	}
 
+	// Ensure policy allows the user to get this key's status
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+	if !checkKMSActionAllowed(r, owner, cred, policy.KMSKeyStatusAction, keyID) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
 	kmsContext := kms.Context{"MinIO admin API": "KMSKeyStatusHandler"} // Context for a test key operation
 	// 1. Generate a new key using the KMS.
 	key, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{Name: keyID, AssociatedData: kmsContext})
@@ -274,3 +318,16 @@ func (a kmsAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Reque
 	}
 	writeSuccessResponseJSON(w, resp)
 }
+
+// checkKMSActionAllowed checks for authorization for a specific action on a resource.
+func checkKMSActionAllowed(r *http.Request, owner bool, cred auth.Credentials, action policy.KMSAction, resource string) bool {
+	return globalIAMSys.IsAllowed(policy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          policy.Action(action),
+		ConditionValues: getConditionValues(r, "", cred),
+		IsOwner:         owner,
+		Claims:          cred.Claims,
+		BucketName:      resource, // overloading BucketName as that's what the policy engine uses to assemble a Resource.
+	})
+}

cmd/kms-handlers_test.go

@@ -0,0 +1,851 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/pkg/v3/policy"
+)
+
+const (
+	// KMS API paths
+	// For example: /minio/kms/v1/key/list?pattern=*
+	kmsURL           = kmsPathPrefix + kmsAPIVersionPrefix
+	kmsStatusPath    = kmsURL + "/status"
+	kmsMetricsPath   = kmsURL + "/metrics"
+	kmsAPIsPath      = kmsURL + "/apis"
+	kmsVersionPath   = kmsURL + "/version"
+	kmsKeyCreatePath = kmsURL + "/key/create"
+	kmsKeyListPath   = kmsURL + "/key/list"
+	kmsKeyStatusPath = kmsURL + "/key/status"
+
+	// Admin API paths
+	// For example: /minio/admin/v3/kms/status
+	adminURL              = adminPathPrefix + adminAPIVersionPrefix
+	kmsAdminStatusPath    = adminURL + "/kms/status"
+	kmsAdminKeyStatusPath = adminURL + "/kms/key/status"
+	kmsAdminKeyCreate     = adminURL + "/kms/key/create"
+)
+
+const (
+	userAccessKey = "miniofakeuseraccesskey"
+	userSecretKey = "miniofakeusersecret"
+)
+
+type kmsTestCase struct {
+	name   string
+	method string
+	path   string
+	query  map[string]string
+
+	// User credentials and policy for request
+	policy string
+	asRoot bool
+
+	// Wanted in response.
+	wantStatusCode int
+	wantKeyNames   []string
+	wantResp       []string
+}
+
+func TestKMSHandlersCreateKey(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, true)
+	defer tearDown()
+
+	tests := []kmsTestCase{
+		// Create key test
+		{
+			name:   "create key as user with no policy want forbidden",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "new-test-key"},
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "create key as user with no resources specified want success",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "new-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:CreateKey"] }`,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "create key as user set policy to allow want success",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "second-new-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:CreateKey"],
+				"Resource": ["arn:minio:kms:::second-new-test-*"] }`,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "create key as user set policy to non matching resource want forbidden",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "third-new-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:CreateKey"],
+				"Resource": ["arn:minio:kms:::non-matching-key-name"] }`,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+	}
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) {
+			execKMSTest(t, test, adminTestBed)
+		})
+	}
+}
+
+func TestKMSHandlersKeyStatus(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, true)
+	defer tearDown()
+
+	tests := []kmsTestCase{
+		{
+			name:   "create a first key root user",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "key status as root want success",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"abc-test-key"},
+		},
+		{
+			name:   "key status as user no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "key status as user legacy no resources specified want success",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:KeyStatus"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"abc-test-key"},
+		},
+		{
+			name:   "key status as user set policy to allow only one key",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:KeyStatus"],
+					"Resource": ["arn:minio:kms:::abc-test-*"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"abc-test-key"},
+		},
+		{
+			name:   "key status as user set policy to allow non-matching key",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:KeyStatus"],
+					"Resource": ["arn:minio:kms:::xyz-test-key"] }`,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+	}
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) {
+			execKMSTest(t, test, adminTestBed)
+		})
+	}
+}
+
+func TestKMSHandlersAPIs(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, true)
+	defer tearDown()
+
+	tests := []kmsTestCase{
+		// Version test
+		{
+			name:   "version as root want success",
+			method: http.MethodGet,
+			path:   kmsVersionPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"version"},
+		},
+		{
+			name:   "version as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsVersionPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "version as user with policy ignores resource want success",
+			method: http.MethodGet,
+			path:   kmsVersionPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:Version"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"version"},
+		},
+
+		// APIs test
+		{
+			name:   "apis as root want success",
+			method: http.MethodGet,
+			path:   kmsAPIsPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"stub/path"},
+		},
+		{
+			name:   "apis as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsAPIsPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "apis as user with policy ignores resource want success",
+			method: http.MethodGet,
+			path:   kmsAPIsPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:API"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"stub/path"},
+		},
+
+		// Metrics test
+		{
+			name:   "metrics as root want success",
+			method: http.MethodGet,
+			path:   kmsMetricsPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"kms"},
+		},
+		{
+			name:   "metrics as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsMetricsPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "metrics as user with policy ignores resource want success",
+			method: http.MethodGet,
+			path:   kmsMetricsPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:Metrics"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"kms"},
+		},
+
+		// Status tests
+		{
+			name:   "status as root want success",
+			method: http.MethodGet,
+			path:   kmsStatusPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"MinIO builtin"},
+		},
+		{
+			name:   "status as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsStatusPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "status as user with policy ignores resource want success",
+			method: http.MethodGet,
+			path:   kmsStatusPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:Status"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"]}`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"MinIO builtin"},
+		},
+	}
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) {
+			execKMSTest(t, test, adminTestBed)
+		})
+	}
+}
+
+func TestKMSHandlersListKeys(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, true)
+	defer tearDown()
+
+	tests := []kmsTestCase{
+		{
+			name:   "create a first key root user",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "create a second key root user",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "xyz-test-key"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+		},
+
+		// List keys tests
+		{
+			name:   "list keys as root want all to be returned",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{"default-test-key", "abc-test-key", "xyz-test-key"},
+		},
+		{
+			name:   "list keys as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "list keys as user with no resources specified want success",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:ListKeys"]
+			}`,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{"default-test-key", "abc-test-key", "xyz-test-key"},
+		},
+		{
+			name:   "list keys as user set policy resource to allow only one key",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:ListKeys"],
+					"Resource": ["arn:minio:kms:::abc*"]}`,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{"abc-test-key"},
+		},
+		{
+			name:   "list keys as user set policy to allow only one key, use pattern that includes correct key",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "abc*"},
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:ListKeys"],
+					"Resource": ["arn:minio:kms:::abc*"]}`,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{"abc-test-key"},
+		},
+		{
+			name:   "list keys as user set policy to allow only one key, use pattern that excludes correct key",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "xyz*"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:ListKeys"],
+					"Resource": ["arn:minio:kms:::abc*"]}`,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{},
+		},
+		{
+			name:   "list keys as user set policy that has no matching key resources",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:ListKeys"],
+					"Resource": ["arn:minio:kms:::nonematch*"]}`,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{},
+		},
+		{
+			name:   "list keys as user set policy that allows listing but denies specific keys",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: false,
+
+			// It looks like this should allow listing any key that isn't "default-test-key", however
+			// the policy engine matches all Deny statements first, without regard to Resources (for KMS).
+			// This is for backwards compatibility where historically KMS statements ignored Resources.
+			policy: `{
+						"Effect": "Allow",
+						"Action": ["kms:ListKeys"]
+					},{
+						"Effect": "Deny",
+						"Action": ["kms:ListKeys"],
+						"Resource": ["arn:minio:kms:::default-test-key"]
+					}`,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+	}
+
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) {
+			execKMSTest(t, test, adminTestBed)
+		})
+	}
+}
+
+func TestKMSHandlerAdminAPI(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, true)
+	defer tearDown()
+
+	tests := []kmsTestCase{
+		// Create key tests
+		{
+			name:   "create a key root user",
+			method: http.MethodPost,
+			path:   kmsAdminKeyCreate,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "create key as user with no policy want forbidden",
+			method: http.MethodPost,
+			path:   kmsAdminKeyCreate,
+			query:  map[string]string{"key-id": "new-test-key"},
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "create key as user with no resources specified want success",
+			method: http.MethodPost,
+			path:   kmsAdminKeyCreate,
+			query:  map[string]string{"key-id": "new-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["admin:KMSCreateKey"] }`,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "create key as user set policy to non matching resource want success",
+			method: http.MethodPost,
+			path:   kmsAdminKeyCreate,
+			query:  map[string]string{"key-id": "third-new-test-key"},
+			asRoot: false,
+
+			// Admin actions ignore Resources
+			policy: `{"Effect": "Allow",
+				"Action": ["admin:KMSCreateKey"],
+				"Resource": ["arn:minio:kms:::this-is-disregarded"] }`,
+
+			wantStatusCode: http.StatusOK,
+		},
+
+		// Status tests
+		{
+			name:   "status as root want success",
+			method: http.MethodPost,
+			path:   kmsAdminStatusPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"MinIO builtin"},
+		},
+		{
+			name:   "status as user with no policy want forbidden",
+			method: http.MethodPost,
+			path:   kmsAdminStatusPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "status as user with policy ignores resource want success",
+			method: http.MethodPost,
+			path:   kmsAdminStatusPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["admin:KMSKeyStatus"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"MinIO builtin"},
+		},
+
+		// Key status tests
+		{
+			name:   "key status as root want success",
+			method: http.MethodGet,
+			path:   kmsAdminKeyStatusPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"key-id"},
+		},
+		{
+			name:   "key status as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsAdminKeyStatusPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "key status as user with policy ignores resource want success",
+			method: http.MethodGet,
+			path:   kmsAdminKeyStatusPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["admin:KMSKeyStatus"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"key-id"},
+		},
+	}
+
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) {
+			execKMSTest(t, test, adminTestBed)
+		})
+	}
+}
+
+// execKMSTest runs a single test case for KMS handlers
+func execKMSTest(t *testing.T, test kmsTestCase, adminTestBed *adminErasureTestBed) {
+	var accessKey, secretKey string
+	if test.asRoot {
+		accessKey, secretKey = globalActiveCred.AccessKey, globalActiveCred.SecretKey
+	} else {
+		setupKMSUser(t, userAccessKey, userSecretKey, test.policy)
+		accessKey = userAccessKey
+		secretKey = userSecretKey
+	}
+
+	req := buildKMSRequest(t, test.method, test.path, accessKey, secretKey, test.query)
+	rec := httptest.NewRecorder()
+	adminTestBed.router.ServeHTTP(rec, req)
+
+	t.Logf("HTTP req: %s, resp code: %d, resp body: %s", req.URL.String(), rec.Code, rec.Body.String())
+
+	// Check status code
+	if rec.Code != test.wantStatusCode {
+		t.Errorf("want status code %d, got %d", test.wantStatusCode, rec.Code)
+	}
+
+	// Check returned key list is correct
+	if test.wantKeyNames != nil {
+		gotKeyNames := keyNamesFromListKeysResp(t, rec.Body.Bytes())
+		if len(test.wantKeyNames) != len(gotKeyNames) {
+			t.Fatalf("want keys len: %d, got len: %d", len(test.wantKeyNames), len(gotKeyNames))
+		}
+		for i, wantKeyName := range test.wantKeyNames {
+			if gotKeyNames[i] != wantKeyName {
+				t.Fatalf("want key name %s, in position %d, got %s", wantKeyName, i, gotKeyNames[i])
+			}
+		}
+	}
+
+	// Check generic text in the response
+	if test.wantResp != nil {
+		for _, want := range test.wantResp {
+			if !strings.Contains(rec.Body.String(), want) {
+				t.Fatalf("want response to contain %s, got %s", want, rec.Body.String())
+			}
+		}
+	}
+}
+
+// TestKMSHandlerNotConfiguredOrInvalidCreds tests KMS handlers for situations where KMS is not configured
+// or invalid credentials are provided.
+func TestKMSHandlerNotConfiguredOrInvalidCreds(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, false)
+	defer tearDown()
+
+	tests := []struct {
+		name   string
+		method string
+		path   string
+		query  map[string]string
+	}{
+		{
+			name:   "GET status",
+			method: http.MethodGet,
+			path:   kmsStatusPath,
+		},
+		{
+			name:   "GET metrics",
+			method: http.MethodGet,
+			path:   kmsMetricsPath,
+		},
+		{
+			name:   "GET apis",
+			method: http.MethodGet,
+			path:   kmsAPIsPath,
+		},
+		{
+			name:   "GET version",
+			method: http.MethodGet,
+			path:   kmsVersionPath,
+		},
+		{
+			name:   "POST key create",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "master-key-id"},
+		},
+		{
+			name:   "GET key list",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+		},
+		{
+			name:   "GET key status",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "master-key-id"},
+		},
+	}
+
+	// Test when the GlobalKMS is not configured
+	for _, test := range tests {
+		t.Run(test.name+" not configured", func(t *testing.T) {
+			req := buildKMSRequest(t, test.method, test.path, "", "", test.query)
+			rec := httptest.NewRecorder()
+			adminTestBed.router.ServeHTTP(rec, req)
+			if rec.Code != http.StatusNotImplemented {
+				t.Errorf("want status code %d, got %d", http.StatusNotImplemented, rec.Code)
+			}
+		})
+	}
+
+	// Test when the GlobalKMS is configured but the credentials are invalid
+	GlobalKMS = kms.NewStub("default-test-key")
+	for _, test := range tests {
+		t.Run(test.name+" invalid credentials", func(t *testing.T) {
+			req := buildKMSRequest(t, test.method, test.path, userAccessKey, userSecretKey, test.query)
+			rec := httptest.NewRecorder()
+			adminTestBed.router.ServeHTTP(rec, req)
+			if rec.Code != http.StatusForbidden {
+				t.Errorf("want status code %d, got %d", http.StatusForbidden, rec.Code)
+			}
+		})
+	}
+}
+
+func setupKMSTest(t *testing.T, enableKMS bool) (*adminErasureTestBed, func()) {
+	adminTestBed, err := prepareAdminErasureTestBed(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	registerKMSRouter(adminTestBed.router)
+
+	if enableKMS {
+		GlobalKMS = kms.NewStub("default-test-key")
+	}
+
+	tearDown := func() {
+		adminTestBed.TearDown()
+		GlobalKMS = nil
+	}
+	return adminTestBed, tearDown
+}
+
+func buildKMSRequest(t *testing.T, method, path, accessKey, secretKey string, query map[string]string) *http.Request {
+	if len(query) > 0 {
+		queryVal := url.Values{}
+		for k, v := range query {
+			queryVal.Add(k, v)
+		}
+		path = path + "?" + queryVal.Encode()
+	}
+
+	if accessKey == "" && secretKey == "" {
+		accessKey = globalActiveCred.AccessKey
+		secretKey = globalActiveCred.SecretKey
+	}
+
+	req, err := newTestSignedRequestV4(method, path, 0, nil, accessKey, secretKey, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return req
+}
+
+// setupKMSUser is a test helper that creates a new user with the provided access key and secret key
+// and applies the given policy to the user.
+func setupKMSUser(t *testing.T, accessKey, secretKey, p string) {
+	ctx := context.Background()
+	createUserParams := madmin.AddOrUpdateUserReq{
+		SecretKey: secretKey,
+		Status:    madmin.AccountEnabled,
+	}
+	_, err := globalIAMSys.CreateUser(ctx, accessKey, createUserParams)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testKMSPolicyName := "testKMSPolicy"
+	if p != "" {
+		p = `{"Version":"2012-10-17","Statement":[` + p + `]}`
+		policyData, err := policy.ParseConfig(strings.NewReader(p))
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, err = globalIAMSys.SetPolicy(ctx, testKMSPolicyName, *policyData)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, err = globalIAMSys.PolicyDBSet(ctx, accessKey, testKMSPolicyName, regUser, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+	} else {
+		err = globalIAMSys.DeletePolicy(ctx, testKMSPolicyName, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, err = globalIAMSys.PolicyDBSet(ctx, accessKey, "", regUser, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func keyNamesFromListKeysResp(t *testing.T, b []byte) []string {
+	var keyInfos []madmin.KMSKeyInfo
+	err := json.Unmarshal(b, &keyInfos)
+	if err != nil {
+		t.Fatalf("cannot unmarshal '%s', err: %v", b, err)
+	}
+
+	var gotKeyNames []string
+	for _, keyInfo := range keyInfos {
+		gotKeyNames = append(gotKeyNames, keyInfo.Name)
+	}
+	return gotKeyNames
+}

cmd/license-update.go

@@ -1,117 +0,0 @@
-// Copyright (c) 2015-2023 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"math/rand"
-	"time"
-
-	"github.com/tidwall/gjson"
-)
-
-const (
-	licUpdateCycle = 24 * time.Hour * 30
-	licRenewPath   = "/api/cluster/renew-license"
-)
-
-// initlicenseUpdateJob start the periodic license update job in the background.
-func initLicenseUpdateJob(ctx context.Context, objAPI ObjectLayer) {
-	go func() {
-		r := rand.New(rand.NewSource(time.Now().UnixNano()))
-		// Leader node (that successfully acquires the lock inside licenceUpdaterLoop)
-		// will keep performing the license update. If the leader goes down for some
-		// reason, the lock will be released and another node will acquire it and
-		// take over because of this loop.
-		for {
-			licenceUpdaterLoop(ctx, objAPI)
-
-			// license update stopped for some reason.
-			// sleep for some time and try again.
-			duration := time.Duration(r.Float64() * float64(time.Hour))
-			if duration < time.Second {
-				// Make sure to sleep at least a second to avoid high CPU ticks.
-				duration = time.Second
-			}
-			time.Sleep(duration)
-		}
-	}()
-}
-
-func licenceUpdaterLoop(ctx context.Context, objAPI ObjectLayer) {
-	ctx, cancel := globalLeaderLock.GetLock(ctx)
-	defer cancel()
-
-	licenseUpdateTimer := time.NewTimer(licUpdateCycle)
-	defer licenseUpdateTimer.Stop()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-licenseUpdateTimer.C:
-
-			if globalSubnetConfig.Registered() {
-				performLicenseUpdate(ctx, objAPI)
-			}
-
-			// Reset the timer for next cycle.
-			licenseUpdateTimer.Reset(licUpdateCycle)
-		}
-	}
-}
-
-func performLicenseUpdate(ctx context.Context, objectAPI ObjectLayer) {
-	// the subnet license renewal api renews the license only
-	// if required e.g. when it is expiring soon
-	url := globalSubnetConfig.BaseURL + licRenewPath
-
-	resp, err := globalSubnetConfig.Post(url, nil)
-	if err != nil {
-		subnetLogIf(ctx, fmt.Errorf("error from %s: %w", url, err))
-		return
-	}
-
-	r := gjson.Parse(resp).Get("license_v2")
-	if r.Index == 0 {
-		internalLogIf(ctx, fmt.Errorf("license not found in response from %s", url))
-		return
-	}
-
-	lic := r.String()
-	if lic == globalSubnetConfig.License {
-		// license hasn't changed.
-		return
-	}
-
-	kv := "subnet license=" + lic
-	result, err := setConfigKV(ctx, objectAPI, []byte(kv))
-	if err != nil {
-		internalLogIf(ctx, fmt.Errorf("error setting subnet license config: %w", err))
-		return
-	}
-
-	if result.Dynamic {
-		if err := applyDynamicConfigForSubSys(GlobalContext, objectAPI, result.Cfg, result.SubSys); err != nil {
-			subnetLogIf(ctx, fmt.Errorf("error applying subnet dynamic config: %w", err))
-			return
-		}
-		globalNotificationSys.SignalConfigReload(result.SubSys)
-	}
-}

cmd/local-locker.go

@@ -31,16 +31,16 @@ import (
 
 // lockRequesterInfo stores various info from the client for each lock that is requested.
 type lockRequesterInfo struct {
-	Name            string    // name of the resource lock was requested for
-	Writer          bool      // Bool whether write or read lock.
-	UID             string    // UID to uniquely identify request of client.
-	Timestamp       time.Time // Timestamp set at the time of initialization.
-	TimeLastRefresh time.Time // Timestamp for last lock refresh.
-	Source          string    // Contains line, function and filename requesting the lock.
-	Group           bool      // indicates if it was a group lock.
-	Owner           string    // Owner represents the UUID of the owner who originally requested the lock.
-	Quorum          int       // Quorum represents the quorum required for this lock to be active.
-	idx             int       `msg:"-"` // index of the lock in the lockMap.
+	Name            string // name of the resource lock was requested for
+	Writer          bool   // Bool whether write or read lock.
+	UID             string // UID to uniquely identify request of client.
+	Timestamp       int64  // Timestamp set at the time of initialization.
+	TimeLastRefresh int64  // Timestamp for last lock refresh.
+	Source          string // Contains line, function and filename requesting the lock.
+	Group           bool   // indicates if it was a group lock.
+	Owner           string // Owner represents the UUID of the owner who originally requested the lock.
+	Quorum          int    // Quorum represents the quorum required for this lock to be active.
+	idx             int    `msg:"-"` // index of the lock in the lockMap.
 }
 
 // isWriteLock returns whether the lock is a write or read lock.
@@ -87,6 +87,7 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
 
 	// No locks held on the all resources, so claim write
 	// lock on all resources at once.
+	now := UTCNow()
 	for i, resource := range args.Resources {
 		l.lockMap[resource] = []lockRequesterInfo{
 			{
@@ -95,10 +96,10 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
 				Source:          args.Source,
 				Owner:           args.Owner,
 				UID:             args.UID,
-				Timestamp:       UTCNow(),
-				TimeLastRefresh: UTCNow(),
+				Timestamp:       now.UnixNano(),
+				TimeLastRefresh: now.UnixNano(),
 				Group:           len(args.Resources) > 1,
-				Quorum:          args.Quorum,
+				Quorum:          *args.Quorum,
 				idx:             i,
 			},
 		}
@@ -108,7 +109,7 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
 }
 
 func formatUUID(s string, idx int) string {
-	return s + strconv.Itoa(idx)
+	return concat(s, strconv.Itoa(idx))
 }
 
 func (l *localLocker) Unlock(_ context.Context, args dsync.LockArgs) (reply bool, err error) {
@@ -167,17 +168,19 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
 	resource := args.Resources[0]
+	now := UTCNow()
 	lrInfo := lockRequesterInfo{
 		Name:            resource,
 		Writer:          false,
 		Source:          args.Source,
 		Owner:           args.Owner,
 		UID:             args.UID,
-		Timestamp:       UTCNow(),
-		TimeLastRefresh: UTCNow(),
-		Quorum:          args.Quorum,
+		Timestamp:       now.UnixNano(),
+		TimeLastRefresh: now.UnixNano(),
+		Quorum:          *args.Quorum,
 	}
-	if lri, ok := l.lockMap[resource]; ok {
+	lri, ok := l.lockMap[resource]
+	if ok {
 		if reply = !isWriteLock(lri); reply {
 			// Unless there is a write lock
 			l.lockMap[resource] = append(l.lockMap[resource], lrInfo)
@@ -351,9 +354,10 @@ func (l *localLocker) Refresh(ctx context.Context, args dsync.LockArgs) (refresh
 				delete(l.lockUID, formatUUID(args.UID, idx))
 				return idx > 0, nil
 			}
+			now := UTCNow()
 			for i := range lris {
 				if lris[i].UID == args.UID {
-					lris[i].TimeLastRefresh = UTCNow()
+					lris[i].TimeLastRefresh = now.UnixNano()
 				}
 			}
 			idx++
@@ -376,7 +380,7 @@ func (l *localLocker) expireOldLocks(interval time.Duration) {
 		modified := false
 		for i := 0; i < len(lris); {
 			lri := &lris[i]
-			if time.Since(lri.TimeLastRefresh) > interval {
+			if time.Since(time.Unix(0, lri.TimeLastRefresh)) > interval {
 				delete(l.lockUID, formatUUID(lri.UID, lri.idx))
 				if len(lris) == 1 {
 					// Remove the write lock.

cmd/local-locker_gen.go

@@ -204,13 +204,13 @@ func (z *lockRequesterInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "Timestamp":
-			z.Timestamp, err = dc.ReadTime()
+			z.Timestamp, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "Timestamp")
 				return
 			}
 		case "TimeLastRefresh":
-			z.TimeLastRefresh, err = dc.ReadTime()
+			z.TimeLastRefresh, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "TimeLastRefresh")
 				return
@@ -288,7 +288,7 @@ func (z *lockRequesterInfo) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteTime(z.Timestamp)
+	err = en.WriteInt64(z.Timestamp)
 	if err != nil {
 		err = msgp.WrapError(err, "Timestamp")
 		return
@@ -298,7 +298,7 @@ func (z *lockRequesterInfo) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteTime(z.TimeLastRefresh)
+	err = en.WriteInt64(z.TimeLastRefresh)
 	if err != nil {
 		err = msgp.WrapError(err, "TimeLastRefresh")
 		return
@@ -361,10 +361,10 @@ func (z *lockRequesterInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendString(o, z.UID)
 	// string "Timestamp"
 	o = append(o, 0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70)
-	o = msgp.AppendTime(o, z.Timestamp)
+	o = msgp.AppendInt64(o, z.Timestamp)
 	// string "TimeLastRefresh"
 	o = append(o, 0xaf, 0x54, 0x69, 0x6d, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68)
-	o = msgp.AppendTime(o, z.TimeLastRefresh)
+	o = msgp.AppendInt64(o, z.TimeLastRefresh)
 	// string "Source"
 	o = append(o, 0xa6, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65)
 	o = msgp.AppendString(o, z.Source)
@@ -417,13 +417,13 @@ func (z *lockRequesterInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				return
 			}
 		case "Timestamp":
-			z.Timestamp, bts, err = msgp.ReadTimeBytes(bts)
+			z.Timestamp, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Timestamp")
 				return
 			}
 		case "TimeLastRefresh":
-			z.TimeLastRefresh, bts, err = msgp.ReadTimeBytes(bts)
+			z.TimeLastRefresh, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "TimeLastRefresh")
 				return
@@ -466,7 +466,7 @@ func (z *lockRequesterInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *lockRequesterInfo) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 7 + msgp.BoolSize + 4 + msgp.StringPrefixSize + len(z.UID) + 10 + msgp.TimeSize + 16 + msgp.TimeSize + 7 + msgp.StringPrefixSize + len(z.Source) + 6 + msgp.BoolSize + 6 + msgp.StringPrefixSize + len(z.Owner) + 7 + msgp.IntSize
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 7 + msgp.BoolSize + 4 + msgp.StringPrefixSize + len(z.UID) + 10 + msgp.Int64Size + 16 + msgp.Int64Size + 7 + msgp.StringPrefixSize + len(z.Source) + 6 + msgp.BoolSize + 6 + msgp.StringPrefixSize + len(z.Owner) + 7 + msgp.IntSize
 	return
 }
 

cmd/local-locker_test.go

@@ -32,6 +32,7 @@ import (
 func TestLocalLockerExpire(t *testing.T) {
 	wResources := make([]string, 1000)
 	rResources := make([]string, 1000)
+	quorum := 0
 	l := newLocker()
 	ctx := context.Background()
 	for i := range wResources {
@@ -40,7 +41,7 @@ func TestLocalLockerExpire(t *testing.T) {
 			Resources: []string{mustGetUUID()},
 			Source:    t.Name(),
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.Lock(ctx, arg)
 		if err != nil {
@@ -58,14 +59,14 @@ func TestLocalLockerExpire(t *testing.T) {
 			Resources: []string{name},
 			Source:    t.Name(),
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.RLock(ctx, arg)
 		if err != nil {
 			t.Fatal(err)
 		}
 		if !ok {
-			t.Fatal("did not get write lock")
+			t.Fatal("did not get read lock")
 		}
 		// RLock twice
 		ok, err = l.RLock(ctx, arg)
@@ -112,6 +113,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 	rUIDs := make([]string, 0, n*2)
 	l := newLocker()
 	ctx := context.Background()
+	quorum := 0
 	for i := range wResources {
 		names := [m]string{}
 		for j := range names {
@@ -123,7 +125,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 			Resources: names[:],
 			Source:    t.Name(),
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.Lock(ctx, arg)
 		if err != nil {
@@ -144,7 +146,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 			Resources: []string{name},
 			Source:    t.Name(),
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.RLock(ctx, arg)
 		if err != nil {
@@ -183,7 +185,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 			Resources: []string{name},
 			Source:    t.Name(),
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.RUnlock(ctx, arg)
 		if err != nil {
@@ -203,6 +205,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 	if len(l.lockUID) != len(rResources)+len(wResources)*m {
 		t.Fatalf("lockUID len, got %d, want %d + %d", len(l.lockUID), len(rResources), len(wResources)*m)
 	}
+
 	// RUnlock again, different uids
 	for i, name := range rResources {
 		arg := dsync.LockArgs{
@@ -210,7 +213,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 			Resources: []string{name},
 			Source:    "minio",
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.RUnlock(ctx, arg)
 		if err != nil {
@@ -238,7 +241,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 			Resources: names[:],
 			Source:    "minio",
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.Unlock(ctx, arg)
 		if err != nil {
@@ -261,6 +264,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 
 func Test_localLocker_expireOldLocksExpire(t *testing.T) {
 	rng := rand.New(rand.NewSource(0))
+	quorum := 0
 	// Numbers of unique locks
 	for _, locks := range []int{100, 1000, 1e6} {
 		if testing.Short() && locks > 100 {
@@ -289,7 +293,7 @@ func Test_localLocker_expireOldLocksExpire(t *testing.T) {
 								Resources: res,
 								Source:    hex.EncodeToString(tmp[:8]),
 								Owner:     hex.EncodeToString(tmp[8:]),
-								Quorum:    0,
+								Quorum:    &quorum,
 							})
 							if !ok || err != nil {
 								t.Fatal("failed:", err, ok)
@@ -311,7 +315,7 @@ func Test_localLocker_expireOldLocksExpire(t *testing.T) {
 					for _, v := range l.lockMap {
 						for i := range v {
 							if rng.Intn(2) == 0 {
-								v[i].TimeLastRefresh = expired
+								v[i].TimeLastRefresh = expired.UnixNano()
 							}
 						}
 					}
@@ -347,6 +351,7 @@ func Test_localLocker_expireOldLocksExpire(t *testing.T) {
 
 func Test_localLocker_RUnlock(t *testing.T) {
 	rng := rand.New(rand.NewSource(0))
+	quorum := 0
 	// Numbers of unique locks
 	for _, locks := range []int{1, 100, 1000, 1e6} {
 		if testing.Short() && locks > 100 {
@@ -375,7 +380,7 @@ func Test_localLocker_RUnlock(t *testing.T) {
 								Resources: res,
 								Source:    hex.EncodeToString(tmp[:8]),
 								Owner:     hex.EncodeToString(tmp[8:]),
-								Quorum:    0,
+								Quorum:    &quorum,
 							})
 							if !ok || err != nil {
 								t.Fatal("failed:", err, ok)

cmd/lock-rest-client.go

@@ -107,5 +107,5 @@ func newLockAPI(endpoint Endpoint) dsync.NetLocker {
 
 // Returns a lock rest client.
 func newlockRESTClient(ep Endpoint) *lockRESTClient {
-	return &lockRESTClient{globalGrid.Load().Connection(ep.GridHost())}
+	return &lockRESTClient{globalLockGrid.Load().Connection(ep.GridHost())}
 }

cmd/lock-rest-client_test.go

@@ -39,7 +39,7 @@ func TestLockRESTlient(t *testing.T) {
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	err = initGlobalGrid(ctx, []PoolEndpoints{{Endpoints: Endpoints{endpoint, endpointLocal}}})
+	err = initGlobalLockGrid(ctx, []PoolEndpoints{{Endpoints: Endpoints{endpoint, endpointLocal}}})
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/lock-rest-server-common_test.go

@@ -44,7 +44,7 @@ func createLockTestServer(ctx context.Context, t *testing.T) (string, *lockRESTS
 		},
 	}
 	creds := globalActiveCred
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey, "")
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -63,15 +63,15 @@ func TestLockRpcServerRemoveEntry(t *testing.T) {
 		Owner:           "owner",
 		Writer:          true,
 		UID:             "0123-4567",
-		Timestamp:       UTCNow(),
-		TimeLastRefresh: UTCNow(),
+		Timestamp:       UTCNow().UnixNano(),
+		TimeLastRefresh: UTCNow().UnixNano(),
 	}
 	lockRequesterInfo2 := lockRequesterInfo{
 		Owner:           "owner",
 		Writer:          true,
 		UID:             "89ab-cdef",
-		Timestamp:       UTCNow(),
-		TimeLastRefresh: UTCNow(),
+		Timestamp:       UTCNow().UnixNano(),
+		TimeLastRefresh: UTCNow().UnixNano(),
 	}
 
 	locker.ll.lockMap["name"] = []lockRequesterInfo{

cmd/lock-rest-server.go

@@ -111,17 +111,17 @@ func newLockHandler(h grid.HandlerID) *grid.SingleHandler[*dsync.LockArgs, *dsyn
 }
 
 // registerLockRESTHandlers - register lock rest router.
-func registerLockRESTHandlers() {
+func registerLockRESTHandlers(gm *grid.Manager) {
 	lockServer := &lockRESTServer{
 		ll: newLocker(),
 	}
 
-	logger.FatalIf(lockRPCForceUnlock.Register(globalGrid.Load(), lockServer.ForceUnlockHandler), "unable to register handler")
-	logger.FatalIf(lockRPCRefresh.Register(globalGrid.Load(), lockServer.RefreshHandler), "unable to register handler")
-	logger.FatalIf(lockRPCLock.Register(globalGrid.Load(), lockServer.LockHandler), "unable to register handler")
-	logger.FatalIf(lockRPCUnlock.Register(globalGrid.Load(), lockServer.UnlockHandler), "unable to register handler")
-	logger.FatalIf(lockRPCRLock.Register(globalGrid.Load(), lockServer.RLockHandler), "unable to register handler")
-	logger.FatalIf(lockRPCRUnlock.Register(globalGrid.Load(), lockServer.RUnlockHandler), "unable to register handler")
+	logger.FatalIf(lockRPCForceUnlock.Register(gm, lockServer.ForceUnlockHandler), "unable to register handler")
+	logger.FatalIf(lockRPCRefresh.Register(gm, lockServer.RefreshHandler), "unable to register handler")
+	logger.FatalIf(lockRPCLock.Register(gm, lockServer.LockHandler), "unable to register handler")
+	logger.FatalIf(lockRPCUnlock.Register(gm, lockServer.UnlockHandler), "unable to register handler")
+	logger.FatalIf(lockRPCRLock.Register(gm, lockServer.RLockHandler), "unable to register handler")
+	logger.FatalIf(lockRPCRUnlock.Register(gm, lockServer.RUnlockHandler), "unable to register handler")
 
 	globalLockServer = lockServer.ll
 

cmd/logging.go

@@ -184,10 +184,6 @@ func etcdLogOnceIf(ctx context.Context, err error, id string, errKind ...interfa
 	logger.LogOnceIf(ctx, "etcd", err, id, errKind...)
 }
 
-func subnetLogIf(ctx context.Context, err error, errKind ...interface{}) {
-	logger.LogIf(ctx, "subnet", err, errKind...)
-}
-
 func metricsLogIf(ctx context.Context, err error, errKind ...interface{}) {
 	logger.LogIf(ctx, "metrics", err, errKind...)
 }

cmd/main.go

@@ -107,6 +107,11 @@ func newApp(name string) *cli.App {
 
 	// registerCommand registers a cli command.
 	registerCommand := func(command cli.Command) {
+		// avoid registering commands which are not being built (via
+		// go:build tags)
+		if command.Name == "" {
+			return
+		}
 		commands = append(commands, command)
 		commandsTree.Insert(command.Name)
 	}
@@ -134,6 +139,7 @@ func newApp(name string) *cli.App {
 
 	// Register all commands.
 	registerCommand(serverCmd)
+	registerCommand(fmtGenCmd)
 
 	// Set up app.
 	cli.HelpFlag = cli.BoolFlag{

cmd/metacache-entries.go

@@ -532,6 +532,9 @@ func (m *metaCacheEntriesSorted) fileInfoVersions(bucket, prefix, delimiter, aft
 			}
 
 			for _, version := range fiVersions {
+				if !version.VersionPurgeStatus().Empty() {
+					continue
+				}
 				versioned := vcfg != nil && vcfg.Versioned(entry.name)
 				versions = append(versions, version.ToObjectInfo(bucket, entry.name, versioned))
 			}
@@ -593,7 +596,7 @@ func (m *metaCacheEntriesSorted) fileInfos(bucket, prefix, delimiter string) (ob
 			}
 
 			fi, err := entry.fileInfo(bucket)
-			if err == nil {
+			if err == nil && fi.VersionPurgeStatus().Empty() {
 				versioned := vcfg != nil && vcfg.Versioned(entry.name)
 				objects = append(objects, fi.ToObjectInfo(bucket, entry.name, versioned))
 			}

cmd/metacache-server-pool.go

@@ -153,19 +153,7 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 			} else {
 				// Continue listing
 				o.ID = c.id
-				go func(meta metacache) {
-					// Continuously update while we wait.
-					t := time.NewTicker(metacacheMaxClientWait / 10)
-					defer t.Stop()
-					select {
-					case <-ctx.Done():
-						// Request is done, stop updating.
-						return
-					case <-t.C:
-						meta.lastHandout = time.Now()
-						meta, _ = rpc.UpdateMetacacheListing(ctx, meta)
-					}
-				}(*c)
+				go c.keepAlive(ctx, rpc)
 			}
 		}
 	}
@@ -219,6 +207,9 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 		o.ID = ""
 	}
 
+	if contextCanceled(ctx) {
+		return entries, ctx.Err()
+	}
 	// Do listing in-place.
 	// Create output for our results.
 	// Create filter for results.
@@ -449,5 +440,10 @@ func (z *erasureServerPools) listAndSave(ctx context.Context, o *listPathOptions
 		xioutil.SafeClose(saveCh)
 	}()
 
-	return filteredResults()
+	entries, err = filteredResults()
+	if err == nil {
+		// Check if listing recorded an error.
+		err = meta.getErr()
+	}
+	return entries, err
 }

cmd/metacache-set.go

@@ -805,6 +805,17 @@ func (m *metaCacheRPC) setErr(err string) {
 	*m.meta = meta
 }
 
+// getErr will return an error if the listing failed.
+// The error is not type safe.
+func (m *metaCacheRPC) getErr() error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.meta.status == scanStateError {
+		return errors.New(m.meta.error)
+	}
+	return nil
+}
+
 func (er *erasureObjects) saveMetaCacheStream(ctx context.Context, mc *metaCacheRPC, entries <-chan metaCacheEntry) (err error) {
 	o := mc.o
 	o.debugf(color.Green("saveMetaCacheStream:")+" with options: %#v", o)

cmd/metacache.go

@@ -24,6 +24,8 @@ import (
 	"path"
 	"strings"
 	"time"
+
+	"github.com/minio/pkg/v3/console"
 )
 
 type scanStatus uint8
@@ -97,6 +99,37 @@ func (m *metacache) worthKeeping() bool {
 	return true
 }
 
+// keepAlive will continuously update lastHandout until ctx is canceled.
+func (m metacache) keepAlive(ctx context.Context, rpc *peerRESTClient) {
+	// we intentionally operate on a copy of m, so we can update without locks.
+	t := time.NewTicker(metacacheMaxClientWait / 10)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			// Request is done, stop updating.
+			return
+		case <-t.C:
+			m.lastHandout = time.Now()
+
+			if m2, err := rpc.UpdateMetacacheListing(ctx, m); err == nil {
+				if m2.status != scanStateStarted {
+					if serverDebugLog {
+						console.Debugln("returning", m.id, "due to scan state", m2.status, time.Now().Format(time.RFC3339))
+					}
+					return
+				}
+				m = m2
+				if serverDebugLog {
+					console.Debugln("refreshed", m.id, time.Now().Format(time.RFC3339))
+				}
+			} else if serverDebugLog {
+				console.Debugln("error refreshing", m.id, time.Now().Format(time.RFC3339))
+			}
+		}
+	}
+}
+
 // baseDirFromPrefix will return the base directory given an object path.
 // For example an object with name prefix/folder/object.ext will return `prefix/folder/`.
 func baseDirFromPrefix(prefix string) string {
@@ -116,13 +149,17 @@ func baseDirFromPrefix(prefix string) string {
 // update cache with new status.
 // The updates are conditional so multiple callers can update with different states.
 func (m *metacache) update(update metacache) {
-	m.lastUpdate = UTCNow()
+	now := UTCNow()
+	m.lastUpdate = now
 
-	if m.lastHandout.After(m.lastHandout) {
-		m.lastHandout = UTCNow()
+	if update.lastHandout.After(m.lastHandout) {
+		m.lastHandout = update.lastUpdate
+		if m.lastHandout.After(now) {
+			m.lastHandout = now
+		}
 	}
 	if m.status == scanStateStarted && update.status == scanStateSuccess {
-		m.ended = UTCNow()
+		m.ended = now
 	}
 
 	if m.status == scanStateStarted && update.status != scanStateStarted {
@@ -138,7 +175,7 @@ func (m *metacache) update(update metacache) {
 	if m.error == "" && update.error != "" {
 		m.error = update.error
 		m.status = scanStateError
-		m.ended = UTCNow()
+		m.ended = now
 	}
 	m.fileNotFound = m.fileNotFound || update.fileNotFound
 }

cmd/metrics-resource.go

@@ -262,7 +262,7 @@ func collectDriveMetrics(m madmin.RealtimeMetrics) {
 	latestDriveStatsMu.Unlock()
 
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 
 	for _, d := range localDrives {

cmd/metrics-router.go

@@ -38,7 +38,8 @@ const (
 
 // Standard env prometheus auth type
 const (
-	EnvPrometheusAuthType = "MINIO_PROMETHEUS_AUTH_TYPE"
+	EnvPrometheusAuthType    = "MINIO_PROMETHEUS_AUTH_TYPE"
+	EnvPrometheusOpenMetrics = "MINIO_PROMETHEUS_OPEN_METRICS"
 )
 
 type prometheusAuthType string
@@ -58,14 +59,15 @@ func registerMetricsRouter(router *mux.Router) {
 	if authType == prometheusPublic {
 		auth = NoAuthMiddleware
 	}
+
 	metricsRouter.Handle(prometheusMetricsPathLegacy, auth(metricsHandler()))
 	metricsRouter.Handle(prometheusMetricsV2ClusterPath, auth(metricsServerHandler()))
 	metricsRouter.Handle(prometheusMetricsV2BucketPath, auth(metricsBucketHandler()))
 	metricsRouter.Handle(prometheusMetricsV2NodePath, auth(metricsNodeHandler()))
 	metricsRouter.Handle(prometheusMetricsV2ResourcePath, auth(metricsResourceHandler()))
 
-	// Metrics v3!
-	metricsV3Server := newMetricsV3Server(authType)
+	// Metrics v3
+	metricsV3Server := newMetricsV3Server(auth)
 
 	// Register metrics v3 handler. It also accepts an optional query
 	// parameter `?list` - see handler for details.

cmd/metrics-v3-api.go

@@ -19,6 +19,8 @@ package cmd
 
 import (
 	"context"
+
+	"github.com/minio/minio-go/v7/pkg/set"
 )
 
 const (
@@ -127,7 +129,8 @@ func loadAPIRequestsHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsC
 // This is a `MetricsLoaderFn`.
 func loadAPIRequestsTTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error {
 	renameLabels := map[string]string{"api": "name"}
-	m.SetHistogram(apiRequestsTTFBSecondsDistribution, httpRequestsDuration, renameLabels, nil,
+	labelsFilter := map[string]set.StringSet{"api": set.CreateStringSet("GetObject")}
+	m.SetHistogram(apiRequestsTTFBSecondsDistribution, httpRequestsDuration, labelsFilter, renameLabels, nil,
 		"type", "s3")
 	return nil
 }
@@ -214,7 +217,8 @@ func loadBucketAPIHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsCac
 // This is a `MetricsLoaderFn`.
 func loadBucketAPITTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache, buckets []string) error {
 	renameLabels := map[string]string{"api": "name"}
-	m.SetHistogram(apiRequestsTTFBSecondsDistribution, bucketHTTPRequestsDuration, renameLabels,
+	labelsFilter := map[string]set.StringSet{"api": set.CreateStringSet("GetObject")}
+	m.SetHistogram(apiRequestsTTFBSecondsDistribution, bucketHTTPRequestsDuration, labelsFilter, renameLabels,
 		buckets, "type", "s3")
 	return nil
 }

cmd/metrics-v3-cluster-usage.go

@@ -139,7 +139,7 @@ var (
 )
 
 // loadClusterUsageBucketMetrics - `MetricsLoaderFn` to load bucket usage metrics.
-func loadClusterUsageBucketMetrics(ctx context.Context, m MetricValues, c *metricsCache, buckets []string) error {
+func loadClusterUsageBucketMetrics(ctx context.Context, m MetricValues, c *metricsCache) error {
 	dataUsageInfo, err := c.dataUsageInfo.Get()
 	if err != nil {
 		metricsLogIf(ctx, err)
@@ -153,11 +153,7 @@ func loadClusterUsageBucketMetrics(ctx context.Context, m MetricValues, c *metri
 
 	m.Set(usageSinceLastUpdateSeconds, float64(time.Since(dataUsageInfo.LastUpdate)))
 
-	for _, bucket := range buckets {
-		usage, ok := dataUsageInfo.BucketsUsage[bucket]
-		if !ok {
-			continue
-		}
+	for bucket, usage := range dataUsageInfo.BucketsUsage {
 		quota, err := globalBucketQuotaSys.Get(ctx, bucket)
 		if err != nil {
 			// Log and continue if we are unable to retrieve metrics for this

cmd/metrics-v3-handler.go

@@ -23,9 +23,12 @@ import (
 	"net/http"
 	"slices"
 	"strings"
+	"sync"
 
+	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/mcontext"
 	"github.com/minio/mux"
+	"github.com/minio/pkg/v3/env"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
@@ -33,41 +36,39 @@ import (
 type promLogger struct{}
 
 func (p promLogger) Println(v ...interface{}) {
-	s := make([]string, 0, len(v))
-	for _, val := range v {
-		s = append(s, fmt.Sprintf("%v", val))
-	}
-	err := fmt.Errorf("metrics handler error: %v", strings.Join(s, " "))
-	metricsLogIf(GlobalContext, err)
+	metricsLogIf(GlobalContext, fmt.Errorf("metrics handler error: %v", v))
 }
 
 type metricsV3Server struct {
 	registry *prometheus.Registry
 	opts     promhttp.HandlerOpts
-	authFn   func(http.Handler) http.Handler
+	auth     func(http.Handler) http.Handler
 
 	metricsData *metricsV3Collection
 }
 
-func newMetricsV3Server(authType prometheusAuthType) *metricsV3Server {
+var (
+	globalMetricsV3CollectorPaths []collectorPath
+	globalMetricsV3Once           sync.Once
+)
+
+func newMetricsV3Server(auth func(h http.Handler) http.Handler) *metricsV3Server {
 	registry := prometheus.NewRegistry()
-	authFn := AuthMiddleware
-	if authType == prometheusPublic {
-		authFn = NoAuthMiddleware
-	}
-
 	metricGroups := newMetricGroups(registry)
-
+	globalMetricsV3Once.Do(func() {
+		globalMetricsV3CollectorPaths = metricGroups.collectorPaths
+	})
 	return &metricsV3Server{
 		registry: registry,
 		opts: promhttp.HandlerOpts{
 			ErrorLog:            promLogger{},
-			ErrorHandling:       promhttp.HTTPErrorOnError,
+			ErrorHandling:       promhttp.ContinueOnError,
 			Registry:            registry,
 			MaxRequestsInFlight: 2,
+			EnableOpenMetrics:   env.Get(EnvPrometheusOpenMetrics, config.EnableOff) == config.EnableOn,
+			ProcessStartTime:    globalBootTime,
 		},
-		authFn: authFn,
-
+		auth:        auth,
 		metricsData: metricGroups,
 	}
 }
@@ -163,7 +164,7 @@ func (h *metricsV3Server) handle(path string, isListingRequest bool, buckets []s
 		http.Error(w, "Metrics Resource Not found", http.StatusNotFound)
 	})
 
-	// Require that metrics path has at least component.
+	// Require that metrics path has one component at least.
 	if path == "/" {
 		return notFoundHandler
 	}
@@ -221,7 +222,7 @@ func (h *metricsV3Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	pathComponents := mux.Vars(r)["pathComps"]
 	isListingRequest := r.Form.Has("list")
 
-	buckets := []string{}
+	var buckets []string
 	if strings.HasPrefix(pathComponents, "/bucket/") {
 		// bucket specific metrics, extract the bucket name from the path.
 		// it's the last part of the path. e.g. /bucket/api/<bucket-name>
@@ -246,5 +247,5 @@ func (h *metricsV3Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	})
 
 	// Add authentication
-	h.authFn(tracedHandler).ServeHTTP(w, r)
+	h.auth(tracedHandler).ServeHTTP(w, r)
 }

cmd/metrics-v3-types.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/logger"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
@@ -35,8 +36,8 @@ type collectorPath string
 // converted to snake-case (by replaced '/' and '-' with '_') and prefixed with
 // `minio_`.
 func (cp collectorPath) metricPrefix() string {
-	s := strings.TrimPrefix(string(cp), "/")
-	s = strings.ReplaceAll(s, "/", "_")
+	s := strings.TrimPrefix(string(cp), SlashSeparator)
+	s = strings.ReplaceAll(s, SlashSeparator, "_")
 	s = strings.ReplaceAll(s, "-", "_")
 	return "minio_" + s
 }
@@ -56,8 +57,8 @@ func (cp collectorPath) isDescendantOf(arg string) bool {
 	if len(arg) >= len(descendant) {
 		return false
 	}
-	if !strings.HasSuffix(arg, "/") {
-		arg += "/"
+	if !strings.HasSuffix(arg, SlashSeparator) {
+		arg += SlashSeparator
 	}
 	return strings.HasPrefix(descendant, arg)
 }
@@ -72,10 +73,11 @@ const (
 	GaugeMT
 	// HistogramMT - represents a histogram metric.
 	HistogramMT
-	// rangeL - represents a range label.
-	rangeL = "range"
 )
 
+// rangeL - represents a range label.
+const rangeL = "range"
+
 func (mt MetricType) String() string {
 	switch mt {
 	case CounterMT:
@@ -246,28 +248,38 @@ func (m *MetricValues) Set(name MetricName, value float64, labels ...string) {
 // SetHistogram - sets values for the given MetricName using the provided
 // histogram.
 //
+// `filterByLabels` is a map of label names to list of allowed label values to
+// filter by. Note that this filtering happens before any renaming of labels.
+//
 // `renameLabels` is a map of label names to rename. The keys are the original
 // label names and the values are the new label names.
 //
-// TODO: bucketFilter doc
+// `bucketFilter` is a list of bucket values to filter. If this is non-empty,
+// only metrics for the given buckets are added.
 //
 // `extraLabels` are additional labels to add to each metric. They are ordered
 // label name and value pairs.
 func (m *MetricValues) SetHistogram(name MetricName, hist *prometheus.HistogramVec,
-	renameLabels map[string]string, bucketFilter []string, extraLabels ...string,
+	filterByLabels map[string]set.StringSet, renameLabels map[string]string, bucketFilter []string,
+	extraLabels ...string,
 ) {
 	if _, ok := m.descriptors[name]; !ok {
 		panic(fmt.Sprintf("metric has no description: %s", name))
 	}
 	dummyDesc := MetricDescription{}
 	metricsV2 := getHistogramMetrics(hist, dummyDesc, false)
+mainLoop:
 	for _, metric := range metricsV2 {
+		for label, allowedValues := range filterByLabels {
+			if !allowedValues.Contains(metric.VariableLabels[label]) {
+				continue mainLoop
+			}
+		}
+
 		// If a bucket filter is provided, only add metrics for the given
 		// buckets.
-		if len(bucketFilter) > 0 {
-			if !slices.Contains(bucketFilter, metric.VariableLabels["bucket"]) {
-				continue
-			}
+		if len(bucketFilter) > 0 && !slices.Contains(bucketFilter, metric.VariableLabels["bucket"]) {
+			continue
 		}
 
 		labels := make([]string, 0, len(metric.VariableLabels)*2)

cmd/metrics-v3.go

@@ -270,7 +270,7 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 		loadClusterUsageObjectMetrics,
 	)
 
-	clusterUsageBucketsMG := NewBucketMetricsGroup(clusterUsageBucketsCollectorPath,
+	clusterUsageBucketsMG := NewMetricsGroup(clusterUsageBucketsCollectorPath,
 		[]MetricDescriptor{
 			usageSinceLastUpdateSecondsMD,
 			usageBucketTotalBytesMD,

cmd/naughty-disk_test.go

@@ -222,6 +222,13 @@ func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string
 	return d.disk.CheckParts(ctx, volume, path, fi)
 }
 
+func (d *naughtyDisk) DeleteBulk(ctx context.Context, volume string, paths ...string) (err error) {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.DeleteBulk(ctx, volume, paths...)
+}
+
 func (d *naughtyDisk) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) {
 	if err := d.calcError(); err != nil {
 		return err

cmd/object-api-utils.go

@@ -34,6 +34,7 @@ import (
 	"sync"
 	"time"
 	"unicode/utf8"
+	"unsafe"
 
 	"github.com/google/uuid"
 	"github.com/klauspost/compress/s2"
@@ -246,6 +247,24 @@ func pathsJoinPrefix(prefix string, elem ...string) (paths []string) {
 	return paths
 }
 
+// string concat alternative to s1 + s2 with low overhead.
+func concat(ss ...string) string {
+	length := len(ss)
+	if length == 0 {
+		return ""
+	}
+	// create & allocate the memory in advance.
+	n := 0
+	for i := 0; i < length; i++ {
+		n += len(ss[i])
+	}
+	b := make([]byte, 0, n)
+	for i := 0; i < length; i++ {
+		b = append(b, ss[i]...)
+	}
+	return unsafe.String(unsafe.SliceData(b), n)
+}
+
 // pathJoin - like path.Join() but retains trailing SlashSeparator of the last element
 func pathJoin(elem ...string) string {
 	sb := bytebufferpool.Get()

cmd/object-api-utils_test.go

@@ -20,8 +20,10 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"encoding/hex"
 	"fmt"
 	"io"
+	"math/rand"
 	"net/http"
 	"net/http/httptest"
 	"path"
@@ -47,6 +49,43 @@ func pathJoinOld(elem ...string) string {
 	return path.Join(elem...) + trailingSlash
 }
 
+func concatNaive(ss ...string) string {
+	rs := ss[0]
+	for i := 1; i < len(ss); i++ {
+		rs += ss[i]
+	}
+	return rs
+}
+
+func benchmark(b *testing.B, data []string) {
+	b.Run("concat naive", func(b *testing.B) {
+		b.ResetTimer()
+		b.ReportAllocs()
+		for i := 0; i < b.N; i++ {
+			concatNaive(data...)
+		}
+	})
+	b.Run("concat fast", func(b *testing.B) {
+		b.ResetTimer()
+		b.ReportAllocs()
+		for i := 0; i < b.N; i++ {
+			concat(data...)
+		}
+	})
+}
+
+func BenchmarkConcatImplementation(b *testing.B) {
+	data := make([]string, 2)
+	rng := rand.New(rand.NewSource(0))
+	for i := 0; i < 2; i++ {
+		var tmp [16]byte
+		rng.Read(tmp[:])
+		data[i] = hex.EncodeToString(tmp[:])
+	}
+	b.ResetTimer()
+	benchmark(b, data)
+}
+
 func BenchmarkPathJoinOld(b *testing.B) {
 	b.Run("PathJoin", func(b *testing.B) {
 		b.ResetTimer()

cmd/object-handlers-common.go

@@ -248,10 +248,19 @@ func checkPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Requ
 	}
 
 	// Check if the part number is correct.
-	if opts.PartNumber > 1 && opts.PartNumber > len(objInfo.Parts) {
-		// According to S3 we don't need to set any object information here.
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidPartNumber), r.URL)
-		return true
+	if opts.PartNumber > 1 {
+		partFound := false
+		for _, pi := range objInfo.Parts {
+			if pi.Number == opts.PartNumber {
+				partFound = true
+				break
+			}
+		}
+		if !partFound {
+			// According to S3 we don't need to set any object information here.
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidPartNumber), r.URL)
+			return true
+		}
 	}
 
 	// If-None-Match : Return the object only if its entity tag (ETag) is different from the

cmd/object-handlers.go

@@ -501,8 +501,8 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 			reader *GetObjectReader
 			perr   error
 		)
-		// avoid proxying if version is a delete marker
-		if !isErrMethodNotAllowed(err) && !(gr != nil && gr.ObjInfo.DeleteMarker) {
+
+		if (isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrReadQuorum(err)) && !(gr != nil && gr.ObjInfo.DeleteMarker) {
 			proxytgts := getProxyTargets(ctx, bucket, object, opts)
 			if !proxytgts.Empty() {
 				globalReplicationStats.incProxy(bucket, getObjectAPI, false)
@@ -1028,7 +1028,7 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 
 	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
 	var proxy proxyResult
-	if err != nil && !objInfo.DeleteMarker && !isErrMethodNotAllowed(err) {
+	if err != nil && !objInfo.DeleteMarker && (isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrReadQuorum(err)) {
 		// proxy HEAD to replication target if active-active replication configured on bucket
 		proxytgts := getProxyTargets(ctx, bucket, object, opts)
 		if !proxytgts.Empty() {

cmd/object-handlers_test.go

@@ -692,8 +692,8 @@ func testAPIGetObjectWithMPHandler(obj ObjectLayer, instanceType, bucketName str
 		{"small-1", []int64{509}, make(map[string]string)},
 		{"small-2", []int64{5 * oneMiB}, make(map[string]string)},
 		// // // cases 4-7: multipart part objects
-		{"mp-0", []int64{5 * oneMiB, 1}, make(map[string]string)},
-		{"mp-1", []int64{5*oneMiB + 1, 1}, make(map[string]string)},
+		{"mp-0", []int64{5 * oneMiB, 10}, make(map[string]string)},
+		{"mp-1", []int64{5*oneMiB + 1, 10}, make(map[string]string)},
 		{"mp-2", []int64{5487701, 5487799, 3}, make(map[string]string)},
 		{"mp-3", []int64{10499807, 10499963, 7}, make(map[string]string)},
 		// cases 8-11: small single part objects with encryption
@@ -702,17 +702,13 @@ func testAPIGetObjectWithMPHandler(obj ObjectLayer, instanceType, bucketName str
 		{"enc-small-1", []int64{509}, mapCopy(metaWithSSEC)},
 		{"enc-small-2", []int64{5 * oneMiB}, mapCopy(metaWithSSEC)},
 		// cases 12-15: multipart part objects with encryption
-		{"enc-mp-0", []int64{5 * oneMiB, 1}, mapCopy(metaWithSSEC)},
-		{"enc-mp-1", []int64{5*oneMiB + 1, 1}, mapCopy(metaWithSSEC)},
+		{"enc-mp-0", []int64{5 * oneMiB, 10}, mapCopy(metaWithSSEC)},
+		{"enc-mp-1", []int64{5*oneMiB + 1, 10}, mapCopy(metaWithSSEC)},
 		{"enc-mp-2", []int64{5487701, 5487799, 3}, mapCopy(metaWithSSEC)},
 		{"enc-mp-3", []int64{10499807, 10499963, 7}, mapCopy(metaWithSSEC)},
 	}
-	// SSEC can't be used with compression
-	globalCompressConfigMu.Lock()
-	globalCompressEnabled := globalCompressConfig.Enabled
-	globalCompressConfigMu.Unlock()
-	if globalCompressEnabled {
-		objectInputs = objectInputs[0:8]
+	if testing.Short() {
+		objectInputs = append(objectInputs[0:5], objectInputs[8:11]...)
 	}
 	// iterate through the above set of inputs and upload the object.
 	for _, input := range objectInputs {
@@ -768,6 +764,7 @@ func testAPIGetObjectWithMPHandler(obj ObjectLayer, instanceType, bucketName str
 			readers = append(readers, NewDummyDataGen(p, cumulativeSum))
 			cumulativeSum += p
 		}
+
 		refReader := io.LimitReader(ioutilx.NewSkipReader(io.MultiReader(readers...), off), length)
 		if ok, msg := cmpReaders(refReader, rec.Body); !ok {
 			t.Fatalf("(%s) Object: %s Case %d ByteRange: %s --> data mismatch! (msg: %s)", instanceType, oi.objectName, i+1, byteRange, msg)

cmd/object-lambda-handlers.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"crypto/subtle"
+	"encoding/hex"
 	"io"
 	"net/http"
 	"net/url"
@@ -33,6 +34,7 @@ import (
 
 	"github.com/minio/minio/internal/auth"
 	levent "github.com/minio/minio/internal/config/lambda/event"
+	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 )
@@ -77,16 +79,13 @@ func getLambdaEventData(bucket, object string, cred auth.Credentials, r *http.Re
 		return levent.Event{}, err
 	}
 
-	token, err := authenticateNode(cred.AccessKey, cred.SecretKey, u.RawQuery)
-	if err != nil {
-		return levent.Event{}, err
-	}
+	ckSum := sha256.Sum256([]byte(cred.AccessKey + u.RawQuery))
 
 	eventData := levent.Event{
 		GetObjectContext: &levent.GetObjectContext{
 			InputS3URL:  u.String(),
 			OutputRoute: shortuuid.New(),
-			OutputToken: token,
+			OutputToken: hex.EncodeToString(ckSum[:]),
 		},
 		UserRequest: levent.UserRequest{
 			URL:     r.URL.String(),
@@ -199,7 +198,7 @@ func fwdStatusToAPIError(resp *http.Response) *APIError {
 	return nil
 }
 
-// GetObjectLamdbaHandler - GET Object with transformed data via lambda functions
+// GetObjectLambdaHandler - GET Object with transformed data via lambda functions
 // ----------
 // This implementation of the GET operation applies lambda functions and returns the
 // response generated via the lambda functions. To use this API, you must have READ access

cmd/object_api_suite_test.go

@@ -559,21 +559,17 @@ func execExtended(t *testing.T, fn func(t *testing.T, init func(), bucketOptions
 	t.Run("default", func(t *testing.T) {
 		fn(t, nil, MakeBucketOptions{})
 	})
-	t.Run("defaultVerioned", func(t *testing.T) {
+	t.Run("default+versioned", func(t *testing.T) {
 		fn(t, nil, MakeBucketOptions{VersioningEnabled: true})
 	})
 
-	if testing.Short() {
-		return
-	}
-
 	t.Run("compressed", func(t *testing.T) {
 		fn(t, func() {
 			resetCompressEncryption()
 			enableCompression(t, false, []string{"*"}, []string{"*"})
 		}, MakeBucketOptions{})
 	})
-	t.Run("compressedVerioned", func(t *testing.T) {
+	t.Run("compressed+versioned", func(t *testing.T) {
 		fn(t, func() {
 			resetCompressEncryption()
 			enableCompression(t, false, []string{"*"}, []string{"*"})
@@ -588,7 +584,7 @@ func execExtended(t *testing.T, fn func(t *testing.T, init func(), bucketOptions
 			enableEncryption(t)
 		}, MakeBucketOptions{})
 	})
-	t.Run("encryptedVerioned", func(t *testing.T) {
+	t.Run("encrypted+versioned", func(t *testing.T) {
 		fn(t, func() {
 			resetCompressEncryption()
 			enableEncryption(t)
@@ -603,7 +599,7 @@ func execExtended(t *testing.T, fn func(t *testing.T, init func(), bucketOptions
 			enableCompression(t, true, []string{"*"}, []string{"*"})
 		}, MakeBucketOptions{})
 	})
-	t.Run("compressed+encryptedVerioned", func(t *testing.T) {
+	t.Run("compressed+encrypted+versioned", func(t *testing.T) {
 		fn(t, func() {
 			resetCompressEncryption()
 			enableCompression(t, true, []string{"*"}, []string{"*"})

cmd/peer-rest-server.go

@@ -664,7 +664,7 @@ var errUnsupportedSignal = fmt.Errorf("unsupported signal")
 
 func waitingDrivesNode() map[string]madmin.DiskMetrics {
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 
 	errs := make([]error, len(localDrives))

cmd/peer-s3-server.go

@@ -34,7 +34,7 @@ const (
 
 func healBucketLocal(ctx context.Context, bucket string, opts madmin.HealOpts) (res madmin.HealResultItem, err error) {
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 
 	// Initialize sync waitgroup.
@@ -158,7 +158,7 @@ func healBucketLocal(ctx context.Context, bucket string, opts madmin.HealOpts) (
 
 func listBucketsLocal(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 
 	quorum := (len(localDrives) / 2)
@@ -204,15 +204,17 @@ func listBucketsLocal(ctx context.Context, opts BucketOptions) (buckets []Bucket
 	return buckets, nil
 }
 
-func cloneDrives(drives []StorageAPI) []StorageAPI {
-	newDrives := make([]StorageAPI, len(drives))
-	copy(newDrives, drives)
-	return newDrives
+func cloneDrives(drives map[string]StorageAPI) []StorageAPI {
+	copyDrives := make([]StorageAPI, 0, len(drives))
+	for _, drive := range drives {
+		copyDrives = append(copyDrives, drive)
+	}
+	return copyDrives
 }
 
 func getBucketInfoLocal(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) {
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 
 	g := errgroup.WithNErrs(len(localDrives)).WithConcurrency(32)
@@ -261,7 +263,7 @@ func getBucketInfoLocal(ctx context.Context, bucket string, opts BucketOptions)
 
 func deleteBucketLocal(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 
 	g := errgroup.WithNErrs(len(localDrives)).WithConcurrency(32)
@@ -299,7 +301,7 @@ func deleteBucketLocal(ctx context.Context, bucket string, opts DeleteBucketOpti
 
 func makeBucketLocal(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 
 	g := errgroup.WithNErrs(len(localDrives)).WithConcurrency(32)

cmd/postpolicyform.go

@@ -364,7 +364,7 @@ func checkPostPolicy(formValues http.Header, postPolicyForm PostPolicyForm) erro
 		for key := range checkHeader {
 			logKeys = append(logKeys, key)
 		}
-		return fmt.Errorf("Each form field that you specify in a form (except %s) must appear in the list of conditions.", strings.Join(logKeys, ", "))
+		return fmt.Errorf("Each form field that you specify in a form must appear in the list of policy conditions. %q not specified in the policy.", strings.Join(logKeys, ", "))
 	}
 
 	return nil

cmd/prepare-storage.go

@@ -166,13 +166,13 @@ func connectLoadInitFormats(verboseLogging bool, firstDisk bool, storageDisks []
 		if err != nil && !errors.Is(err, errXLBackend) && !errors.Is(err, errUnformattedDisk) {
 			if errors.Is(err, errDiskNotFound) && verboseLogging {
 				if globalEndpoints.NEndpoints() > 1 {
-					logger.Error("Unable to connect to %s: %v", endpoints[i], isServerResolvable(endpoints[i], time.Second))
+					logger.Info("Unable to connect to %s: %v, will be retried", endpoints[i], isServerResolvable(endpoints[i], time.Second))
 				} else {
 					logger.Fatal(err, "Unable to connect to %s: %v", endpoints[i], isServerResolvable(endpoints[i], time.Second))
 				}
 			} else {
 				if globalEndpoints.NEndpoints() > 1 {
-					logger.Error("Unable to use the drive %s: %v", endpoints[i], err)
+					logger.Info("Unable to use the drive %s: %v, will be retried", endpoints[i], err)
 				} else {
 					logger.Fatal(errInvalidArgument, "Unable to use the drive %s: %v", endpoints[i], err)
 				}

cmd/routers.go

@@ -26,20 +26,28 @@ import (
 
 // Composed function registering routers for only distributed Erasure setup.
 func registerDistErasureRouters(router *mux.Router, endpointServerPools EndpointServerPools) {
+	var (
+		lockGrid   = globalLockGrid.Load()
+		commonGrid = globalGrid.Load()
+	)
+
 	// Register storage REST router only if its a distributed setup.
-	registerStorageRESTHandlers(router, endpointServerPools, globalGrid.Load())
+	registerStorageRESTHandlers(router, endpointServerPools, commonGrid)
 
 	// Register peer REST router only if its a distributed setup.
-	registerPeerRESTHandlers(router, globalGrid.Load())
+	registerPeerRESTHandlers(router, commonGrid)
 
 	// Register bootstrap REST router for distributed setups.
-	registerBootstrapRESTHandlers(globalGrid.Load())
+	registerBootstrapRESTHandlers(commonGrid)
 
 	// Register distributed namespace lock routers.
-	registerLockRESTHandlers()
+	registerLockRESTHandlers(lockGrid)
+
+	// Add lock grid to router
+	router.Handle(grid.RouteLockPath, adminMiddleware(lockGrid.Handler(storageServerRequestValidate), noGZFlag, noObjLayerFlag))
 
 	// Add grid to router
-	router.Handle(grid.RoutePath, adminMiddleware(globalGrid.Load().Handler(), noGZFlag, noObjLayerFlag))
+	router.Handle(grid.RoutePath, adminMiddleware(commonGrid.Handler(storageServerRequestValidate), noGZFlag, noObjLayerFlag))
 }
 
 // List of some generic middlewares which are applied for all incoming requests.

cmd/server-main.go

@@ -841,13 +841,14 @@ func serverMain(ctx *cli.Context) {
 
 	// Verify kernel release and version.
 	if oldLinux() {
-		warnings = append(warnings, color.YellowBold("- Detected Linux kernel version older than 4.0.0 release, there are some known potential performance problems with this kernel version. MinIO recommends a minimum of 4.x.x linux kernel version for best performance"))
+		warnings = append(warnings, color.YellowBold("Detected Linux kernel version older than 4.0 release, there are some known potential performance problems with this kernel version. MinIO recommends a minimum of 4.x linux kernel version for best performance"))
 	}
 
 	maxProcs := runtime.GOMAXPROCS(0)
 	cpuProcs := runtime.NumCPU()
 	if maxProcs < cpuProcs {
-		warnings = append(warnings, color.YellowBold("- Detected GOMAXPROCS(%d) < NumCPU(%d), please make sure to provide all PROCS to MinIO for optimal performance", maxProcs, cpuProcs))
+		warnings = append(warnings, color.YellowBold("Detected GOMAXPROCS(%d) < NumCPU(%d), please make sure to provide all PROCS to MinIO for optimal performance",
+			maxProcs, cpuProcs))
 	}
 
 	// Initialize grid
@@ -855,6 +856,11 @@ func serverMain(ctx *cli.Context) {
 		logger.FatalIf(initGlobalGrid(GlobalContext, globalEndpoints), "Unable to configure server grid RPC services")
 	})
 
+	// Initialize lock grid
+	bootstrapTrace("initLockGrid", func() {
+		logger.FatalIf(initGlobalLockGrid(GlobalContext, globalEndpoints), "Unable to configure server lock grid RPC services")
+	})
+
 	// Configure server.
 	bootstrapTrace("configureServer", func() {
 		handler, err := configureServerHandler(globalEndpoints)
@@ -862,7 +868,8 @@ func serverMain(ctx *cli.Context) {
 			logger.Fatal(config.ErrUnexpectedError(err), "Unable to configure one of server's RPC services")
 		}
 		// Allow grid to start after registering all services.
-		xioutil.SafeClose(globalGridStart)
+		close(globalGridStart)
+		close(globalLockGridStart)
 
 		httpServer := xhttp.NewServer(getServerListenAddrs()).
 			UseHandler(setCriticalErrorHandler(corsHandler(handler))).
@@ -921,16 +928,18 @@ func serverMain(ctx *cli.Context) {
 	}
 
 	bootstrapTrace("waitForQuorum", func() {
-		result := newObject.Health(context.Background(), HealthOptions{})
+		result := newObject.Health(context.Background(), HealthOptions{NoLogging: true})
 		for !result.HealthyRead {
 			if debugNoExit {
-				logger.Info("Not waiting for quorum since we are debugging.. possible cause unhealthy sets (%s)", result)
+				logger.Info("Not waiting for quorum since we are debugging.. possible cause unhealthy sets")
+				logger.Info(result.String())
 				break
 			}
 			d := time.Duration(r.Float64() * float64(time.Second))
-			logger.Info("Waiting for quorum READ healthcheck to succeed.. possible cause unhealthy sets (%s), retrying in %s", result, d)
+			logger.Info("Waiting for quorum READ healthcheck to succeed retrying in %s.. possible cause unhealthy sets", d)
+			logger.Info(result.String())
 			time.Sleep(d)
-			result = newObject.Health(context.Background(), HealthOptions{})
+			result = newObject.Health(context.Background(), HealthOptions{NoLogging: true})
 		}
 	})
 
@@ -953,11 +962,11 @@ func serverMain(ctx *cli.Context) {
 		}
 
 		if !globalServerCtxt.StrictS3Compat {
-			warnings = append(warnings, color.YellowBold("- Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production"))
+			warnings = append(warnings, color.YellowBold("Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production"))
 		}
 	})
 	if globalActiveCred.Equal(auth.DefaultCredentials) {
-		msg := fmt.Sprintf("- Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables",
+		msg := fmt.Sprintf("Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables",
 			globalActiveCred)
 		warnings = append(warnings, color.YellowBold(msg))
 	}
@@ -1030,16 +1039,6 @@ func serverMain(ctx *cli.Context) {
 			globalTransitionState.Init(newObject)
 		})
 
-		// Initialize batch job pool.
-		bootstrapTrace("newBatchJobPool", func() {
-			globalBatchJobPool = newBatchJobPool(GlobalContext, newObject, 100)
-		})
-
-		// Initialize the license update job
-		bootstrapTrace("initLicenseUpdateJob", func() {
-			initLicenseUpdateJob(GlobalContext, newObject)
-		})
-
 		go func() {
 			// Initialize transition tier configuration manager
 			bootstrapTrace("globalTierConfigMgr.Init", func() {
@@ -1104,22 +1103,21 @@ func serverMain(ctx *cli.Context) {
 			})
 		}
 
+		// Initialize batch job pool.
+		bootstrapTrace("newBatchJobPool", func() {
+			globalBatchJobPool = newBatchJobPool(GlobalContext, newObject, 100)
+		})
+
 		// Prints the formatted startup message, if err is not nil then it prints additional information as well.
 		printStartupMessage(getAPIEndpoints(), err)
 
 		// Print a warning at the end of the startup banner so it is more noticeable
-		if newObject.BackendInfo().StandardSCParity == 0 {
-			warnings = append(warnings, color.YellowBold("- The standard parity is set to 0. This can lead to data loss."))
+		if newObject.BackendInfo().StandardSCParity == 0 && !globalIsErasureSD {
+			warnings = append(warnings, color.YellowBold("The standard parity is set to 0. This can lead to data loss."))
 		}
-		objAPI := newObjectLayerFn()
-		if objAPI != nil {
-			printStorageInfo(objAPI.StorageInfo(GlobalContext, true))
-		}
-		if len(warnings) > 0 {
-			logger.Info(color.Yellow("STARTUP WARNINGS:"))
-			for _, warn := range warnings {
-				logger.Info(warn)
-			}
+
+		for _, warn := range warnings {
+			logger.Warning(warn)
 		}
 	}()
 

cmd/server-rlimit.go

@@ -82,11 +82,7 @@ func setMaxResources(ctx serverCtxt) (err error) {
 	}
 
 	if ctx.MemLimit > 0 {
-		maxLimit = ctx.MemLimit
-	}
-
-	if maxLimit > 0 {
-		debug.SetMemoryLimit(int64(maxLimit))
+		debug.SetMemoryLimit(int64(ctx.MemLimit))
 	}
 
 	// Do not use RLIMIT_AS as that is not useful and at times on systems < 4Gi

cmd/server-startup-msg.go

@@ -23,7 +23,6 @@ import (
 	"net/url"
 	"strings"
 
-	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/logger"
 	xnet "github.com/minio/pkg/v3/net"
@@ -37,7 +36,11 @@ func getFormatStr(strLen int, padding int) string {
 
 // Prints the formatted startup message.
 func printStartupMessage(apiEndpoints []string, err error) {
-	logger.Info(color.Bold(MinioBannerName))
+	banner := strings.Repeat("-", len(MinioBannerName))
+	if globalIsDistErasure {
+		logger.Startup(color.Bold(banner))
+	}
+	logger.Startup(color.Bold(MinioBannerName))
 	if err != nil {
 		if globalConsoleSys != nil {
 			globalConsoleSys.Send(GlobalContext, fmt.Sprintf("Server startup failed with '%v', some features may be missing", err))
@@ -47,7 +50,7 @@ func printStartupMessage(apiEndpoints []string, err error) {
 	if !globalSubnetConfig.Registered() {
 		var builder strings.Builder
 		startupBanner(&builder)
-		logger.Info(builder.String())
+		logger.Startup(builder.String())
 	}
 
 	strippedAPIEndpoints := stripStandardPorts(apiEndpoints, globalMinioHost)
@@ -61,6 +64,9 @@ func printStartupMessage(apiEndpoints []string, err error) {
 
 	// Prints documentation message.
 	printObjectAPIMsg()
+	if globalIsDistErasure {
+		logger.Startup(color.Bold(banner))
+	}
 }
 
 // Returns true if input is IPv6
@@ -113,21 +119,21 @@ func printServerCommonMsg(apiEndpoints []string) {
 
 	apiEndpointStr := strings.TrimSpace(strings.Join(apiEndpoints, "  "))
 	// Colorize the message and print.
-	logger.Info(color.Blue("API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
+	logger.Startup(color.Blue("API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
 	if color.IsTerminal() && (!globalServerCtxt.Anonymous && !globalServerCtxt.JSON && globalAPIConfig.permitRootAccess()) {
-		logger.Info(color.Blue("   RootUser: ") + color.Bold("%s ", cred.AccessKey))
-		logger.Info(color.Blue("   RootPass: ") + color.Bold("%s \n", cred.SecretKey))
+		logger.Startup(color.Blue("   RootUser: ") + color.Bold("%s ", cred.AccessKey))
+		logger.Startup(color.Blue("   RootPass: ") + color.Bold("%s \n", cred.SecretKey))
 		if region != "" {
-			logger.Info(color.Blue("   Region: ") + color.Bold("%s", fmt.Sprintf(getFormatStr(len(region), 2), region)))
+			logger.Startup(color.Blue("   Region: ") + color.Bold("%s", fmt.Sprintf(getFormatStr(len(region), 2), region)))
 		}
 	}
 
 	if globalBrowserEnabled {
 		consoleEndpointStr := strings.Join(stripStandardPorts(getConsoleEndpoints(), globalMinioConsoleHost), " ")
-		logger.Info(color.Blue("WebUI: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
+		logger.Startup(color.Blue("WebUI: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
 		if color.IsTerminal() && (!globalServerCtxt.Anonymous && !globalServerCtxt.JSON && globalAPIConfig.permitRootAccess()) {
-			logger.Info(color.Blue("   RootUser: ") + color.Bold("%s ", cred.AccessKey))
-			logger.Info(color.Blue("   RootPass: ") + color.Bold("%s ", cred.SecretKey))
+			logger.Startup(color.Blue("   RootUser: ") + color.Bold("%s ", cred.AccessKey))
+			logger.Startup(color.Blue("   RootPass: ") + color.Bold("%s ", cred.SecretKey))
 		}
 	}
 
@@ -137,7 +143,7 @@ func printServerCommonMsg(apiEndpoints []string) {
 
 // Prints startup message for Object API access, prints link to our SDK documentation.
 func printObjectAPIMsg() {
-	logger.Info(color.Blue("\nDocs: ") + "https://min.io/docs/minio/linux/index.html")
+	logger.Startup(color.Blue("\nDocs: ") + "https://min.io/docs/minio/linux/index.html")
 }
 
 func printLambdaTargets() {
@@ -149,7 +155,7 @@ func printLambdaTargets() {
 	for _, arn := range globalLambdaTargetList.List(globalSite.Region()) {
 		arnMsg += color.Bold(fmt.Sprintf("%s ", arn))
 	}
-	logger.Info(arnMsg + "\n")
+	logger.Startup(arnMsg + "\n")
 }
 
 // Prints bucket notification configurations.
@@ -168,7 +174,7 @@ func printEventNotifiers() {
 		arnMsg += color.Bold(fmt.Sprintf("%s ", arn))
 	}
 
-	logger.Info(arnMsg + "\n")
+	logger.Startup(arnMsg + "\n")
 }
 
 // Prints startup message for command line access. Prints link to our documentation
@@ -181,35 +187,9 @@ func printCLIAccessMsg(endPoint string, alias string) {
 
 	// Configure 'mc', following block prints platform specific information for minio client.
 	if color.IsTerminal() && (!globalServerCtxt.Anonymous && globalAPIConfig.permitRootAccess()) {
-		logger.Info(color.Blue("\nCLI: ") + mcQuickStartGuide)
+		logger.Startup(color.Blue("\nCLI: ") + mcQuickStartGuide)
 		mcMessage := fmt.Sprintf("$ mc alias set '%s' '%s' '%s' '%s'", alias,
 			endPoint, cred.AccessKey, cred.SecretKey)
-		logger.Info(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
-	}
-}
-
-// Get formatted disk/storage info message.
-func getStorageInfoMsg(storageInfo StorageInfo) string {
-	var msg string
-	var mcMessage string
-	onlineDisks, offlineDisks := getOnlineOfflineDisksStats(storageInfo.Disks)
-	if storageInfo.Backend.Type == madmin.Erasure {
-		if offlineDisks.Sum() > 0 {
-			mcMessage = "Use `mc admin info` to look for latest server/drive info\n"
-		}
-
-		diskInfo := fmt.Sprintf(" %d Online, %d Offline. ", onlineDisks.Sum(), offlineDisks.Sum())
-		msg += color.Blue("Status:") + fmt.Sprintf(getFormatStr(len(diskInfo), 8), diskInfo)
-		if len(mcMessage) > 0 {
-			msg = fmt.Sprintf("%s %s", mcMessage, msg)
-		}
-	}
-	return msg
-}
-
-// Prints startup message of storage capacity and erasure information.
-func printStorageInfo(storageInfo StorageInfo) {
-	if msg := getStorageInfoMsg(storageInfo); msg != "" {
-		logger.Info(msg)
+		logger.Startup(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
 	}
 }

cmd/server-startup-msg_test.go

@@ -21,32 +21,9 @@ import (
 	"context"
 	"os"
 	"reflect"
-	"strings"
 	"testing"
-
-	"github.com/minio/madmin-go/v3"
 )
 
-// Tests if we generate storage info.
-func TestStorageInfoMsg(t *testing.T) {
-	infoStorage := StorageInfo{}
-	infoStorage.Disks = []madmin.Disk{
-		{Endpoint: "http://127.0.0.1:9000/data/1/", State: madmin.DriveStateOk},
-		{Endpoint: "http://127.0.0.1:9000/data/2/", State: madmin.DriveStateOk},
-		{Endpoint: "http://127.0.0.1:9000/data/3/", State: madmin.DriveStateOk},
-		{Endpoint: "http://127.0.0.1:9000/data/4/", State: madmin.DriveStateOk},
-		{Endpoint: "http://127.0.0.1:9001/data/1/", State: madmin.DriveStateOk},
-		{Endpoint: "http://127.0.0.1:9001/data/2/", State: madmin.DriveStateOk},
-		{Endpoint: "http://127.0.0.1:9001/data/3/", State: madmin.DriveStateOk},
-		{Endpoint: "http://127.0.0.1:9001/data/4/", State: madmin.DriveStateOffline},
-	}
-	infoStorage.Backend.Type = madmin.Erasure
-
-	if msg := getStorageInfoMsg(infoStorage); !strings.Contains(msg, "7 Online, 1 Offline") {
-		t.Fatal("Unexpected storage info message, found:", msg)
-	}
-}
-
 // Tests stripping standard ports from apiEndpoints.
 func TestStripStandardPorts(t *testing.T) {
 	apiEndpoints := []string{"http://127.0.0.1:9000", "http://127.0.0.2:80", "https://127.0.0.3:443"}

cmd/server_test.go

@@ -35,6 +35,7 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
+	jwtgo "github.com/golang-jwt/jwt/v4"
 	"github.com/minio/minio-go/v7/pkg/set"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/pkg/v3/policy"
@@ -122,6 +123,9 @@ func runAllTests(suite *TestSuiteCommon, c *check) {
 	suite.TestObjectMultipartListError(c)
 	suite.TestObjectValidMD5(c)
 	suite.TestObjectMultipart(c)
+	suite.TestMetricsV3Handler(c)
+	suite.TestBucketSQSNotificationWebHook(c)
+	suite.TestBucketSQSNotificationAMQP(c)
 	suite.TearDownSuite(c)
 }
 
@@ -189,6 +193,36 @@ func (s *TestSuiteCommon) TearDownSuite(c *check) {
 	s.testServer.Stop()
 }
 
+const (
+	defaultPrometheusJWTExpiry = 100 * 365 * 24 * time.Hour
+)
+
+func (s *TestSuiteCommon) TestMetricsV3Handler(c *check) {
+	jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.StandardClaims{
+		ExpiresAt: time.Now().UTC().Add(defaultPrometheusJWTExpiry).Unix(),
+		Subject:   s.accessKey,
+		Issuer:    "prometheus",
+	})
+
+	token, err := jwt.SignedString([]byte(s.secretKey))
+	c.Assert(err, nil)
+
+	for _, cpath := range globalMetricsV3CollectorPaths {
+		request, err := newTestSignedRequest(http.MethodGet, s.endPoint+minioReservedBucketPath+metricsV3Path+string(cpath),
+			0, nil, s.accessKey, s.secretKey, s.signer)
+		c.Assert(err, nil)
+
+		request.Header.Set("Authorization", "Bearer "+token)
+
+		// execute the request.
+		response, err := s.client.Do(request)
+		c.Assert(err, nil)
+
+		// assert the http response status code.
+		c.Assert(response.StatusCode, http.StatusOK)
+	}
+}
+
 func (s *TestSuiteCommon) TestBucketSQSNotificationWebHook(c *check) {
 	// Sample bucket notification.
 	bucketNotificationBuf := `<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Queue>arn:minio:sqs:us-east-1:444455556666:webhook</Queue></QueueConfiguration></NotificationConfiguration>`

cmd/sftp-server.go

@@ -161,11 +161,13 @@ internalAuth:
 		return nil, errNoSuchUser
 	}
 
-	if caPublicKey != nil {
+	if caPublicKey != nil && pass == nil {
+
 		err := validateKey(c, key)
 		if err != nil {
 			return nil, errAuthentication
 		}
+
 	} else {
 
 		// Temporary credentials are not allowed.

cmd/site-replication.go

@@ -1129,7 +1129,7 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 		if err != nil {
 			return err
 		}
-		sameTarget, apiErr := validateReplicationDestination(ctx, bucket, newReplicationConfig, true)
+		sameTarget, apiErr := validateReplicationDestination(ctx, bucket, newReplicationConfig, &validateReplicationDestinationOptions{CheckRemoteBucket: true})
 		if apiErr != noError {
 			return fmt.Errorf("bucket replication config validation error: %#v", apiErr)
 		}
@@ -2250,10 +2250,18 @@ func (c *SiteReplicationSys) toErrorFromErrMap(errMap map[string]error, actionNa
 		return nil
 	}
 
+	// Get ordered list of keys of errMap
+	keys := []string{}
+	for d := range errMap {
+		keys = append(keys, d)
+	}
+	sort.Strings(keys)
+
 	var success int
 	msgs := []string{}
-	for d, err := range errMap {
+	for _, d := range keys {
 		name := c.state.Peers[d].Name
+		err := errMap[d]
 		if err == nil {
 			msgs = append(msgs, fmt.Sprintf("'%s' on site %s (%s): succeeded", actionName, name, d))
 			success++
@@ -2261,7 +2269,7 @@ func (c *SiteReplicationSys) toErrorFromErrMap(errMap map[string]error, actionNa
 			msgs = append(msgs, fmt.Sprintf("'%s' on site %s (%s): failed(%v)", actionName, name, d, err))
 		}
 	}
-	if success == len(errMap) {
+	if success == len(keys) {
 		return nil
 	}
 	return fmt.Errorf("Site replication error(s): \n%s", strings.Join(msgs, "\n"))
@@ -4445,6 +4453,7 @@ func (c *SiteReplicationSys) healBuckets(ctx context.Context, objAPI ObjectLayer
 		return err
 	}
 	ilmExpiryCfgHealed := false
+	opts := validateReplicationDestinationOptions{CheckReady: true}
 	for _, bi := range buckets {
 		bucket := bi.Name
 		info, err := c.siteReplicationStatus(ctx, objAPI, madmin.SRStatusOptions{
@@ -4464,7 +4473,7 @@ func (c *SiteReplicationSys) healBuckets(ctx context.Context, objAPI ObjectLayer
 			c.healVersioningMetadata(ctx, objAPI, bucket, info)
 			c.healOLockConfigMetadata(ctx, objAPI, bucket, info)
 			c.healSSEMetadata(ctx, objAPI, bucket, info)
-			c.healBucketReplicationConfig(ctx, objAPI, bucket, info)
+			c.healBucketReplicationConfig(ctx, objAPI, bucket, info, &opts)
 			c.healBucketPolicies(ctx, objAPI, bucket, info)
 			c.healTagMetadata(ctx, objAPI, bucket, info)
 			c.healBucketQuotaConfig(ctx, objAPI, bucket, info)
@@ -5164,7 +5173,7 @@ func (c *SiteReplicationSys) healBucket(ctx context.Context, objAPI ObjectLayer,
 	return nil
 }
 
-func (c *SiteReplicationSys) healBucketReplicationConfig(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error {
+func (c *SiteReplicationSys) healBucketReplicationConfig(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo, opts *validateReplicationDestinationOptions) error {
 	bs := info.BucketStats[bucket]
 
 	c.RLock()
@@ -5218,14 +5227,14 @@ func (c *SiteReplicationSys) healBucketReplicationConfig(ctx context.Context, ob
 
 	if rcfg != nil && !replMismatch {
 		// validate remote targets on current cluster for this bucket
-		_, apiErr := validateReplicationDestination(ctx, bucket, rcfg, false)
+		_, apiErr := validateReplicationDestination(ctx, bucket, rcfg, opts)
 		if apiErr != noError {
 			replMismatch = true
 		}
 	}
 
 	if replMismatch {
-		replLogIf(ctx, c.annotateErr(configureReplication, c.PeerBucketConfigureReplHandler(ctx, bucket)))
+		replLogOnceIf(ctx, c.annotateErr(configureReplication, c.PeerBucketConfigureReplHandler(ctx, bucket)), "heal-bucket-relication-config")
 	}
 	return nil
 }
@@ -5318,7 +5327,10 @@ func (c *SiteReplicationSys) healPolicies(ctx context.Context, objAPI ObjectLaye
 			UpdatedAt: lastUpdate,
 		})
 		if err != nil {
-			replLogIf(ctx, fmt.Errorf("Unable to heal IAM policy %s from peer site %s -> site %s : %w", policy, latestPeerName, peerName, err))
+			replLogOnceIf(
+				ctx,
+				fmt.Errorf("Unable to heal IAM policy %s from peer site %s -> site %s : %w", policy, latestPeerName, peerName, err),
+				fmt.Sprintf("heal-policy-%s", policy))
 		}
 	}
 	return nil
@@ -5379,7 +5391,8 @@ func (c *SiteReplicationSys) healUserPolicies(ctx context.Context, objAPI Object
 			UpdatedAt: lastUpdate,
 		})
 		if err != nil {
-			replLogIf(ctx, fmt.Errorf("Unable to heal IAM user policy mapping for %s from peer site %s -> site %s : %w", user, latestPeerName, peerName, err))
+			replLogOnceIf(ctx, fmt.Errorf("Unable to heal IAM user policy mapping from peer site %s -> site %s : %w", latestPeerName, peerName, err),
+				fmt.Sprintf("heal-user-policy-%s", user))
 		}
 	}
 	return nil
@@ -5442,7 +5455,9 @@ func (c *SiteReplicationSys) healGroupPolicies(ctx context.Context, objAPI Objec
 			UpdatedAt: lastUpdate,
 		})
 		if err != nil {
-			replLogIf(ctx, fmt.Errorf("Unable to heal IAM group policy mapping for %s from peer site %s -> site %s : %w", group, latestPeerName, peerName, err))
+			replLogOnceIf(ctx,
+				fmt.Errorf("Unable to heal IAM group policy mapping for from peer site %s -> site %s : %w", latestPeerName, peerName, err),
+				fmt.Sprintf("heal-group-policy-%s", group))
 		}
 	}
 	return nil
@@ -5503,13 +5518,17 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
 		if creds.IsServiceAccount() {
 			claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, creds.AccessKey)
 			if err != nil {
-				replLogIf(ctx, fmt.Errorf("Unable to heal service account %s from peer site %s -> %s : %w", user, latestPeerName, peerName, err))
+				replLogOnceIf(ctx,
+					fmt.Errorf("Unable to heal service account from peer site %s -> %s : %w", latestPeerName, peerName, err),
+					fmt.Sprintf("heal-user-%s", user))
 				continue
 			}
 
 			_, policy, err := globalIAMSys.GetServiceAccount(ctx, creds.AccessKey)
 			if err != nil {
-				replLogIf(ctx, fmt.Errorf("Unable to heal service account %s from peer site %s -> %s : %w", user, latestPeerName, peerName, err))
+				replLogOnceIf(ctx,
+					fmt.Errorf("Unable to heal service account from peer site %s -> %s : %w", latestPeerName, peerName, err),
+					fmt.Sprintf("heal-user-%s", user))
 				continue
 			}
 
@@ -5517,7 +5536,9 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
 			if policy != nil {
 				policyJSON, err = json.Marshal(policy)
 				if err != nil {
-					replLogIf(ctx, fmt.Errorf("Unable to heal service account %s from peer site %s -> %s : %w", user, latestPeerName, peerName, err))
+					replLogOnceIf(ctx,
+						fmt.Errorf("Unable to heal service account from peer site %s -> %s : %w", latestPeerName, peerName, err),
+						fmt.Sprintf("heal-user-%s", user))
 					continue
 				}
 			}
@@ -5540,7 +5561,9 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
 				},
 				UpdatedAt: lastUpdate,
 			}); err != nil {
-				replLogIf(ctx, fmt.Errorf("Unable to heal service account %s from peer site %s -> %s : %w", user, latestPeerName, peerName, err))
+				replLogOnceIf(ctx,
+					fmt.Errorf("Unable to heal service account from peer site %s -> %s : %w", latestPeerName, peerName, err),
+					fmt.Sprintf("heal-user-%s", user))
 			}
 			continue
 		}
@@ -5553,7 +5576,9 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
 				// policy. The session token will contain info about policy to
 				// be applied.
 				if !errors.Is(err, errNoSuchUser) {
-					replLogIf(ctx, fmt.Errorf("Unable to heal temporary credentials %s from peer site %s -> %s : %w", user, latestPeerName, peerName, err))
+					replLogOnceIf(ctx,
+						fmt.Errorf("Unable to heal temporary credentials from peer site %s -> %s : %w", latestPeerName, peerName, err),
+						fmt.Sprintf("heal-user-%s", user))
 					continue
 				}
 			} else {
@@ -5571,7 +5596,9 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
 				},
 				UpdatedAt: lastUpdate,
 			}); err != nil {
-				replLogIf(ctx, fmt.Errorf("Unable to heal temporary credentials %s from peer site %s -> %s : %w", user, latestPeerName, peerName, err))
+				replLogOnceIf(ctx,
+					fmt.Errorf("Unable to heal temporary credentials from peer site %s -> %s : %w", latestPeerName, peerName, err),
+					fmt.Sprintf("heal-user-%s", user))
 			}
 			continue
 		}
@@ -5587,7 +5614,9 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
 			},
 			UpdatedAt: lastUpdate,
 		}); err != nil {
-			replLogIf(ctx, fmt.Errorf("Unable to heal user %s from peer site %s -> %s : %w", user, latestPeerName, peerName, err))
+			replLogOnceIf(ctx,
+				fmt.Errorf("Unable to heal user from peer site %s -> %s : %w", latestPeerName, peerName, err),
+				fmt.Sprintf("heal-user-%s", user))
 		}
 	}
 	return nil
@@ -5651,7 +5680,9 @@ func (c *SiteReplicationSys) healGroups(ctx context.Context, objAPI ObjectLayer,
 			},
 			UpdatedAt: lastUpdate,
 		}); err != nil {
-			replLogIf(ctx, fmt.Errorf("Unable to heal group %s from peer site %s -> site %s : %w", group, latestPeerName, peerName, err))
+			replLogOnceIf(ctx,
+				fmt.Errorf("Unable to heal group from peer site %s -> site %s : %w", latestPeerName, peerName, err),
+				fmt.Sprintf("heal-group-%s", group))
 		}
 	}
 	return nil

cmd/storage-datatypes.go

@@ -264,6 +264,24 @@ type FileInfo struct {
 	Versioned bool `msg:"vs"`
 }
 
+func (fi FileInfo) shardSize() int64 {
+	return ceilFrac(fi.Erasure.BlockSize, int64(fi.Erasure.DataBlocks))
+}
+
+// ShardFileSize - returns final erasure size from original size.
+func (fi FileInfo) ShardFileSize(totalLength int64) int64 {
+	if totalLength == 0 {
+		return 0
+	}
+	if totalLength == -1 {
+		return -1
+	}
+	numShards := totalLength / fi.Erasure.BlockSize
+	lastBlockSize := totalLength % fi.Erasure.BlockSize
+	lastShardSize := ceilFrac(lastBlockSize, int64(fi.Erasure.DataBlocks))
+	return numShards*fi.shardSize() + lastShardSize
+}
+
 // ShallowCopy - copies minimal information for READ MRF checks.
 func (fi FileInfo) ShallowCopy() (n FileInfo) {
 	n.Volume = fi.Volume
@@ -374,24 +392,24 @@ func newFileInfo(object string, dataBlocks, parityBlocks int) (fi FileInfo) {
 
 // ReadMultipleReq contains information of multiple files to read from disk.
 type ReadMultipleReq struct {
-	Bucket       string   // Bucket. Can be empty if multiple buckets.
-	Prefix       string   // Shared prefix of all files. Can be empty. Will be joined to filename without modification.
-	Files        []string // Individual files to read.
-	MaxSize      int64    // Return error if size is exceed.
-	MetadataOnly bool     // Read as XL meta and truncate data.
-	AbortOn404   bool     // Stop reading after first file not found.
-	MaxResults   int      // Stop after this many successful results. <= 0 means all.
+	Bucket       string   `msg:"bk"`           // Bucket. Can be empty if multiple buckets.
+	Prefix       string   `msg:"pr,omitempty"` // Shared prefix of all files. Can be empty. Will be joined to filename without modification.
+	Files        []string `msg:"fl"`           // Individual files to read.
+	MaxSize      int64    `msg:"ms"`           // Return error if size is exceed.
+	MetadataOnly bool     `msg:"mo"`           // Read as XL meta and truncate data.
+	AbortOn404   bool     `msg:"ab"`           // Stop reading after first file not found.
+	MaxResults   int      `msg:"mr"`           // Stop after this many successful results. <= 0 means all.
 }
 
 // ReadMultipleResp contains a single response from a ReadMultipleReq.
 type ReadMultipleResp struct {
-	Bucket  string    // Bucket as given by request.
-	Prefix  string    // Prefix as given by request.
-	File    string    // File name as given in request.
-	Exists  bool      // Returns whether the file existed on disk.
-	Error   string    // Returns any error when reading.
-	Data    []byte    // Contains all data of file.
-	Modtime time.Time // Modtime of file on disk.
+	Bucket  string    `msg:"bk"`           // Bucket as given by request.
+	Prefix  string    `msg:"pr,omitempty"` // Prefix as given by request.
+	File    string    `msg:"fl"`           // File name as given in request.
+	Exists  bool      `msg:"ex"`           // Returns whether the file existed on disk.
+	Error   string    `msg:"er,omitempty"` // Returns any error when reading.
+	Data    []byte    `msg:"d"`            // Contains all data of file.
+	Modtime time.Time `msg:"m"`            // Modtime of file on disk.
 }
 
 // DeleteVersionHandlerParams are parameters for DeleteVersionHandler
@@ -498,8 +516,8 @@ type WriteAllHandlerParams struct {
 //     only after as a 2-phase call, allowing the older dataDir to
 //     hang-around in-case we need some form of recovery.
 type RenameDataResp struct {
-	Sign       []byte
-	OldDataDir string // contains '<uuid>', it is designed to be passed as value to Delete(bucket, pathJoin(object, dataDir))
+	Sign       []byte `msg:"s"`
+	OldDataDir string `msg:"od"` // contains '<uuid>', it is designed to be passed as value to Delete(bucket, pathJoin(object, dataDir))
 }
 
 const (
@@ -516,15 +534,26 @@ const (
 
 // CheckPartsResp is a response of the storage CheckParts and VerifyFile APIs
 type CheckPartsResp struct {
-	Results []int
+	Results []int `msg:"r"`
 }
 
 // LocalDiskIDs - GetLocalIDs response.
 type LocalDiskIDs struct {
-	IDs []string
+	IDs []string `msg:"i"`
 }
 
 // ListDirResult - ListDir()'s response.
 type ListDirResult struct {
 	Entries []string `msg:"e"`
 }
+
+// DeleteBulkReq - send multiple paths in same delete request.
+type DeleteBulkReq struct {
+	Paths []string `msg:"p"`
+}
+
+// DeleteVersionsErrsResp - collection of delete errors
+// for bulk version deletes
+type DeleteVersionsErrsResp struct {
+	Errs []string `msg:"e"`
+}

cmd/storage-datatypes_gen_test.go

@@ -348,6 +348,119 @@ func BenchmarkDecodeCheckPartsResp(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalDeleteBulkReq(t *testing.T) {
+	v := DeleteBulkReq{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgDeleteBulkReq(b *testing.B) {
+	v := DeleteBulkReq{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgDeleteBulkReq(b *testing.B) {
+	v := DeleteBulkReq{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalDeleteBulkReq(b *testing.B) {
+	v := DeleteBulkReq{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeDeleteBulkReq(t *testing.T) {
+	v := DeleteBulkReq{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeDeleteBulkReq Msgsize() is inaccurate")
+	}
+
+	vn := DeleteBulkReq{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeDeleteBulkReq(b *testing.B) {
+	v := DeleteBulkReq{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeDeleteBulkReq(b *testing.B) {
+	v := DeleteBulkReq{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalDeleteFileHandlerParams(t *testing.T) {
 	v := DeleteFileHandlerParams{}
 	bts, err := v.MarshalMsg(nil)
@@ -687,6 +800,119 @@ func BenchmarkDecodeDeleteVersionHandlerParams(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalDeleteVersionsErrsResp(t *testing.T) {
+	v := DeleteVersionsErrsResp{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgDeleteVersionsErrsResp(b *testing.B) {
+	v := DeleteVersionsErrsResp{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgDeleteVersionsErrsResp(b *testing.B) {
+	v := DeleteVersionsErrsResp{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalDeleteVersionsErrsResp(b *testing.B) {
+	v := DeleteVersionsErrsResp{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeDeleteVersionsErrsResp(t *testing.T) {
+	v := DeleteVersionsErrsResp{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeDeleteVersionsErrsResp Msgsize() is inaccurate")
+	}
+
+	vn := DeleteVersionsErrsResp{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeDeleteVersionsErrsResp(b *testing.B) {
+	v := DeleteVersionsErrsResp{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeDeleteVersionsErrsResp(b *testing.B) {
+	v := DeleteVersionsErrsResp{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalDiskInfo(t *testing.T) {
 	v := DiskInfo{}
 	bts, err := v.MarshalMsg(nil)