feat: allow expiration of all versions via ILM Expiration action (#17521)

Following extension allows users to specify immediate purge of
all versions as soon as the latest version of this object has
expired.

```
<LifecycleConfiguration>
    <Rule>
        <ID>ClassADocRule</ID>
        <Filter>
           <Prefix>classA/</Prefix>
        </Filter>
        <Status>Enabled</Status>
        <Expiration>
             <Days>3650</Days>
	     <ExpiredObjectAllVersions>true</ExpiredObjectAllVersions>
        </Expiration>
    </Rule>
    ...
```

CREDITS

@@ -2465,6 +2465,39 @@ SOFTWARE.
 
 ================================================================
 
+github.com/alexbrainman/sspi
+https://github.com/alexbrainman/sspi
+----------------------------------------------------------------
+Copyright (c) 2012 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
 github.com/apache/thrift
 https://github.com/apache/thrift
 ----------------------------------------------------------------
@@ -3027,7 +3060,7 @@ https://github.com/charmbracelet/bubbles
 ----------------------------------------------------------------
 MIT License
 
-Copyright (c) 2020 Charmbracelet, Inc
+Copyright (c) 2020-2023 Charmbracelet, Inc
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -9210,214 +9243,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ================================================================
 
-github.com/google/martian
-https://github.com/google/martian
-----------------------------------------------------------------
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-================================================================
-
 github.com/google/martian/v3
 https://github.com/google/martian/v3
 ----------------------------------------------------------------
@@ -17739,8 +17564,8 @@ For more information on this, and how to apply and follow the GNU AGPL, see
 
 ================================================================
 
-github.com/minio/madmin-go/v2
-https://github.com/minio/madmin-go/v2
+github.com/minio/madmin-go/v3
+https://github.com/minio/madmin-go/v3
 ----------------------------------------------------------------
                     GNU AFFERO GENERAL PUBLIC LICENSE
                        Version 3, 19 November 2007
@@ -29247,6 +29072,422 @@ https://google.golang.org/genproto
 
 ================================================================
 
+google.golang.org/genproto/googleapis/api
+https://google.golang.org/genproto/googleapis/api
+----------------------------------------------------------------
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+================================================================
+
+google.golang.org/genproto/googleapis/rpc
+https://google.golang.org/genproto/googleapis/rpc
+----------------------------------------------------------------
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+================================================================
+
 google.golang.org/grpc
 https://google.golang.org/grpc
 ----------------------------------------------------------------

buildscripts/heal-manual.go

@@ -27,7 +27,7 @@ import (
 	"os"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 )
 
 func main() {

cmd/admin-bucket-handlers.go

@@ -32,7 +32,7 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/kes-go"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"

cmd/admin-handler-utils.go

@@ -24,7 +24,7 @@ import (
 	"net/http"
 
 	"github.com/minio/kes-go"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
 	iampolicy "github.com/minio/pkg/iam/policy"

cmd/admin-handlers-config-kv.go

@@ -26,7 +26,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/etcd"
@@ -241,6 +241,8 @@ func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (re
 // 1. `subsys:target` -> request for config of a single subsystem and target pair.
 // 2. `subsys:` -> request for config of a single subsystem and the default target.
 // 3. `subsys` -> request for config of all targets for the given subsystem.
+//
+// This is a reporting API and config secrets are redacted in the response.
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfigKV")
 
@@ -268,7 +270,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		}
 	}
 
-	subSysConfigs, err := cfg.GetSubsysInfo(subSys, target)
+	subSysConfigs, err := cfg.GetSubsysInfo(subSys, target, true)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -490,7 +492,9 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 }
 
 // GetConfigHandler - GET /minio/admin/v3/config
-// Get config.json of this minio setup.
+//
+// This endpoint is mainly for exporting and backing up the configuration.
+// Secrets are not redacted.
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfig")
 
@@ -507,7 +511,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 	hkvs := config.HelpSubSysMap[""]
 	for _, hkv := range hkvs {
 		// We ignore the error below, as we cannot get one.
-		cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "")
+		cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "", false)
 
 		for _, item := range cfgSubsysItems {
 			off := item.Config.Get(config.Enable) == config.EnableOff

cmd/admin-handlers-idp-config.go

@@ -26,7 +26,7 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	cfgldap "github.com/minio/minio/internal/config/identity/ldap"

cmd/admin-handlers-idp-ldap.go

@@ -22,7 +22,7 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -150,7 +150,7 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 	}
 
 	// Call IAM subsystem
-	updatedAt, addedOrRemoved, err := globalIAMSys.PolicyDBUpdateLDAP(ctx, isAttach, par)
+	updatedAt, addedOrRemoved, _, err := globalIAMSys.PolicyDBUpdateLDAP(ctx, isAttach, par)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return

cmd/admin-handlers-site-replication.go

@@ -26,7 +26,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/mux"
 
 	"github.com/minio/minio/internal/logger"

cmd/admin-handlers-users-race_test.go

@@ -30,7 +30,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	minio "github.com/minio/minio-go/v7"
 	"github.com/minio/pkg/sync/errgroup"
 )

cmd/admin-handlers-users.go

@@ -26,11 +26,10 @@ import (
 	"net/http"
 	"os"
 	"sort"
-	"strings"
 	"time"
 
 	"github.com/klauspost/compress/zip"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/logger"
@@ -1771,24 +1770,40 @@ func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *ht
 	writeSuccessResponseJSON(w, econfigData)
 }
 
-// AttachPolicyBuiltin - POST /minio/admin/v3/idp/builtin/policy/attach
-func (a adminAPIHandlers) AttachPolicyBuiltin(w http.ResponseWriter, r *http.Request) {
+// AttachDetachPolicyBuiltin - POST /minio/admin/v3/idp/builtin/policy/{operation}
+func (a adminAPIHandlers) AttachDetachPolicyBuiltin(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AttachPolicyBuiltin")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.UpdatePolicyAssociationAction,
+		iampolicy.AttachPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
 
-	cred, _, s3Err := validateAdminSignature(ctx, r, "")
-	if s3Err != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
 		return
 	}
-	password := cred.SecretKey
 
+	// Ensure body content type is opaque to ensure that request body has not
+	// been interpreted as form data.
+	contentType := r.Header.Get("Content-Type")
+	if contentType != "application/octet-stream" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
+		return
+	}
+
+	operation := mux.Vars(r)["operation"]
+	if operation != "attach" && operation != "detach" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
+		return
+	}
+	isAttach := operation == "attach"
+
+	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -1806,235 +1821,42 @@ func (a adminAPIHandlers) AttachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	userOrGroup := par.User
-	isGroup := false
-
-	if userOrGroup == "" {
-		userOrGroup = par.Group
-		isGroup = true
-	}
-
-	if isGroup {
-		_, err := globalIAMSys.GetGroupDescription(userOrGroup)
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-	} else {
-		ok, _, err := globalIAMSys.IsTempUser(userOrGroup)
-		if err != nil && err != errNoSuchUser {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-		if ok {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
-			return
-		}
-
-		// When the user is root credential you are not allowed to
-		// add policies for root user.
-		if userOrGroup == globalActiveCred.AccessKey {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
-			return
-		}
-
-		// Validate that user exists.
-		_, ok = globalIAMSys.GetUser(ctx, userOrGroup)
-		if !ok {
+	updatedAt, addedOrRemoved, _, err := globalIAMSys.PolicyDBUpdateBuiltin(ctx, isAttach, par)
+	if err != nil {
+		if err == errNoSuchUser || err == errNoSuchGroup {
 			if globalIAMSys.LDAPConfig.Enabled() {
 				// When LDAP is enabled, warn user that they are using the wrong
-				// API.
+				// API. FIXME: error can be no such group as well - fix errNoSuchUserLDAPWarn
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUserLDAPWarn), r.URL)
 				return
 			}
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
-			return
 		}
-	}
-
-	var existingPolicies []string
-	if isGroup {
-		existingPolicies, err = globalIAMSys.PolicyDBGet(userOrGroup, true)
-	} else {
-		existingPolicies, err = globalIAMSys.GetUserPolicies(userOrGroup)
-	}
-	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	policyMap := make(map[string]bool)
-	for _, p := range existingPolicies {
-		policyMap[p] = true
-	}
-
-	policiesToAttach := par.Policies
-
-	// Check if policy is already attached to user.
-	for _, p := range policiesToAttach {
-		if _, ok := policyMap[p]; ok {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrPolicyAlreadyAttached), r.URL)
-			return
-		}
-	}
-
-	existingPolicies = append(existingPolicies, policiesToAttach...)
-	newPolicies := strings.Join(existingPolicies, ",")
-
-	userType := regUser
-	updatedAt, err := globalIAMSys.PolicyDBSet(ctx, userOrGroup, newPolicies, userType, isGroup)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
-		Type: madmin.SRIAMItemPolicyMapping,
-		PolicyMapping: &madmin.SRPolicyMapping{
-			UserOrGroup: userOrGroup,
-			UserType:    int(userType),
-			IsGroup:     isGroup,
-			Policy:      strings.Join(policiesToAttach, ","),
-		},
+	respBody := madmin.PolicyAssociationResp{
 		UpdatedAt: updatedAt,
-	}))
-
-	writeResponse(w, http.StatusCreated, nil, mimeNone)
-}
-
-// DetachPolicyBuiltin - POST /minio/admin/v3/idp/builtin/policy/detach
-func (a adminAPIHandlers) DetachPolicyBuiltin(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "DetachPolicyBuiltin")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
-	if objectAPI == nil {
-		return
 	}
-
-	cred, _, s3Err := validateAdminSignature(ctx, r, "")
-	if s3Err != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
-		return
-	}
-	password := cred.SecretKey
-
-	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	var par madmin.PolicyAssociationReq
-	if err = json.Unmarshal(reqBytes, &par); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	if err = par.IsValid(); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	userOrGroup := par.User
-	isGroup := false
-
-	if userOrGroup == "" {
-		userOrGroup = par.Group
-		isGroup = true
-	}
-
-	if isGroup {
-		_, err := globalIAMSys.GetGroupDescription(userOrGroup)
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
+	if isAttach {
+		respBody.PoliciesAttached = addedOrRemoved
 	} else {
-		ok, _, err := globalIAMSys.IsTempUser(userOrGroup)
-		if err != nil && err != errNoSuchUser {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-		if ok {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
-			return
-		}
-
-		// Validate that user exists.
-		_, ok = globalIAMSys.GetUser(ctx, userOrGroup)
-		if !ok {
-			if globalIAMSys.LDAPConfig.Enabled() {
-				// When LDAP is enabled, warn user that they are using the wrong
-				// API.
-				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUserLDAPWarn), r.URL)
-				return
-			}
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
-			return
-		}
+		respBody.PoliciesDetached = addedOrRemoved
 	}
 
-	userType := regUser
-	if globalIAMSys.GetUsersSysType() == LDAPUsersSysType {
-		userType = stsUser
-	}
-
-	var existingPolicies []string
-	if isGroup {
-		existingPolicies, err = globalIAMSys.PolicyDBGet(userOrGroup, true)
-	} else {
-		existingPolicies, err = globalIAMSys.GetUserPolicies(userOrGroup)
-	}
+	data, err := json.Marshal(respBody)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	policyMap := make(map[string]bool)
-	for _, p := range existingPolicies {
-		policyMap[p] = true
-	}
-
-	policiesToDetach := par.Policies
-
-	// Check if policy is already attached to user.
-	for _, p := range policiesToDetach {
-		if _, ok := policyMap[p]; ok {
-			delete(policyMap, p)
-		} else {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrPolicyNotAttached), r.URL)
-			return
-		}
-	}
-
-	newPoliciesSl := []string{}
-	for p := range policyMap {
-		newPoliciesSl = append(newPoliciesSl, p)
-	}
-
-	newPolicies := strings.Join(newPoliciesSl, ",")
-
-	updatedAt, err := globalIAMSys.PolicyDBSet(ctx, userOrGroup, newPolicies, userType, isGroup)
+	encryptedData, err := madmin.EncryptData(password, data)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
-		Type: madmin.SRIAMItemPolicyMapping,
-		PolicyMapping: &madmin.SRPolicyMapping{
-			UserOrGroup: userOrGroup,
-			UserType:    int(userType),
-			IsGroup:     isGroup,
-			Policy:      newPolicies,
-		},
-		UpdatedAt: updatedAt,
-	}))
-
-	// Return successful JSON response
-	writeSuccessNoContent(w)
+	writeSuccessResponseJSON(w, encryptedData)
 }
 
 const (

cmd/admin-handlers-users_test.go

@@ -33,10 +33,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
-	cr "github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/signer"
@@ -877,7 +876,7 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
 
 	// Create an madmin client with user creds
 	userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
-		Creds:  cr.NewStaticV4(accessKey, secretKey, ""),
+		Creds:  credentials.NewStaticV4(accessKey, secretKey, ""),
 		Secure: s.secure,
 	})
 	if err != nil {
@@ -1072,7 +1071,7 @@ func (s *TestSuiteIAM) TestAccMgmtPlugin(c *check) {
 
 	// Create an madmin client with user creds
 	userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
-		Creds:  cr.NewStaticV4(accessKey, secretKey, ""),
+		Creds:  credentials.NewStaticV4(accessKey, secretKey, ""),
 		Secure: s.secure,
 	})
 	if err != nil {

cmd/admin-handlers.go

@@ -45,8 +45,8 @@ import (
 
 	"github.com/dustin/go-humanize"
 	"github.com/klauspost/compress/zip"
-	"github.com/minio/madmin-go/v2"
-	"github.com/minio/madmin-go/v2/estream"
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/madmin-go/v3/estream"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/handlers"
@@ -2103,6 +2103,27 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 		}
 	}
 
+	// collect all realtime metrics except disk
+	// disk metrics are already included under drive info of each server
+	getRealtimeMetrics := func() *madmin.RealtimeMetrics {
+		var m madmin.RealtimeMetrics
+		var types madmin.MetricType = madmin.MetricsAll &^ madmin.MetricsDisk
+		mLocal := collectLocalMetrics(types, collectMetricsOpts{})
+		m.Merge(&mLocal)
+		cctx, cancel := context.WithTimeout(healthCtx, time.Second/2)
+		mRemote := collectRemoteMetrics(cctx, types, collectMetricsOpts{})
+		cancel()
+		m.Merge(&mRemote)
+		for idx, host := range m.Hosts {
+			m.Hosts[idx] = anonAddr(host)
+		}
+		for host, metrics := range m.ByHost {
+			m.ByHost[anonAddr(host)] = metrics
+			delete(m.ByHost, host)
+		}
+		return &m
+	}
+
 	anonymizeCmdLine := func(cmdLine string) string {
 		if !globalIsDistErasure {
 			// FS mode - single server - hard code to `server1`
@@ -2280,6 +2301,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 				TLS:          &tls,
 				IsKubernetes: &isK8s,
 				IsDocker:     &isDocker,
+				Metrics:      getRealtimeMetrics(),
 			}
 			partialWrite(healthInfo)
 		}

cmd/admin-handlers_test.go

@@ -31,7 +31,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/mux"
 )

cmd/admin-heal-ops.go

@@ -27,7 +27,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 )
 

cmd/admin-router.go

@@ -22,7 +22,7 @@ import (
 
 	"github.com/klauspost/compress/gzhttp"
 	"github.com/klauspost/compress/gzip"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 )
@@ -162,11 +162,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			HandlerFunc(gz(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup))).
 			Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
 
-		// Attach policies to user or group
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/attach").HandlerFunc(gz(httpTraceHdrs(adminAPI.AttachPolicyBuiltin)))
-
-		// Detach policies from user or group
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/detach").HandlerFunc(gz(httpTraceHdrs(adminAPI.DetachPolicyBuiltin)))
+		// Attach/Detach policies to/from user or group
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/{operation}").HandlerFunc(gz(httpTraceHdrs(adminAPI.AttachDetachPolicyBuiltin)))
 
 		// Remove user IAM
 		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveUser))).Queries("accessKey", "{accessKey:.*}")

cmd/admin-server-info.go

@@ -26,7 +26,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"

cmd/api-errors.go

@@ -31,7 +31,7 @@ import (
 	"github.com/minio/minio/internal/ioutil"
 	"google.golang.org/api/googleapi"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/auth"
@@ -184,6 +184,7 @@ const (
 	ErrBucketAlreadyExists
 	ErrMetadataTooLarge
 	ErrUnsupportedMetadata
+	ErrUnsupportedHostHeader
 	ErrMaximumExpires
 	ErrSlowDownRead
 	ErrSlowDownWrite
@@ -1442,6 +1443,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Your metadata headers are not supported.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrUnsupportedHostHeader: {
+		Code:           "InvalidArgument",
+		Description:    "Your Host header is malformed.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrObjectTampered: {
 		Code:           "XMinioObjectTampered",
 		Description:    errObjectTampered.Error(),

cmd/background-heal-ops.go

@@ -24,7 +24,7 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/env"
 )

cmd/background-newdisks-heal-ops.go

@@ -30,7 +30,7 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/logger"

cmd/batch-handlers.go

@@ -37,8 +37,7 @@ import (
 
 	"github.com/dustin/go-humanize"
 	"github.com/lithammer/shortuuid/v4"
-	"github.com/minio/madmin-go/v2"
-	"github.com/minio/minio-go/v7"
+	"github.com/minio/madmin-go/v3"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
@@ -311,7 +310,7 @@ func (r BatchJobReplicateV1) Notify(ctx context.Context, body io.Reader) error {
 }
 
 // ReplicateFromSource - this is not implemented yet where source is 'remote' and target is local.
-func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, core *minio.Core, srcObjInfo ObjectInfo, retry bool) error {
+func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, core *miniogo.Core, srcObjInfo ObjectInfo, retry bool) error {
 	srcBucket := r.Source.Bucket
 	tgtBucket := r.Target.Bucket
 	srcObject := srcObjInfo.Name
@@ -360,7 +359,7 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
 		}
 		return r.copyWithMultipartfromSource(ctx, api, core, srcObjInfo, opts, partsCount)
 	}
-	gopts := minio.GetObjectOptions{
+	gopts := miniogo.GetObjectOptions{
 		VersionID: srcObjInfo.VersionID,
 	}
 	if err := gopts.SetMatchETag(srcObjInfo.ETag); err != nil {
@@ -381,7 +380,7 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
 	return err
 }
 
-func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, api ObjectLayer, c *minio.Core, srcObjInfo ObjectInfo, opts ObjectOptions, partsCount int) (err error) {
+func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, api ObjectLayer, c *miniogo.Core, srcObjInfo ObjectInfo, opts ObjectOptions, partsCount int) (err error) {
 	srcBucket := r.Source.Bucket
 	tgtBucket := r.Target.Bucket
 	srcObject := srcObjInfo.Name
@@ -422,7 +421,7 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a
 	)
 
 	for i := 0; i < partsCount; i++ {
-		gopts := minio.GetObjectOptions{
+		gopts := miniogo.GetObjectOptions{
 			VersionID:  srcObjInfo.VersionID,
 			PartNumber: i + 1,
 		}
@@ -556,7 +555,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	}
 
 	c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
-	core := &minio.Core{Client: c}
+	core := &miniogo.Core{Client: c}
 
 	workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_REPLICATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
 	if err != nil {
@@ -577,7 +576,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 		s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
 		minioSrc := r.Source.Type == BatchJobReplicateResourceMinIO
 		ctx, cancel := context.WithCancel(ctx)
-		objInfoCh := c.ListObjects(ctx, r.Source.Bucket, minio.ListObjectsOptions{
+		objInfoCh := c.ListObjects(ctx, r.Source.Bucket, miniogo.ListObjectsOptions{
 			Prefix:       r.Source.Prefix,
 			WithVersions: minioSrc,
 			Recursive:    true,
@@ -589,7 +588,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 		for obj := range objInfoCh {
 			oi := toObjectInfo(r.Source.Bucket, obj.Key, obj)
 			if !minioSrc {
-				oi2, err := c.StatObject(ctx, r.Source.Bucket, obj.Key, minio.StatObjectOptions{})
+				oi2, err := c.StatObject(ctx, r.Source.Bucket, obj.Key, miniogo.StatObjectOptions{})
 				if err == nil {
 					oi = toObjectInfo(r.Source.Bucket, obj.Key, oi2)
 				} else {
@@ -671,7 +670,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 }
 
 // toObjectInfo converts minio.ObjectInfo to ObjectInfo
-func toObjectInfo(bucket, object string, objInfo minio.ObjectInfo) ObjectInfo {
+func toObjectInfo(bucket, object string, objInfo miniogo.ObjectInfo) ObjectInfo {
 	tags, _ := tags.MapToObjectTags(objInfo.UserTags)
 	oi := ObjectInfo{
 		Bucket:                    bucket,

cmd/bitrot-streaming.go

@@ -165,9 +165,9 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 			b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
 			if err != nil {
 				if !IsErr(err, ignoredErrs...) {
-					logger.LogIf(GlobalContext,
+					logger.LogOnceIf(GlobalContext,
 						fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
-							b.disk, b.volume, b.filePath, err))
+							b.disk, b.volume, b.filePath, err), "bitrot-read-file-stream"+b.volume+b.filePath)
 				}
 			}
 		} else {

cmd/bootstrap-messages.go

@@ -23,7 +23,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/pubsub"
 )
 

cmd/bucket-encryption-handlers.go

@@ -26,7 +26,7 @@ import (
 	"net/http"
 
 	"github.com/minio/kes-go"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"

cmd/bucket-handlers.go

@@ -41,7 +41,7 @@ import (
 	"github.com/minio/mux"
 	"github.com/valyala/bytebufferpool"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
@@ -654,6 +654,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		if deleteResult.errInfo.Code != "" {
 			deleteErrors = append(deleteErrors, deleteResult.errInfo)
 		} else {
+			// All deletes on directory objects was with `nullVersionID`.
+			// Remove it from response.
+			if isDirObject(deleteResult.delInfo.ObjectName) && deleteResult.delInfo.VersionID == nullVersionID {
+				deleteResult.delInfo.VersionID = ""
+			}
 			deletedObjects = append(deletedObjects, deleteResult.delInfo)
 		}
 	}
@@ -669,6 +674,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		}
 
 		if replicateDeletes && (dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == Pending) {
+			// copy so we can re-add null ID.
+			dobj := dobj
+			if isDirObject(dobj.ObjectName) && dobj.VersionID == "" {
+				dobj.VersionID = nullVersionID
+			}
 			dv := DeletedObjectReplicationInfo{
 				DeletedObject: dobj,
 				Bucket:        bucket,

cmd/bucket-handlers_test.go

@@ -749,7 +749,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := range requestList[0].Objects {
 		var vid string
 		if isDirObject(requestList[0].Objects[i].ObjectName) {
-			vid = nullVersionID
+			vid = ""
 		}
 		deletedObjects[i] = DeletedObject{
 			ObjectName: requestList[0].Objects[i].ObjectName,
@@ -766,7 +766,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := range requestList[1].Objects {
 		var vid string
 		if isDirObject(requestList[0].Objects[i].ObjectName) {
-			vid = nullVersionID
+			vid = ""
 		}
 		deletedObjects[i] = DeletedObject{
 			ObjectName: requestList[1].Objects[i].ObjectName,

cmd/bucket-lifecycle.go

@@ -32,7 +32,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/amztime"
 	sse "github.com/minio/minio/internal/bucket/encryption"

cmd/bucket-metadata-sys.go

@@ -25,7 +25,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	bucketsse "github.com/minio/minio/internal/bucket/encryption"

cmd/bucket-metadata.go

@@ -29,7 +29,7 @@ import (
 	"path"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	bucketsse "github.com/minio/minio/internal/bucket/encryption"
 	"github.com/minio/minio/internal/bucket/lifecycle"

cmd/bucket-object-lock.go

@@ -83,12 +83,6 @@ func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locke
 // governance bypass headers are set and user has governance bypass permissions.
 // Objects in "Compliance" mode can be overwritten only if retention date is past.
 func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucket string, object ObjectToDelete, oi ObjectInfo, gerr error) APIErrorCode {
-	opts, err := getOpts(ctx, r, bucket, object.ObjectName)
-	if err != nil {
-		return toAPIErrorCode(ctx, err)
-	}
-
-	opts.VersionID = object.VersionID
 	if gerr != nil { // error from GetObjectInfo
 		if _, ok := gerr.(MethodNotAllowed); ok {
 			// This happens usually for a delete marker

cmd/bucket-policy-handlers.go

@@ -24,7 +24,7 @@ import (
 	"net/http"
 
 	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/bucket/policy"

cmd/bucket-quota.go

@@ -24,7 +24,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 )
 

cmd/bucket-replication-utils.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -28,7 +28,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 )
@@ -512,7 +512,10 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 				ObjectName: oi.Name,
 				VersionID:  oi.VersionID,
 			},
-		}, oi, ObjectOptions{VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name)}, nil)
+		}, oi, ObjectOptions{
+			Versioned:        globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
+			VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
+		}, nil)
 	} else {
 		dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(ObjectInfo{
 			UserDefined: oi.UserDefined,

cmd/bucket-replication.go

@@ -35,7 +35,7 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
@@ -170,7 +170,7 @@ func checkRemoteEndpoint(ctx context.Context, epURL *url.URL) error {
 	reqURL := &url.URL{
 		Scheme: epURL.Scheme,
 		Host:   epURL.Host,
-		Path:   healthCheckReadinessPath,
+		Path:   healthCheckPathPrefix + healthCheckReadinessPath,
 	}
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL.String(), nil)
@@ -605,7 +605,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		return
 	}
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN))
+		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete"+tgt.ARN)
 		sendEvent(eventArgs{
 			BucketName: dobj.Bucket,
 			Object: ObjectInfo{
@@ -970,7 +970,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, "get-replication-config"+bucket)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1121,7 +1121,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	}
 
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN))
+		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN), "replication-target-offline-obj"+tgt.ARN)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1271,7 +1271,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	}
 
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN))
+		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN), "replication-target-offline-all"+tgt.ARN)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,

cmd/bucket-replication_test.go

@@ -24,7 +24,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 )

cmd/bucket-targets.go

@@ -25,7 +25,7 @@ import (
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio/internal/bucket/replication"

cmd/bucket-versioning-handler.go

@@ -24,7 +24,7 @@ import (
 	"net/http"
 
 	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"

cmd/callhome.go

@@ -28,7 +28,7 @@ import (
 	"net/url"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 )
 

cmd/common-main.go

@@ -52,7 +52,7 @@ import (
 	"github.com/minio/console/restapi"
 	"github.com/minio/console/restapi/operations"
 	"github.com/minio/kes-go"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/set"

cmd/config-current.go

@@ -24,7 +24,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/api"
 	"github.com/minio/minio/internal/config/cache"
@@ -616,6 +616,9 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		}
 		for n, l := range loggerCfg.AuditKafka {
 			if l.Enabled {
+				if l.TLS.Enable {
+					l.TLS.RootCAs = globalRootCAs
+				}
 				l.LogOnce = logger.LogOnceIf
 				loggerCfg.AuditKafka[n] = l
 			}

cmd/config-encrypted_test.go

@@ -21,7 +21,7 @@ import (
 	"bytes"
 	"testing"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 )
 

cmd/config.go

@@ -27,7 +27,7 @@ import (
 	"strings"
 
 	jsoniter "github.com/json-iterator/go"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
 )

cmd/consolelogger.go

@@ -23,7 +23,7 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/logger/target/console"
 	"github.com/minio/minio/internal/logger/target/types"

cmd/data-scanner-metric.go

@@ -7,7 +7,7 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 )
 

cmd/data-scanner.go

@@ -34,7 +34,7 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
@@ -941,7 +941,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), lcEvt.Action)
 		}
 	}
-	defer globalScannerMetrics.timeILM(lcEvt.Action)
+	defer globalScannerMetrics.timeILM(lcEvt.Action)()
 
 	switch lcEvt.Action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
@@ -1187,6 +1187,9 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		opts.Versioned = globalBucketVersioningSys.PrefixEnabled(obj.Bucket, obj.Name)
 		opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(obj.Bucket, obj.Name)
 	}
+	if lcEvent.Action.DeleteAll() {
+		opts.DeletePrefix = true
+	}
 
 	obj, err := objLayer.DeleteObject(ctx, obj.Bucket, obj.Name, opts)
 	if err != nil {
@@ -1194,7 +1197,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 			return false
 		}
 		// Assume it is still there.
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, "non-transition-expiry")
 		return false
 	}
 

cmd/data-usage-cache.go

@@ -33,7 +33,7 @@ import (
 
 	"github.com/cespare/xxhash/v2"
 	"github.com/klauspost/compress/zstd"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"

cmd/data-usage-utils.go

@@ -21,7 +21,7 @@ import (
 	"sort"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 )
 
 // BucketTargetUsageInfo - bucket target usage info provides

cmd/endpoint-ellipses.go

@@ -359,6 +359,9 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		if err != nil {
 			return nil, -1, err
 		}
+		for i := range endpointList {
+			endpointList[i].SetPool(0)
+		}
 		endpointServerPools = append(endpointServerPools, PoolEndpoints{
 			Legacy:       true,
 			SetCount:     len(setArgs),

cmd/endpoint.go

@@ -38,6 +38,7 @@ import (
 	"github.com/minio/minio/internal/mountinfo"
 	"github.com/minio/pkg/env"
 	xnet "github.com/minio/pkg/net"
+	"golang.org/x/exp/slices"
 )
 
 // EndpointType - enum for endpoint type.
@@ -58,9 +59,17 @@ type ProxyEndpoint struct {
 	Transport http.RoundTripper
 }
 
+// Node holds information about a node in this cluster
+type Node struct {
+	*url.URL
+	Pools   []int
+	IsLocal bool
+}
+
 // Endpoint - any type of endpoint.
 type Endpoint struct {
 	*url.URL
+	Pool    int
 	IsLocal bool
 }
 
@@ -97,6 +106,11 @@ func (endpoint *Endpoint) UpdateIsLocal() (err error) {
 	return nil
 }
 
+// SetPool sets a specific pool number to this node
+func (endpoint *Endpoint) SetPool(i int) {
+	endpoint.Pool = i
+}
+
 // NewEndpoint - returns new endpoint based on given arguments.
 func NewEndpoint(arg string) (ep Endpoint, e error) {
 	// isEmptyPath - check whether given path is not empty.
@@ -209,6 +223,35 @@ type PoolEndpoints struct {
 // EndpointServerPools - list of list of endpoints
 type EndpointServerPools []PoolEndpoints
 
+// GetNodes returns a sorted list of nodes in this cluster
+func (l EndpointServerPools) GetNodes() (nodes []Node) {
+	nodesMap := make(map[string]Node)
+	for _, pool := range l {
+		for _, ep := range pool.Endpoints {
+			node, ok := nodesMap[ep.Host]
+			if !ok {
+				node.IsLocal = ep.IsLocal
+				node.URL = &url.URL{
+					Scheme: ep.Scheme,
+					Host:   ep.Host,
+				}
+			}
+			if !slices.Contains(node.Pools, ep.Pool) {
+				node.Pools = append(node.Pools, ep.Pool)
+			}
+			nodesMap[ep.Host] = node
+		}
+	}
+	nodes = make([]Node, 0, len(nodesMap))
+	for _, v := range nodesMap {
+		nodes = append(nodes, v)
+	}
+	sort.Slice(nodes, func(i, j int) bool {
+		return nodes[i].Host < nodes[j].Host
+	})
+	return
+}
+
 // GetPoolIdx return pool index
 func (l EndpointServerPools) GetPoolIdx(pool string) int {
 	for id, ep := range globalEndpoints {
@@ -768,6 +811,8 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 			return nil, setupType, config.ErrInvalidEndpoint(nil).Msg("use path style endpoint for single node setup")
 		}
 
+		endpoint.SetPool(0)
+
 		var endpoints Endpoints
 		endpoints = append(endpoints, endpoint)
 		setupType = ErasureSDSetupType
@@ -781,7 +826,7 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 		return poolEndpoints, setupType, nil
 	}
 
-	for i, args := range poolArgs {
+	for poolIdx, args := range poolArgs {
 		var endpoints Endpoints
 		for _, iargs := range args {
 			// Convert args to endpoints
@@ -795,6 +840,10 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 				return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 			}
 
+			for i := range eps {
+				eps[i].SetPool(poolIdx)
+			}
+
 			endpoints = append(endpoints, eps...)
 		}
 
@@ -802,7 +851,7 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 			return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg("invalid number of endpoints")
 		}
 
-		poolEndpoints[i] = endpoints
+		poolEndpoints[poolIdx] = endpoints
 	}
 
 	for _, endpoints := range poolEndpoints {
@@ -960,7 +1009,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType,
 
 	_, serverAddrPort := mustSplitHostPort(serverAddr)
 
-	// For single arg, return FS setup.
+	// For single arg, return single drive setup.
 	if len(args) == 1 && len(args[0]) == 1 {
 		var endpoint Endpoint
 		endpoint, err = NewEndpoint(args[0][0])

cmd/erasure-decode.go

@@ -317,7 +317,7 @@ func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.Rea
 		}
 
 		if err = e.DecodeDataAndParityBlocks(ctx, bufs); err != nil {
-			logger.LogIf(ctx, err)
+			logger.LogOnceIf(ctx, err, "erasure-heal-decode")
 			return err
 		}
 
@@ -328,7 +328,7 @@ func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.Rea
 		}
 
 		if err = w.Write(ctx, bufs); err != nil {
-			logger.LogIf(ctx, err)
+			logger.LogOnceIf(ctx, err, "erasure-heal-write")
 			return err
 		}
 	}

cmd/erasure-healing-common.go

@@ -22,7 +22,7 @@ import (
 	"context"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 )
 
 func commonETags(etags []string) (etag string, maxima int) {

cmd/erasure-healing-common_test.go

@@ -27,7 +27,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 )
 
 // Returns the latest updated FileInfo files and error in case of failure.

cmd/erasure-healing.go

@@ -23,10 +23,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"strings"
 	"sync"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/sync/errgroup"
 )
@@ -66,6 +67,68 @@ func (fi FileInfo) DataShardFixed() bool {
 	return fi.Metadata[reservedMetadataPrefixLowerDataShardFix] == "true"
 }
 
+func (er erasureObjects) listAndHeal(bucket, prefix string, healEntry func(string, metaCacheEntry) error) error {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	disks, _ := er.getOnlineDisksWithHealing()
+	if len(disks) == 0 {
+		return errors.New("listAndHeal: No non-healing drives found")
+	}
+
+	// How to resolve partial results.
+	resolver := metadataResolutionParams{
+		dirQuorum: 1,
+		objQuorum: 1,
+		bucket:    bucket,
+		strict:    false, // Allow less strict matching.
+	}
+
+	path := baseDirFromPrefix(prefix)
+	filterPrefix := strings.Trim(strings.TrimPrefix(prefix, path), slashSeparator)
+	if path == prefix {
+		filterPrefix = ""
+	}
+
+	lopts := listPathRawOptions{
+		disks:          disks,
+		bucket:         bucket,
+		path:           path,
+		filterPrefix:   filterPrefix,
+		recursive:      true,
+		forwardTo:      "",
+		minDisks:       1,
+		reportNotFound: false,
+		agreed: func(entry metaCacheEntry) {
+			if err := healEntry(bucket, entry); err != nil {
+				logger.LogIf(ctx, err)
+				cancel()
+			}
+		},
+		partial: func(entries metaCacheEntries, _ []error) {
+			entry, ok := entries.resolve(&resolver)
+			if !ok {
+				// check if we can get one entry atleast
+				// proceed to heal nonetheless.
+				entry, _ = entries.firstFound()
+			}
+
+			if err := healEntry(bucket, *entry); err != nil {
+				logger.LogIf(ctx, err)
+				cancel()
+				return
+			}
+		},
+		finished: nil,
+	}
+
+	if err := listPathRaw(ctx, lopts); err != nil {
+		return fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)
+	}
+
+	return nil
+}
+
 // HealBucket heals a bucket if it doesn't exist on one of the disks, additionally
 // also heals the missing entries for bucket metadata files
 // `policy.json, notification.xml, listeners.json`.
@@ -504,7 +567,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	if !latestMeta.XLV1 && !latestMeta.Deleted && !recreate && disksToHealCount > latestMeta.Erasure.ParityBlocks {
 		// When disk to heal count is greater than parity blocks we should simply error out.
 		err := fmt.Errorf("(%d > %d) more drives are expected to heal than parity, returned errors: %v (dataErrs %v) -> %s/%s(%s)", disksToHealCount, latestMeta.Erasure.ParityBlocks, errs, dataErrs, bucket, object, versionID)
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, "heal-object-count-gt-parity")
 		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
 			bucket, object, versionID), err
 	}
@@ -528,7 +591,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(availableDisks) {
 		err := fmt.Errorf("unexpected file distribution (%v) from available disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
 			latestMeta.Erasure.Distribution, availableDisks, bucket, object, versionID)
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, "heal-object-available-disks")
 		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
 			bucket, object, versionID), err
 	}
@@ -538,7 +601,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(outDatedDisks) {
 		err := fmt.Errorf("unexpected file distribution (%v) from outdated disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
 			latestMeta.Erasure.Distribution, outDatedDisks, bucket, object, versionID)
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, "heal-object-outdated-disks")
 		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
 			bucket, object, versionID), err
 	}
@@ -548,7 +611,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(partsMetadata) {
 		err := fmt.Errorf("unexpected file distribution (%v) from metadata entries (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
 			latestMeta.Erasure.Distribution, len(partsMetadata), bucket, object, versionID)
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, "heal-object-metadata-entries")
 		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
 			bucket, object, versionID), err
 	}

cmd/erasure-healing_test.go

@@ -32,7 +32,7 @@ import (
 
 	"github.com/dustin/go-humanize"
 	uuid2 "github.com/google/uuid"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config/storageclass"
 )
 

cmd/erasure-multipart.go

@@ -1249,7 +1249,14 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	}
 
 	if !opts.Speedtest && versionsDisparity {
-		listAndHeal(ctx, bucket, object, &er, healObjectVersionsDisparity)
+		globalMRFState.addPartialOp(partialOperation{
+			bucket:      bucket,
+			object:      object,
+			queued:      time.Now(),
+			allVersions: true,
+			setIndex:    er.setIndex,
+			poolIndex:   er.poolIndex,
+		})
 	}
 
 	// Check if there is any offline disk and add it to the MRF list

cmd/erasure-object.go

@@ -32,7 +32,7 @@ import (
 	"time"
 
 	"github.com/klauspost/readahead"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/object/lock"
@@ -673,7 +673,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	if !fi.Deleted && len(fi.Erasure.Distribution) != len(onlineDisks) {
 		err := fmt.Errorf("unexpected file distribution (%v) from online disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
 			fi.Erasure.Distribution, onlineDisks, bucket, object, opts.VersionID)
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, "get-object-file-info-manually-modified")
 		return fi, nil, nil, toObjectErr(err, bucket, object, opts.VersionID)
 	}
 
@@ -691,9 +691,20 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 			missingBlocks++
 			continue
 		}
-		if metaArr[i].IsValid() && metaArr[i].ModTime.Equal(modTime) {
-			continue
-		}
+
+		// verify metadata is valid, it has similar erasure info
+		// as well as common modtime, if modtime is not possible
+		// verify if it has common "etag" atleast.
+		if metaArr[i].IsValid() && metaArr[i].Erasure.Equal(fi.Erasure) {
+			ok := metaArr[i].ModTime.Equal(modTime)
+			if modTime.IsZero() || modTime.Equal(timeSentinel) {
+				ok = etag != "" && etag == fi.Metadata["etag"]
+			}
+			if ok {
+				continue
+			}
+		} // in all other cases metadata is corrupt, do not read from it.
+
 		metaArr[i] = FileInfo{}
 		onlineDisks[i] = nil
 		missingBlocks++
@@ -1311,7 +1322,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		if errors.Is(err, errFileNotFound) {
 			return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object)
 		}
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, "erasure-object-rename"+bucket+object)
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
@@ -1338,7 +1349,14 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		}
 
 		if versionsDisparity {
-			listAndHeal(ctx, bucket, object, &er, healObjectVersionsDisparity)
+			globalMRFState.addPartialOp(partialOperation{
+				bucket:      bucket,
+				object:      object,
+				queued:      time.Now(),
+				allVersions: true,
+				setIndex:    er.setIndex,
+				poolIndex:   er.poolIndex,
+			})
 		}
 	}
 

cmd/erasure-server-pool-decom.go

@@ -31,7 +31,7 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
@@ -62,10 +62,10 @@ type PoolDecommissionInfo struct {
 	Object string `json:"-" msg:"obj"`
 
 	// Verbose information
-	ItemsDecommissioned     int64 `json:"-" msg:"id"`
-	ItemsDecommissionFailed int64 `json:"-" msg:"idf"`
-	BytesDone               int64 `json:"-" msg:"bd"`
-	BytesFailed             int64 `json:"-" msg:"bf"`
+	ItemsDecommissioned     int64 `json:"objectsDecommissioned" msg:"id"`
+	ItemsDecommissionFailed int64 `json:"objectsDecommissionedFailed" msg:"idf"`
+	BytesDone               int64 `json:"bytesDecommissioned" msg:"bd"`
+	BytesFailed             int64 `json:"bytesDecommissionedFailed" msg:"bf"`
 }
 
 // bucketPop should be called when a bucket is done decommissioning.
@@ -419,7 +419,9 @@ func (p poolMeta) save(ctx context.Context, pools []*erasureSets) error {
 	// Saves on all pools to make sure decommissioning of first pool is allowed.
 	for i, eset := range pools {
 		if err = saveConfig(ctx, eset, poolMetaName, buf); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("saving pool.bin for pool index %d failed with: %v", i, err))
+			if !errors.Is(err, context.Canceled) {
+				logger.LogIf(ctx, fmt.Errorf("saving pool.bin for pool index %d failed with: %v", i, err))
+			}
 			return err
 		}
 	}
@@ -459,22 +461,34 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 	if !update {
 		z.poolMeta = meta
 	} else {
-		meta = poolMeta{} // to update write poolMeta fresh.
+		newMeta := poolMeta{} // to update write poolMeta fresh.
 		// looks like new pool was added we need to update,
 		// or this is a fresh installation (or an existing
 		// installation with pool removed)
-		meta.Version = poolMetaVersion
+		newMeta.Version = poolMetaVersion
 		for idx, pool := range z.serverPools {
-			meta.Pools = append(meta.Pools, PoolStatus{
+			var skip bool
+			for _, currentPool := range meta.Pools {
+				// Preserve any current pool status.
+				if currentPool.CmdLine == pool.endpoints.CmdLine {
+					newMeta.Pools = append(newMeta.Pools, currentPool)
+					skip = true
+					break
+				}
+			}
+			if skip {
+				continue
+			}
+			newMeta.Pools = append(newMeta.Pools, PoolStatus{
 				CmdLine:    pool.endpoints.CmdLine,
 				ID:         idx,
 				LastUpdate: UTCNow(),
 			})
 		}
-		if err = meta.save(ctx, z.serverPools); err != nil {
+		if err = newMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
-		z.poolMeta = meta
+		z.poolMeta = newMeta
 	}
 
 	pools := meta.returnResumablePools()
@@ -720,18 +734,24 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			// to create the appropriate stack.
 			versionsSorter(fivs.Versions).reverse()
 
-			var decommissionedCount int
+			var decommissioned, expired int
 			for _, version := range fivs.Versions {
+				stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
 				// Apply lifecycle rules on the objects that are expired.
 				if filterLifecycle(bi.Name, version.Name, version) {
-					decommissionedCount++
+					expired++
+					decommissioned++
+					stopFn(errors.New("ILM expired object/version will be skipped"))
 					continue
 				}
 
 				// any object with only single DEL marker we don't need
-				// to decommission, just skip it.
-				if version.Deleted && len(fivs.Versions) == 1 {
-					decommissionedCount++
+				// to decommission, just skip it, this also includes
+				// any other versions that have already expired.
+				remainingVersions := len(fivs.Versions) - expired
+				if version.Deleted && remainingVersions == 1 {
+					decommissioned++
+					stopFn(errors.New("DELETE marked object with no other non-current versions will be skipped"))
 					continue
 				}
 
@@ -756,6 +776,12 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							SkipDecommissioned: true, // make sure we skip the decommissioned pool
 						})
 					var failure bool
+					if err != nil {
+						if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+							err = nil
+						}
+					}
+					stopFn(err)
 					if err != nil {
 						logger.LogIf(ctx, err)
 						failure = true
@@ -765,7 +791,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					z.poolMetaMutex.Unlock()
 					if !failure {
 						// Success keep a count.
-						decommissionedCount++
+						decommissioned++
 					}
 					continue
 				}
@@ -774,7 +800,6 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				// gr.Close() is ensured by decommissionObject().
 				for try := 0; try < 3; try++ {
 					if version.IsRemote() {
-						stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
 						if err := z.DecomTieredObject(ctx, bi.Name, version.Name, version, ObjectOptions{
 							VersionID:   versionID,
 							MTime:       version.ModTime,
@@ -789,7 +814,6 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						failure = false
 						break
 					}
-					stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
 					gr, err := set.GetObjectNInfo(ctx,
 						bi.Name,
 						encodeDirObject(version.Name),
@@ -840,11 +864,11 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				if failure {
 					break // break out on first error
 				}
-				decommissionedCount++
+				decommissioned++
 			}
 
 			// if all versions were decommissioned, then we can delete the object versions.
-			if decommissionedCount == len(fivs.Versions) {
+			if decommissioned == len(fivs.Versions) {
 				stopFn := globalDecommissionMetrics.log(decomMetricDecommissionRemoveObject, idx, bi.Name, entry.name)
 				_, err := set.DeleteObject(ctx,
 					bi.Name,
@@ -1097,9 +1121,13 @@ func (z *erasureServerPools) Decommission(ctx context.Context, indices ...int) e
 		return err
 	}
 
-	for _, idx := range indices {
-		go z.doDecommissionInRoutine(ctx, idx)
-	}
+	go func() {
+		for _, idx := range indices {
+			// decommission all pools serially one after
+			// the other.
+			z.doDecommissionInRoutine(ctx, idx)
+		}
+	}()
 
 	// Successfully started decommissioning.
 	return nil

cmd/erasure-server-pool-rebalance.go

@@ -32,7 +32,7 @@ import (
 	"time"
 
 	"github.com/lithammer/shortuuid/v4"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/env"
@@ -500,7 +500,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 			// to create the appropriate stack.
 			versionsSorter(fivs.Versions).reverse()
 
-			var rebalanced int
+			var rebalanced, expired int
 			for _, version := range fivs.Versions {
 				// Skip transitioned objects for now. TBD
 				if version.IsRemote() {
@@ -510,12 +510,15 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 				// Apply lifecycle rules on the objects that are expired.
 				if filterLifecycle(bucket, version.Name, version) {
 					rebalanced++
+					expired++
 					continue
 				}
 
-				// Empty delete markers we can assume that they can be purged
-				// as there is no more data associated with them.
-				if version.Deleted && len(fivs.Versions) == 1 {
+				// any object with only single DEL marker we don't need
+				// to rebalance, just skip it, this also includes
+				// any other versions that have already expired.
+				remainingVersions := len(fivs.Versions) - expired
+				if version.Deleted && remainingVersions == 1 {
 					rebalanced++
 					continue
 				}
@@ -550,9 +553,10 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 					continue
 				}
 
-				var failure bool
+				var failure, ignore bool
 				for try := 0; try < 3; try++ {
 					// GetObjectReader.Close is called by rebalanceObject
+					stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceObject, poolIdx, bucket, version.Name, version.VersionID)
 					gr, err := set.GetObjectNInfo(ctx,
 						bucket,
 						encodeDirObject(version.Name),
@@ -565,19 +569,21 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 						})
 					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 						// object deleted by the application, nothing to do here we move on.
-						return
+						ignore = true
+						stopFn(nil)
+						break
 					}
 					if err != nil {
 						failure = true
 						logger.LogIf(ctx, err)
+						stopFn(err)
 						continue
 					}
 
-					stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceObject, poolIdx, bucket, version.Name)
 					if err = z.rebalanceObject(ctx, bucket, gr); err != nil {
-						stopFn(err)
 						failure = true
 						logger.LogIf(ctx, err)
+						stopFn(err)
 						continue
 					}
 
@@ -585,7 +591,9 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 					failure = false
 					break
 				}
-
+				if ignore {
+					continue
+				}
 				if failure {
 					break // break out on first error
 				}

cmd/erasure-server-pool.go

@@ -33,7 +33,7 @@ import (
 
 	"github.com/dustin/go-humanize"
 	"github.com/google/uuid"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
@@ -141,6 +141,10 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 	}
 
 	z.decommissionCancelers = make([]context.CancelFunc, len(z.serverPools))
+
+	// initialize the object layer.
+	setObjectLayer(z)
+
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	for {
 		err := z.Init(ctx) // Initializes all pools.
@@ -159,6 +163,7 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 	globalLocalDrivesMu.Lock()
 	globalLocalDrives = localDrives
 	defer globalLocalDrivesMu.Unlock()
+
 	return z, nil
 }
 
@@ -945,7 +950,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 		if !isMinioMetaBucketName(bucket) {
 			avail, err := hasSpaceFor(getDiskInfos(ctx, z.serverPools[0].getHashedSet(object).getDisks()...), data.Size())
 			if err != nil {
-				logger.LogIf(ctx, err)
+				logger.LogOnceIf(ctx, err, "erasure-write-quorum")
 				return ObjectInfo{}, toObjectErr(errErasureWriteQuorum)
 			}
 			if !avail {
@@ -988,13 +993,10 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 		return objInfo, err
 	}
 
-	if opts.DeletePrefix {
-		err := z.deletePrefix(ctx, bucket, object)
-		return ObjectInfo{}, err
+	if !opts.DeletePrefix { // DeletePrefix handles dir object encoding differently.
+		object = encodeDirObject(object)
 	}
 
-	object = encodeDirObject(object)
-
 	// Acquire a write lock before deleting the object.
 	lk := z.NewNSLock(bucket, object)
 	lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
@@ -1004,6 +1006,10 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 	ctx = lkctx.Context()
 	defer lk.Unlock(lkctx)
 
+	if opts.DeletePrefix {
+		return ObjectInfo{}, z.deletePrefix(ctx, bucket, object)
+	}
+
 	gopts := opts
 	gopts.NoLock = true
 	pinfo, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, gopts)
@@ -1337,7 +1343,7 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	merged, err := z.listPath(ctx, &opts)
 	if err != nil && err != io.EOF {
 		if !isErrBucketNotFound(err) {
-			logger.LogIf(ctx, err)
+			logger.LogOnceIf(ctx, err, "erasure-list-objects-path"+bucket)
 		}
 		return loi, err
 	}
@@ -1730,7 +1736,7 @@ func (z *erasureServerPools) HealFormat(ctx context.Context, dryRun bool) (madmi
 	for _, pool := range z.serverPools {
 		result, err := pool.HealFormat(ctx, dryRun)
 		if err != nil && !errors.Is(err, errNoHealRequired) {
-			logger.LogIf(ctx, err)
+			logger.LogOnceIf(ctx, err, "erasure-heal-format")
 			continue
 		}
 		// Count errNoHealRequired across all serverPools,
@@ -1900,68 +1906,6 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 // HealObjectFn closure function heals the object.
 type HealObjectFn func(bucket, object, versionID string) error
 
-func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects, healEntry func(string, metaCacheEntry) error) error {
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
-
-	disks, _ := set.getOnlineDisksWithHealing()
-	if len(disks) == 0 {
-		return errors.New("listAndHeal: No non-healing drives found")
-	}
-
-	// How to resolve partial results.
-	resolver := metadataResolutionParams{
-		dirQuorum: 1,
-		objQuorum: 1,
-		bucket:    bucket,
-		strict:    false, // Allow less strict matching.
-	}
-
-	path := baseDirFromPrefix(prefix)
-	filterPrefix := strings.Trim(strings.TrimPrefix(prefix, path), slashSeparator)
-	if path == prefix {
-		filterPrefix = ""
-	}
-
-	lopts := listPathRawOptions{
-		disks:          disks,
-		bucket:         bucket,
-		path:           path,
-		filterPrefix:   filterPrefix,
-		recursive:      true,
-		forwardTo:      "",
-		minDisks:       1,
-		reportNotFound: false,
-		agreed: func(entry metaCacheEntry) {
-			if err := healEntry(bucket, entry); err != nil {
-				logger.LogIf(ctx, err)
-				cancel()
-			}
-		},
-		partial: func(entries metaCacheEntries, _ []error) {
-			entry, ok := entries.resolve(&resolver)
-			if !ok {
-				// check if we can get one entry atleast
-				// proceed to heal nonetheless.
-				entry, _ = entries.firstFound()
-			}
-
-			if err := healEntry(bucket, *entry); err != nil {
-				logger.LogIf(ctx, err)
-				cancel()
-				return
-			}
-		},
-		finished: nil,
-	}
-
-	if err := listPathRaw(ctx, lopts); err != nil {
-		return fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)
-	}
-
-	return nil
-}
-
 func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObjectFn HealObjectFn) error {
 	healEntry := func(bucket string, entry metaCacheEntry) error {
 		if entry.isDir() {
@@ -2019,7 +1963,7 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 			go func(idx int, set *erasureObjects) {
 				defer wg.Done()
 
-				errs[idx] = listAndHeal(ctx, bucket, prefix, set, healEntry)
+				errs[idx] = set.listAndHeal(bucket, prefix, healEntry)
 			}(idx, set)
 		}
 		wg.Wait()

cmd/erasure-sets.go

@@ -33,7 +33,7 @@ import (
 	"github.com/dchest/siphash"
 	"github.com/dustin/go-humanize"
 	"github.com/google/uuid"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bpool"

cmd/erasure.go

@@ -28,7 +28,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bpool"
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/logger"
@@ -176,7 +176,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 		g.Go(func() error {
 			diskEndpoint := endpoints[index].String()
 			if disks[index] == OfflineDisk {
-				logger.LogIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]))
+				logger.LogOnceIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]), "get-disks-info-offline"+diskEndpoint)
 				disksInfo[index] = madmin.Disk{
 					State:    diskErrToDriveState(errDiskNotFound),
 					Endpoint: diskEndpoint,
@@ -410,7 +410,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				if cache.Info.LastUpdate.Equal(lastSave) {
 					continue
 				}
-				logger.LogIf(ctx, cache.save(ctx, er, dataUsageCacheName))
+				logger.LogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-cache-update")
 				updates <- cache.clone()
 				lastSave = cache.Info.LastUpdate
 			case v, ok := <-bucketResults:
@@ -418,7 +418,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 					// Save final state...
 					cache.Info.NextCycle = wantCycle
 					cache.Info.LastUpdate = time.Now()
-					logger.LogIf(ctx, cache.save(ctx, er, dataUsageCacheName))
+					logger.LogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-channel-closed")
 					updates <- cache
 					return
 				}

cmd/etcd.go

@@ -77,7 +77,7 @@ func readKeyEtcd(ctx context.Context, client *etcd.Client, key string) ([]byte,
 	defer cancel()
 	resp, err := client.Get(timeoutCtx, key)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, "etcd-retrieve-keys")
 		return nil, etcdErrToErr(err, client.Endpoints())
 	}
 	if resp.Count == 0 {

cmd/event-notification.go

@@ -35,20 +35,18 @@ import (
 // EventNotifier - notifies external systems about events in MinIO.
 type EventNotifier struct {
 	sync.RWMutex
-	targetList                 *event.TargetList
-	targetResCh                chan event.TargetIDResult
-	bucketRulesMap             map[string]event.RulesMap
-	bucketRemoteTargetRulesMap map[string]map[event.TargetID]event.RulesMap
+	targetList     *event.TargetList
+	targetResCh    chan event.TargetIDResult
+	bucketRulesMap map[string]event.RulesMap
 }
 
 // NewEventNotifier - creates new event notification object.
 func NewEventNotifier() *EventNotifier {
 	// targetList/bucketRulesMap/bucketRemoteTargetRulesMap are populated by NotificationSys.InitBucketTargets()
 	return &EventNotifier{
-		targetList:                 event.NewTargetList(),
-		targetResCh:                make(chan event.TargetIDResult),
-		bucketRulesMap:             make(map[string]event.RulesMap),
-		bucketRemoteTargetRulesMap: make(map[string]map[event.TargetID]event.RulesMap),
+		targetList:     event.NewTargetList(),
+		targetResCh:    make(chan event.TargetIDResult),
+		bucketRulesMap: make(map[string]event.RulesMap),
 	}
 }
 
@@ -122,10 +120,6 @@ func (evnot *EventNotifier) AddRulesMap(bucketName string, rulesMap event.RulesM
 
 	rulesMap = rulesMap.Clone()
 
-	for _, targetRulesMap := range evnot.bucketRemoteTargetRulesMap[bucketName] {
-		rulesMap.Add(targetRulesMap)
-	}
-
 	// Do not add for an empty rulesMap.
 	if len(rulesMap) == 0 {
 		delete(evnot.bucketRulesMap, bucketName)
@@ -174,29 +168,18 @@ func (evnot *EventNotifier) RemoveNotification(bucketName string) {
 	defer evnot.Unlock()
 
 	delete(evnot.bucketRulesMap, bucketName)
-
-	targetIDSet := event.NewTargetIDSet()
-	for targetID := range evnot.bucketRemoteTargetRulesMap[bucketName] {
-		targetIDSet[targetID] = struct{}{}
-		delete(evnot.bucketRemoteTargetRulesMap[bucketName], targetID)
-	}
-	evnot.targetList.Remove(targetIDSet)
-
-	delete(evnot.bucketRemoteTargetRulesMap, bucketName)
 }
 
-// RemoveAllRemoteTargets - closes and removes all notification targets.
-func (evnot *EventNotifier) RemoveAllRemoteTargets() {
+// RemoveAllBucketTargets - closes and removes all notification targets.
+func (evnot *EventNotifier) RemoveAllBucketTargets() {
 	evnot.Lock()
 	defer evnot.Unlock()
 
-	for _, targetMap := range evnot.bucketRemoteTargetRulesMap {
-		targetIDSet := event.NewTargetIDSet()
-		for k := range targetMap {
-			targetIDSet[k] = struct{}{}
-		}
-		evnot.targetList.Remove(targetIDSet)
+	targetIDSet := event.NewTargetIDSet()
+	for k := range evnot.targetList.TargetMap() {
+		targetIDSet[k] = struct{}{}
 	}
+	evnot.targetList.Remove(targetIDSet)
 }
 
 // Send - sends the event to all registered notification targets
@@ -209,7 +192,8 @@ func (evnot *EventNotifier) Send(args eventArgs) {
 		return
 	}
 
-	evnot.targetList.Send(args.ToEvent(true), targetIDSet, evnot.targetResCh)
+	// If MINIO_API_SYNC_EVENTS is set, send events synchronously.
+	evnot.targetList.Send(args.ToEvent(true), targetIDSet, evnot.targetResCh, globalAPIConfig.isSyncEventsEnabled())
 }
 
 type eventArgs struct {

cmd/ftp-server-driver.go

@@ -28,7 +28,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio/internal/auth"
@@ -253,11 +253,7 @@ func (driver *ftpDriver) CheckPasswd(c *ftp.Context, username, password string)
 			return false, err
 		}
 		ldapPolicies, _ := globalIAMSys.PolicyDBGet(ldapUserDN, false, groupDistNames...)
-		if len(ldapPolicies) == 0 {
-			// no policy associated reject it.
-			return false, nil
-		}
-		return true, nil
+		return len(ldapPolicies) > 0, nil
 	}
 
 	ui, ok := globalIAMSys.GetUser(context.Background(), username)
@@ -362,12 +358,20 @@ func (driver *ftpDriver) DeleteDir(ctx *ftp.Context, path string) (err error) {
 	cctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	if prefix == "" {
+		// if all objects are not deleted yet this call may fail.
+		return clnt.RemoveBucket(cctx, bucket)
+	}
+
 	objectsCh := make(chan minio.ObjectInfo)
 
 	// Send object names that are needed to be removed to objectsCh
 	go func() {
 		defer close(objectsCh)
-		opts := minio.ListObjectsOptions{Prefix: prefix, Recursive: true}
+		opts := minio.ListObjectsOptions{
+			Prefix:    prefix,
+			Recursive: true,
+		}
 		for object := range clnt.ListObjects(cctx, bucket, opts) {
 			if object.Err != nil {
 				return
@@ -427,6 +431,10 @@ func (driver *ftpDriver) MakeDir(ctx *ftp.Context, path string) (err error) {
 		return err
 	}
 
+	if prefix == "" {
+		return clnt.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{Region: globalSite.Region})
+	}
+
 	dirPath := buildMinioDir(prefix)
 
 	_, err = clnt.PutObject(context.Background(), bucket, dirPath, bytes.NewReader([]byte("")), 0,

cmd/ftp-server.go

@@ -129,10 +129,29 @@ func startSFTPServer(c *cli.Context) {
 	// certificate details and handles authentication of ServerConns.
 	config := &ssh.ServerConfig{
 		PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
+			if globalIAMSys.LDAPConfig.Enabled() {
+				targetUser, targetGroups, err := globalIAMSys.LDAPConfig.Bind(c.User(), string(pass))
+				if err != nil {
+					return nil, err
+				}
+				ldapPolicies, _ := globalIAMSys.PolicyDBGet(targetUser, false, targetGroups...)
+				if len(ldapPolicies) == 0 {
+					return nil, errAuthentication
+				}
+				return &ssh.Permissions{
+					CriticalOptions: map[string]string{
+						ldapUser:  targetUser,
+						ldapUserN: c.User(),
+					},
+					Extensions: make(map[string]string),
+				}, nil
+			}
+
 			ui, ok := globalIAMSys.GetUser(context.Background(), c.User())
 			if !ok {
 				return nil, errNoSuchUser
 			}
+
 			if subtle.ConstantTimeCompare([]byte(ui.Credentials.SecretKey), pass) == 1 {
 				return &ssh.Permissions{
 					CriticalOptions: map[string]string{

cmd/global-heal.go

@@ -24,7 +24,7 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/logger"

cmd/globals.go

@@ -27,7 +27,7 @@ import (
 	"time"
 
 	"github.com/minio/console/restapi"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/bucket/bandwidth"
@@ -224,12 +224,15 @@ var (
 	// registered listeners
 	globalConsoleSys *HTTPConsoleLoggerSys
 
+	// All unique drives for this deployment
 	globalEndpoints EndpointServerPools
+	// All unique nodes for this deployment
+	globalNodes []Node
 
 	// The name of this local node, fetched from arguments
 	globalLocalNodeName    string
 	globalLocalNodeNameHex string
-	globalNodeNamesHex     map[string]struct{}
+	globalNodeNamesHex     = make(map[string]struct{})
 
 	// The global subnet config
 	globalSubnetConfig subnet.Config
@@ -237,16 +240,15 @@ var (
 	// The global callhome config
 	globalCallhomeConfig callhome.Config
 
-	globalRemoteEndpoints map[string]Endpoint
-
 	// Global server's network statistics
 	globalConnStats = newConnStats()
 
 	// Global HTTP request statisitics
 	globalHTTPStats = newHTTPStats()
 
-	// Global bucket network statistics
+	// Global bucket network and API statistics
 	globalBucketConnStats = newBucketConnStats()
+	globalBucketHTTPStats = newBucketHTTPStats()
 
 	// Time when the server is started
 	globalBootTime = UTCNow()

cmd/handler-api.go

@@ -52,6 +52,7 @@ type apiConfig struct {
 	disableODirect              bool
 	gzipObjects                 bool
 	rootAccess                  bool
+	syncEvents                  bool
 }
 
 const cgroupLimitFile = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
@@ -166,6 +167,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	t.disableODirect = cfg.DisableODirect
 	t.gzipObjects = cfg.GzipObjects
 	t.rootAccess = cfg.RootAccess
+	t.syncEvents = cfg.SyncEvents
 }
 
 func (t *apiConfig) isDisableODirect() bool {
@@ -353,3 +355,10 @@ func (t *apiConfig) getTransitionWorkers() int {
 
 	return t.transitionWorkers
 }
+
+func (t *apiConfig) isSyncEventsEnabled() bool {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	return t.syncEvents
+}

cmd/handler-utils.go

@@ -26,7 +26,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
@@ -48,7 +48,7 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError
 	locationConstraint := createBucketLocationConfiguration{}
 	err := xmlDecoder(r.Body, &locationConstraint, r.ContentLength)
 	if err != nil && r.ContentLength != 0 {
-		logger.LogIf(GlobalContext, err)
+		logger.LogOnceIf(GlobalContext, err, "location-constraint-xml-parsing")
 		// Treat all other failures as XML parsing errors.
 		return "", ErrMalformedXML
 	} // else for both err as nil or io.EOF
@@ -269,9 +269,26 @@ func trimAwsChunkedContentEncoding(contentEnc string) (trimmedContentEnc string)
 
 func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
+		resource, err := getResource(r.URL.Path, r.Host, globalDomainNames)
+		if err != nil {
+			defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
+
+			apiErr := errorCodes.ToAPIErr(ErrUnsupportedHostHeader)
+			apiErr.Description = fmt.Sprintf("%s: %v", apiErr.Description, err)
+
+			writeErrorResponse(r.Context(), w, apiErr, r.URL)
+			return
+		}
+
+		bucket, _ := path2BucketObject(resource)
+
 		globalHTTPStats.currentS3Requests.Inc(api)
 		defer globalHTTPStats.currentS3Requests.Dec(api)
 
+		if bucket != "" && bucket != minioReservedBucket {
+			globalBucketHTTPStats.updateHTTPStats(bucket, api, nil)
+		}
+
 		f.ServeHTTP(w, r)
 
 		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
@@ -299,16 +316,10 @@ func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 			globalConnStats.incS3InputBytes(int64(tc.RequestRecorder.Size()))
 			globalConnStats.incS3OutputBytes(int64(tc.ResponseRecorder.Size()))
 
-			resource, err := getResource(r.URL.Path, r.Host, globalDomainNames)
-			if err != nil {
-				logger.LogIf(r.Context(), fmt.Errorf("Unable to get the actual resource in the incoming request: %v", err))
-				return
-			}
-
-			bucket, _ := path2BucketObject(resource)
 			if bucket != "" && bucket != minioReservedBucket {
 				globalBucketConnStats.incS3InputBytes(bucket, int64(tc.RequestRecorder.Size()))
 				globalBucketConnStats.incS3OutputBytes(bucket, int64(tc.ResponseRecorder.Size()))
+				globalBucketHTTPStats.updateHTTPStats(bucket, api, tc.ResponseRecorder)
 			}
 		}
 	}

cmd/http-stats.go

@@ -120,6 +120,105 @@ type bucketS3RXTX struct {
 	s3OutputBytes uint64
 }
 
+type bucketHTTPAPIStats struct {
+	currentS3Requests *HTTPAPIStats
+	totalS3Requests   *HTTPAPIStats
+	totalS34xxErrors  *HTTPAPIStats
+	totalS35xxErrors  *HTTPAPIStats
+	totalS3Canceled   *HTTPAPIStats
+}
+
+type bucketHTTPStats struct {
+	sync.RWMutex
+	httpStats map[string]bucketHTTPAPIStats
+}
+
+func newBucketHTTPStats() *bucketHTTPStats {
+	return &bucketHTTPStats{
+		httpStats: make(map[string]bucketHTTPAPIStats),
+	}
+}
+
+func (bh *bucketHTTPStats) delete(bucket string) {
+	bh.Lock()
+	defer bh.Unlock()
+
+	delete(bh.httpStats, bucket)
+}
+
+func (bh *bucketHTTPStats) updateHTTPStats(bucket, api string, w *xhttp.ResponseRecorder) {
+	if bh == nil {
+		return
+	}
+
+	bh.Lock()
+	defer bh.Unlock()
+
+	hstats, ok := bh.httpStats[bucket]
+	if !ok {
+		hstats = bucketHTTPAPIStats{
+			currentS3Requests: &HTTPAPIStats{},
+			totalS3Requests:   &HTTPAPIStats{},
+			totalS3Canceled:   &HTTPAPIStats{},
+			totalS34xxErrors:  &HTTPAPIStats{},
+			totalS35xxErrors:  &HTTPAPIStats{},
+		}
+	}
+
+	if w == nil { // when response recorder nil, this is an active request
+		hstats.currentS3Requests.Inc(api)
+		bh.httpStats[bucket] = hstats
+		return
+	} // else {
+	hstats.currentS3Requests.Dec(api) // decrement this once we have the response recorder.
+
+	hstats.totalS3Requests.Inc(api)
+	code := w.StatusCode
+
+	switch {
+	case code == 0:
+	case code == 499:
+		// 499 is a good error, shall be counted as canceled.
+		hstats.totalS3Canceled.Inc(api)
+	case code >= http.StatusBadRequest:
+		if code >= http.StatusInternalServerError {
+			hstats.totalS35xxErrors.Inc(api)
+		} else {
+			hstats.totalS34xxErrors.Inc(api)
+		}
+	}
+
+	bh.httpStats[bucket] = hstats
+}
+
+func (bh *bucketHTTPStats) load(bucket string) bucketHTTPAPIStats {
+	if bh == nil {
+		return bucketHTTPAPIStats{
+			currentS3Requests: &HTTPAPIStats{},
+			totalS3Requests:   &HTTPAPIStats{},
+			totalS3Canceled:   &HTTPAPIStats{},
+			totalS34xxErrors:  &HTTPAPIStats{},
+			totalS35xxErrors:  &HTTPAPIStats{},
+		}
+	}
+
+	bh.RLock()
+	defer bh.RUnlock()
+
+	val, ok := bh.httpStats[bucket]
+	if ok {
+		return val
+	}
+
+	return bucketHTTPAPIStats{
+		currentS3Requests: &HTTPAPIStats{},
+		totalS3Requests:   &HTTPAPIStats{},
+		totalS3Canceled:   &HTTPAPIStats{},
+		totalS34xxErrors:  &HTTPAPIStats{},
+		totalS35xxErrors:  &HTTPAPIStats{},
+	}
+}
+
 type bucketConnStats struct {
 	sync.RWMutex
 	stats map[string]*bucketS3RXTX
@@ -225,10 +324,32 @@ func (stats *HTTPAPIStats) Dec(api string) {
 	}
 }
 
+// Get returns the current counter on input API string
+func (stats *HTTPAPIStats) Get(api string) int {
+	if stats == nil {
+		return 0
+	}
+
+	stats.RLock()
+	defer stats.RUnlock()
+
+	val, ok := stats.apiStats[api]
+	if ok {
+		return val
+	}
+
+	return 0
+}
+
 // Load returns the recorded stats.
 func (stats *HTTPAPIStats) Load() map[string]int {
-	stats.Lock()
-	defer stats.Unlock()
+	if stats == nil {
+		return map[string]int{}
+	}
+
+	stats.RLock()
+	defer stats.RUnlock()
+
 	apiStats := make(map[string]int, len(stats.apiStats))
 	for k, v := range stats.apiStats {
 		apiStats[k] = v

cmd/http-tracer.go

@@ -28,7 +28,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/mcontext"

cmd/iam-object-store.go

@@ -28,7 +28,7 @@ import (
 	"unicode/utf8"
 
 	jsoniter "github.com/json-iterator/go"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"

cmd/iam-store.go

@@ -28,7 +28,7 @@ import (
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/identity/openid"
@@ -914,11 +914,12 @@ func (store *IAMStoreSys) listGroups(ctx context.Context) (res []string, err err
 // PolicyDBUpdate - adds or removes given policies to/from the user or group's
 // policy associations.
 func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGroup bool,
-	userType IAMUserType, policies []string, isAttach bool) (updatedAt time.Time, addedOrRemoved []string,
-	err error,
+	userType IAMUserType, policies []string, isAttach bool) (updatedAt time.Time,
+	addedOrRemoved, effectivePolicies []string, err error,
 ) {
 	if name == "" {
-		return updatedAt, nil, errInvalidArgument
+		err = errInvalidArgument
+		return
 	}
 
 	cache := store.lock()
@@ -932,11 +933,13 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
 		if store.getUsersSysType() == MinIOUsersSysType {
 			g, ok := cache.iamGroupsMap[name]
 			if !ok {
-				return updatedAt, nil, errNoSuchGroup
+				err = errNoSuchGroup
+				return
 			}
 
 			if g.Status == statusDisabled {
-				return updatedAt, nil, errGroupDisabled
+				err = errGroupDisabled
+				return
 			}
 		}
 		mp = cache.iamGroupPolicyMap[name]
@@ -945,6 +948,7 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
 	// Compute net policy change effect and updated policy mapping
 	existingPolicySet := mp.policySet()
 	policiesToUpdate := set.CreateStringSet(policies...)
+	var newPolicySet set.StringSet
 	newPolicyMapping := mp
 	if isAttach {
 		// new policies to attach => inputPolicies - existing (set difference)
@@ -952,26 +956,29 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
 		// validate that new policies to add are defined.
 		for _, p := range policiesToUpdate.ToSlice() {
 			if _, found := cache.iamPolicyDocsMap[p]; !found {
-				return updatedAt, nil, errNoSuchPolicy
+				err = errNoSuchPolicy
+				return
 			}
 		}
-		newPolicyMapping.Policies = strings.Join(existingPolicySet.Union(policiesToUpdate).ToSlice(), ",")
+		newPolicySet = existingPolicySet.Union(policiesToUpdate)
 	} else {
 		// policies to detach => inputPolicies ∩ existing (intersection)
 		policiesToUpdate = policiesToUpdate.Intersection(existingPolicySet)
-		newPolicyMapping.Policies = strings.Join(existingPolicySet.Difference(policiesToUpdate).ToSlice(), ",")
+		newPolicySet = existingPolicySet.Difference(policiesToUpdate)
 	}
-	newPolicyMapping.UpdatedAt = UTCNow()
-
 	// We return an error if the requested policy update will have no effect.
 	if policiesToUpdate.IsEmpty() {
-		return updatedAt, nil, errNoPolicyToAttachOrDetach
+		err = errNoPolicyToAttachOrDetach
+		return
 	}
 
+	newPolicies := newPolicySet.ToSlice()
+	newPolicyMapping.Policies = strings.Join(newPolicies, ",")
+	newPolicyMapping.UpdatedAt = UTCNow()
 	addedOrRemoved = policiesToUpdate.ToSlice()
 
-	if err := store.saveMappedPolicy(ctx, name, userType, isGroup, newPolicyMapping); err != nil {
-		return updatedAt, addedOrRemoved, err
+	if err = store.saveMappedPolicy(ctx, name, userType, isGroup, newPolicyMapping); err != nil {
+		return
 	}
 	if !isGroup {
 		cache.iamUserPolicyMap[name] = newPolicyMapping
@@ -979,7 +986,7 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
 		cache.iamGroupPolicyMap[name] = newPolicyMapping
 	}
 	cache.updatedAt = UTCNow()
-	return cache.updatedAt, addedOrRemoved, nil
+	return cache.updatedAt, addedOrRemoved, newPolicies, nil
 }
 
 // PolicyDBSet - update the policy mapping for the given user or group in
@@ -1451,23 +1458,6 @@ func (store *IAMStoreSys) GetUserInfo(name string) (u madmin.UserInfo, err error
 	}, nil
 }
 
-// GetUserPolicies - returns the policies attached to a user.
-func (store *IAMStoreSys) GetUserPolicies(name string) ([]string, error) {
-	if name == "" {
-		return nil, errInvalidArgument
-	}
-
-	cache := store.rlock()
-	defer store.runlock()
-
-	if cache.iamUserPolicyMap[name].Policies == "" {
-		return []string{}, nil
-	}
-
-	policies := cache.iamUserPolicyMap[name].toSlice()
-	return policies, nil
-}
-
 // PolicyMappingNotificationHandler - handles updating a policy mapping from storage.
 func (store *IAMStoreSys) PolicyMappingNotificationHandler(ctx context.Context, userOrGroup string, isGroup bool, userType IAMUserType) error {
 	if userOrGroup == "" {

cmd/iam.go

@@ -32,7 +32,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/arn"
 	"github.com/minio/minio/internal/auth"
@@ -813,7 +813,7 @@ func (sys *IAMSys) QueryLDAPPolicyEntities(ctx context.Context, q madmin.PolicyE
 	}
 }
 
-// IsTempUser - returns if given key is a temporary user.
+// IsTempUser - returns if given key is a temporary user and parent user.
 func (sys *IAMSys) IsTempUser(name string) (bool, string, error) {
 	if !sys.Initialized() {
 		return false, "", errServerNotInitialized
@@ -889,15 +889,6 @@ func (sys *IAMSys) QueryPolicyEntities(ctx context.Context, q madmin.PolicyEntit
 	}
 }
 
-// GetUserPolicies - get policies attached to a user.
-func (sys *IAMSys) GetUserPolicies(name string) (p []string, err error) {
-	if !sys.Initialized() {
-		return p, errServerNotInitialized
-	}
-
-	return sys.store.GetUserPolicies(name)
-}
-
 // SetUserStatus - sets current user status, supports disabled or enabled.
 func (sys *IAMSys) SetUserStatus(ctx context.Context, accessKey string, status madmin.AccountStatus) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
@@ -1582,13 +1573,93 @@ func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, userTyp
 	return updatedAt, nil
 }
 
+// PolicyDBUpdateBuiltin - adds or removes policies from a user or a group
+// verified to be an internal IDP user.
+func (sys *IAMSys) PolicyDBUpdateBuiltin(ctx context.Context, isAttach bool,
+	r madmin.PolicyAssociationReq,
+) (updatedAt time.Time, addedOrRemoved, effectivePolicies []string, err error) {
+	if !sys.Initialized() {
+		err = errServerNotInitialized
+		return
+	}
+
+	userOrGroup := r.User
+	var isGroup bool
+	if userOrGroup == "" {
+		isGroup = true
+		userOrGroup = r.Group
+	}
+
+	if isGroup {
+		_, err = sys.GetGroupDescription(userOrGroup)
+		if err != nil {
+			return
+		}
+	} else {
+		var isTemp bool
+		isTemp, _, err = sys.IsTempUser(userOrGroup)
+		if err != nil && err != errNoSuchUser {
+			return
+		}
+		if isTemp {
+			err = errIAMActionNotAllowed
+			return
+		}
+
+		// When the user is root credential you are not allowed to
+		// add policies for root user.
+		if userOrGroup == globalActiveCred.AccessKey {
+			err = errIAMActionNotAllowed
+			return
+		}
+
+		// Validate that user exists.
+		var userExists bool
+		_, userExists = sys.GetUser(ctx, userOrGroup)
+		if !userExists {
+			err = errNoSuchUser
+			return
+		}
+	}
+
+	updatedAt, addedOrRemoved, effectivePolicies, err = sys.store.PolicyDBUpdate(ctx, userOrGroup, isGroup,
+		regUser, r.Policies, isAttach)
+	if err != nil {
+		return
+	}
+
+	// Notify all other MinIO peers to reload policy
+	if !sys.HasWatcher() {
+		for _, nerr := range globalNotificationSys.LoadPolicyMapping(userOrGroup, regUser, isGroup) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
+		}
+	}
+
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemPolicyMapping,
+		PolicyMapping: &madmin.SRPolicyMapping{
+			UserOrGroup: userOrGroup,
+			UserType:    int(regUser),
+			IsGroup:     isGroup,
+			Policy:      strings.Join(effectivePolicies, ","),
+		},
+		UpdatedAt: updatedAt,
+	}))
+
+	return
+}
+
 // PolicyDBUpdateLDAP - adds or removes policies from a user or a group verified
 // to be in the LDAP directory.
 func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
 	r madmin.PolicyAssociationReq,
-) (updatedAt time.Time, addedOrRemoved []string, err error) {
+) (updatedAt time.Time, addedOrRemoved, effectivePolicies []string, err error) {
 	if !sys.Initialized() {
-		return updatedAt, nil, errServerNotInitialized
+		err = errServerNotInitialized
+		return
 	}
 
 	var dn string
@@ -1597,28 +1668,31 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
 		dn, err = sys.LDAPConfig.DoesUsernameExist(r.User)
 		if err != nil {
 			logger.LogIf(ctx, err)
-			return updatedAt, nil, err
+			return
 		}
 		if dn == "" {
-			return updatedAt, nil, errNoSuchUser
+			err = errNoSuchUser
+			return
 		}
 		isGroup = false
 	} else {
-		if exists, err := sys.LDAPConfig.DoesGroupDNExist(r.Group); err != nil {
+		var exists bool
+		if exists, err = sys.LDAPConfig.DoesGroupDNExist(r.Group); err != nil {
 			logger.LogIf(ctx, err)
-			return updatedAt, nil, err
+			return
 		} else if !exists {
-			return updatedAt, nil, errNoSuchGroup
+			err = errNoSuchGroup
+			return
 		}
 		dn = r.Group
 		isGroup = true
 	}
 
 	userType := stsUser
-	updatedAt, addedOrRemoved, err = sys.store.PolicyDBUpdate(ctx, dn, isGroup,
+	updatedAt, addedOrRemoved, effectivePolicies, err = sys.store.PolicyDBUpdate(ctx, dn, isGroup,
 		userType, r.Policies, isAttach)
 	if err != nil {
-		return updatedAt, nil, err
+		return
 	}
 
 	// Notify all other MinIO peers to reload policy
@@ -1631,7 +1705,18 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
 		}
 	}
 
-	return updatedAt, addedOrRemoved, nil
+	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemPolicyMapping,
+		PolicyMapping: &madmin.SRPolicyMapping{
+			UserOrGroup: dn,
+			UserType:    int(userType),
+			IsGroup:     isGroup,
+			Policy:      strings.Join(effectivePolicies, ","),
+		},
+		UpdatedAt: updatedAt,
+	}))
+
+	return
 }
 
 // PolicyDBGet - gets policy set on a user or group. If a list of groups is

cmd/kms-handlers.go

@@ -24,7 +24,7 @@ import (
 	"time"
 
 	"github.com/minio/kes-go"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"

cmd/last-minute.go

@@ -22,7 +22,7 @@ package cmd
 import (
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 )
 
 const (

cmd/metacache-walk.go

@@ -149,7 +149,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 		if err != nil {
 			// Folder could have gone away in-between
 			if err != errVolumeNotFound && err != errFileNotFound {
-				logger.LogIf(ctx, err)
+				logger.LogOnceIf(ctx, err, "metacache-walk-scan-dir")
 			}
 			if opts.ReportNotFound && err == errFileNotFound && current == opts.BaseDir {
 				return errFileNotFound
@@ -211,7 +211,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 					// while being concurrently listed at the same time in
 					// such scenarios the 'xl.meta' might get truncated
 					if !IsErrIgnored(err, io.EOF, io.ErrUnexpectedEOF) {
-						logger.LogIf(ctx, err)
+						logger.LogOnceIf(ctx, err, "metacache-walk-read-metadata")
 					}
 					continue
 				}

cmd/metrics-realtime.go

@@ -21,7 +21,7 @@ import (
 	"context"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/disk"
 )
 

cmd/metrics-v2.go

@@ -28,7 +28,7 @@ import (
 	"time"
 
 	"github.com/minio/kes-go"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/mcontext"
@@ -483,7 +483,7 @@ func getUsageLastScanActivityMD() MetricDescription {
 		Namespace: minioMetricNamespace,
 		Subsystem: usageSubsystem,
 		Name:      lastActivityTime,
-		Help:      "Time elapsed (in nano seconds) since last scan activity. This is set to 0 until first scan cycle",
+		Help:      "Time elapsed (in nano seconds) since last scan activity.",
 		Type:      gaugeMetric,
 	}
 }
@@ -703,7 +703,7 @@ func getS3RequestsInQueueMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsSubsystem,
 		Name:      waitingTotal,
-		Help:      "Number of S3 requests in the waiting queue",
+		Help:      "Total number of S3 requests in the waiting queue",
 		Type:      gaugeMetric,
 	}
 }
@@ -713,7 +713,7 @@ func getIncomingS3RequestsMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsSubsystem,
 		Name:      incomingTotal,
-		Help:      "Volatile number of total incoming S3 requests",
+		Help:      "Total number of incoming S3 requests",
 		Type:      gaugeMetric,
 	}
 }
@@ -723,7 +723,7 @@ func getS3RequestsTotalMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsSubsystem,
 		Name:      total,
-		Help:      "Total number S3 requests",
+		Help:      "Total number of S3 requests",
 		Type:      counterMetric,
 	}
 }
@@ -733,7 +733,7 @@ func getS3RequestsErrorsMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsSubsystem,
 		Name:      errorsTotal,
-		Help:      "Total number S3 requests with (4xx and 5xx) errors",
+		Help:      "Total number of S3 requests with (4xx and 5xx) errors",
 		Type:      counterMetric,
 	}
 }
@@ -743,7 +743,7 @@ func getS3Requests4xxErrorsMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsSubsystem,
 		Name:      "4xx_" + errorsTotal,
-		Help:      "Total number S3 requests with (4xx) errors",
+		Help:      "Total number of S3 requests with (4xx) errors",
 		Type:      counterMetric,
 	}
 }
@@ -753,7 +753,7 @@ func getS3Requests5xxErrorsMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsSubsystem,
 		Name:      "5xx_" + errorsTotal,
-		Help:      "Total number S3 requests with (5xx) errors",
+		Help:      "Total number of S3 requests with (5xx) errors",
 		Type:      counterMetric,
 	}
 }
@@ -763,7 +763,7 @@ func getS3RequestsCanceledMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsSubsystem,
 		Name:      canceledTotal,
-		Help:      "Total number S3 requests that were canceled from the client while processing",
+		Help:      "Total number of S3 requests that were canceled by the client",
 		Type:      counterMetric,
 	}
 }
@@ -773,7 +773,7 @@ func getS3RejectedAuthRequestsTotalMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsRejectedSubsystem,
 		Name:      authTotal,
-		Help:      "Total number S3 requests rejected for auth failure",
+		Help:      "Total number of S3 requests rejected for auth failure",
 		Type:      counterMetric,
 	}
 }
@@ -783,7 +783,7 @@ func getS3RejectedHeaderRequestsTotalMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsRejectedSubsystem,
 		Name:      headerTotal,
-		Help:      "Total number S3 requests rejected for invalid header",
+		Help:      "Total number of S3 requests rejected for invalid header",
 		Type:      counterMetric,
 	}
 }
@@ -793,7 +793,7 @@ func getS3RejectedTimestampRequestsTotalMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsRejectedSubsystem,
 		Name:      timestampTotal,
-		Help:      "Total number S3 requests rejected for invalid timestamp",
+		Help:      "Total number of S3 requests rejected for invalid timestamp",
 		Type:      counterMetric,
 	}
 }
@@ -803,7 +803,7 @@ func getS3RejectedInvalidRequestsTotalMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsRejectedSubsystem,
 		Name:      invalidTotal,
-		Help:      "Total number S3 invalid requests",
+		Help:      "Total number of invalid S3 requests",
 		Type:      counterMetric,
 	}
 }
@@ -913,7 +913,7 @@ func getHealLastActivityTimeMD() MetricDescription {
 		Namespace: healMetricNamespace,
 		Subsystem: timeSubsystem,
 		Name:      lastActivityTime,
-		Help:      "Time elapsed (in nano seconds) since last self healing activity. This is set to -1 until initial self heal activity",
+		Help:      "Time elapsed (in nano seconds) since last self healing activity.",
 		Type:      gaugeMetric,
 	}
 }
@@ -1309,6 +1309,56 @@ func getExpiryPendingTasksMD() MetricDescription {
 	}
 }
 
+func getBucketS3RequestsInFlightMD() MetricDescription {
+	return MetricDescription{
+		Namespace: bucketMetricNamespace,
+		Subsystem: requestsSubsystem,
+		Name:      inflightTotal,
+		Help:      "Total number of S3 requests currently in flight on a bucket",
+		Type:      gaugeMetric,
+	}
+}
+
+func getBucketS3RequestsTotalMD() MetricDescription {
+	return MetricDescription{
+		Namespace: bucketMetricNamespace,
+		Subsystem: requestsSubsystem,
+		Name:      total,
+		Help:      "Total number of S3 requests on a bucket",
+		Type:      counterMetric,
+	}
+}
+
+func getBucketS3Requests4xxErrorsMD() MetricDescription {
+	return MetricDescription{
+		Namespace: bucketMetricNamespace,
+		Subsystem: requestsSubsystem,
+		Name:      "4xx_" + errorsTotal,
+		Help:      "Total number of S3 requests with (4xx) errors on a bucket",
+		Type:      counterMetric,
+	}
+}
+
+func getBucketS3Requests5xxErrorsMD() MetricDescription {
+	return MetricDescription{
+		Namespace: bucketMetricNamespace,
+		Subsystem: requestsSubsystem,
+		Name:      "5xx_" + errorsTotal,
+		Help:      "Total number of S3 requests with (5xx) errors on a bucket",
+		Type:      counterMetric,
+	}
+}
+
+func getBucketS3RequestsCanceledMD() MetricDescription {
+	return MetricDescription{
+		Namespace: bucketMetricNamespace,
+		Subsystem: requestsSubsystem,
+		Name:      canceledTotal,
+		Help:      "Total number of S3 requests that were canceled from the client while processing on a bucket",
+		Type:      counterMetric,
+	}
+}
+
 func getILMNodeMetrics() *MetricsGroup {
 	mg := &MetricsGroup{
 		cacheInterval: 10 * time.Second,
@@ -1456,7 +1506,7 @@ func getIAMNodeMetrics() *MetricsGroup {
 					Namespace: nodeMetricNamespace,
 					Subsystem: iamSubsystem,
 					Name:      "since_last_sync_millis",
-					Help:      "Time (in milliseconds) since last successful IAM data sync. This is set to 0 until the first sync after server start.",
+					Help:      "Time (in milliseconds) since last successful IAM data sync.",
 					Type:      gaugeMetric,
 				},
 				Value: float64(sinceLastSyncMillis),
@@ -2072,6 +2122,47 @@ func getBucketUsageMetrics() *MetricsGroup {
 				})
 			}
 
+			httpStats := globalBucketHTTPStats.load(bucket)
+			for k, v := range httpStats.currentS3Requests.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3RequestsInFlightMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS3Requests.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3RequestsTotalMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS3Canceled.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3RequestsCanceledMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS34xxErrors.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3Requests4xxErrorsMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS35xxErrors.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3Requests5xxErrorsMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
 			if stats.hasReplicationUsage() {
 				for arn, stat := range stats.Stats {
 					metrics = append(metrics, Metric{

cmd/mrf.go

@@ -22,7 +22,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 )
 
 const (
@@ -32,17 +32,19 @@ const (
 // partialOperation is a successful upload/delete of an object
 // but not written in all disks (having quorum)
 type partialOperation struct {
-	bucket    string
-	object    string
-	versionID string
-	queued    time.Time
+	bucket              string
+	object              string
+	versionID           string
+	allVersions         bool
+	setIndex, poolIndex int
+	queued              time.Time
 }
 
 // mrfState sncapsulates all the information
 // related to the global background MRF.
 type mrfState struct {
-	ctx       context.Context
-	objectAPI ObjectLayer
+	ctx   context.Context
+	pools *erasureServerPools
 
 	mu   sync.Mutex
 	opCh chan partialOperation
@@ -55,9 +57,12 @@ func (m *mrfState) init(ctx context.Context, objAPI ObjectLayer) {
 
 	m.ctx = ctx
 	m.opCh = make(chan partialOperation, mrfOpsQueueSize)
-	m.objectAPI = objAPI
 
-	go globalMRFState.healRoutine()
+	var ok bool
+	m.pools, ok = objAPI.(*erasureServerPools)
+	if ok {
+		go m.healRoutine()
+	}
 }
 
 // Add a partial S3 operation (put/delete) when one or more disks are offline.
@@ -101,7 +106,11 @@ func (m *mrfState) healRoutine() {
 			if u.object == "" {
 				healBucket(u.bucket, madmin.HealNormalScan)
 			} else {
-				healObject(u.bucket, u.object, u.versionID, madmin.HealNormalScan)
+				if u.allVersions {
+					m.pools.serverPools[u.poolIndex].sets[u.setIndex].listAndHeal(u.bucket, u.object, healObjectVersionsDisparity)
+				} else {
+					healObject(u.bucket, u.object, u.versionID, madmin.HealNormalScan)
+				}
 			}
 
 			wait()

cmd/naughty-disk_test.go

@@ -23,7 +23,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 )
 
 // naughtyDisk wraps a POSIX disk and returns programmed errors

cmd/notification-summary.go

@@ -18,7 +18,7 @@
 package cmd
 
 import (
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 )
 
 // GetTotalCapacity gets the total capacity in the cluster.

cmd/notification.go

@@ -29,7 +29,7 @@ import (
 
 	"github.com/cespare/xxhash/v2"
 	"github.com/klauspost/compress/zip"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/logger"
 	xnet "github.com/minio/pkg/net"
@@ -453,6 +453,7 @@ func (sys *NotificationSys) DeleteBucketMetadata(ctx context.Context, bucketName
 	globalBucketTargetSys.Delete(bucketName)
 	globalEventNotifier.RemoveNotification(bucketName)
 	globalBucketConnStats.delete(bucketName)
+	globalBucketHTTPStats.delete(bucketName)
 	if localMetacacheMgr != nil {
 		localMetacacheMgr.deleteBucketCache(bucketName)
 	}
@@ -803,7 +804,7 @@ func (sys *NotificationSys) addNodeErr(nodeInfo madmin.NodeInfo, peerClient *pee
 	addr := peerClient.host.String()
 	reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
 	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-	logger.LogIf(ctx, err)
+	logger.LogOnceIf(ctx, err, "add-node-err"+addr)
 	nodeInfo.SetAddr(addr)
 	nodeInfo.SetError(err.Error())
 }
@@ -1032,7 +1033,6 @@ func (sys *NotificationSys) GetPeerOnlineCount() (nodesOnline, nodesOffline int)
 
 // NewNotificationSys - creates new notification system object.
 func NewNotificationSys(endpoints EndpointServerPools) *NotificationSys {
-	// targetList/bucketRulesMap/bucketRemoteTargetRulesMap are populated by NotificationSys.Init()
 	remote, all := newPeerRestClients(endpoints)
 	return &NotificationSys{
 		peerClients:    remote,

cmd/object-api-datatypes.go

@@ -23,7 +23,7 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"

cmd/object-api-interface.go

@@ -23,7 +23,7 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/hash"

cmd/object-handlers.go

@@ -550,7 +550,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 			return
 		}
 		if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients
-			logger.LogIf(ctx, fmt.Errorf("Unable to write all the data to client: %w", err))
+			logger.LogOnceIf(ctx, fmt.Errorf("Unable to write all the data to client: %w", err), "get-object-handler-write")
 		}
 		return
 	}
@@ -561,7 +561,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 			return
 		}
 		if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients
-			logger.LogIf(ctx, fmt.Errorf("Unable to write all the data to client: %w", err))
+			logger.LogOnceIf(ctx, fmt.Errorf("Unable to write all the data to client: %w", err), "get-object-handler-close")
 		}
 		return
 	}
@@ -2298,6 +2298,14 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
+	replica := r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String()
+	if replica {
+		if s3Error := checkRequestAuthType(ctx, r, policy.ReplicateDeleteAction, bucket, object); s3Error != ErrNone {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+			return
+		}
+	}
+
 	if globalDNSConfig != nil {
 		_, err := globalDNSConfig.Get(bucket)
 		if err != nil && err != dns.ErrNotImplemented {
@@ -2311,10 +2319,12 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	var (
-		goi  ObjectInfo
-		gerr error
-	)
+
+	rcfg, _ := globalBucketObjectLockSys.Get(bucket)
+	if rcfg.LockEnabled && opts.DeletePrefix {
+		writeErrorResponse(ctx, w, toAPIError(ctx, errors.New("force-delete is forbidden in a locked-enabled bucket")), r.URL)
+		return
+	}
 
 	getObjectInfo := objectAPI.GetObjectInfo
 	if api.CacheAPI() != nil {
@@ -2327,7 +2337,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 	// the null version's remote object to delete if
 	// transitioned.
 	goiOpts := os.GetOpts()
-	goi, gerr = getObjectInfo(ctx, bucket, object, goiOpts)
+	goi, gerr := getObjectInfo(ctx, bucket, object, goiOpts)
 	if gerr == nil {
 		os.SetTransitionState(goi.TransitionedObject)
 	}
@@ -2338,17 +2348,12 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 			VersionID:  opts.VersionID,
 		},
 	}, goi, opts, gerr)
+
+	vID := opts.VersionID
 	if dsc.ReplicateAny() {
 		opts.SetDeleteReplicationState(dsc, opts.VersionID)
 	}
-
-	vID := opts.VersionID
-	if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() {
-		// check if replica has permission to be deleted.
-		if apiErrCode := checkRequestAuthType(ctx, r, policy.ReplicateDeleteAction, bucket, object); apiErrCode != ErrNone {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL)
-			return
-		}
+	if replica {
 		opts.SetReplicaStatus(replication.Replica)
 		if opts.VersionPurgeStatus().Empty() {
 			// opts.VersionID holds delete marker version ID to replicate and not yet present on disk
@@ -2357,23 +2362,17 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 	}
 
 	apiErr := ErrNone
-	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
-		if opts.DeletePrefix {
-			writeErrorResponse(ctx, w, toAPIError(ctx, errors.New("force-delete is forbidden in a locked-enabled bucket")), r.URL)
+	if vID != "" {
+		apiErr = enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
+			ObjectV: ObjectV{
+				ObjectName: object,
+				VersionID:  vID,
+			},
+		}, goi, gerr)
+		if apiErr != ErrNone && apiErr != ErrNoSuchKey {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL)
 			return
 		}
-		if vID != "" {
-			apiErr = enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
-				ObjectV: ObjectV{
-					ObjectName: object,
-					VersionID:  vID,
-				},
-			}, goi, gerr)
-			if apiErr != ErrNone && apiErr != ErrNoSuchKey {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL)
-				return
-			}
-		}
 	}
 
 	if apiErr == ErrNoSuchKey {

cmd/object-lambda-handlers.go

@@ -47,7 +47,7 @@ func getLambdaEventData(bucket, object string, cred auth.Credentials, r *http.Re
 	}
 
 	duration := time.Until(cred.Expiration)
-	if cred.Expiration.IsZero() || duration > time.Hour {
+	if duration > time.Hour || duration < time.Hour {
 		// Always limit to 1 hour.
 		duration = time.Hour
 	}

cmd/object-multipart-handlers.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2022 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -459,12 +459,14 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
+	_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
+
 	// Read compression metadata preserved in the init multipart for the decision.
 	_, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
 	// Compress only if the compression is enabled during initial multipart.
 	var idxCb func() []byte
 	if isCompressed {
-		wantEncryption := crypto.Requested(r.Header)
+		wantEncryption := crypto.Requested(r.Header) || isEncrypted
 		s2c, cb := newS2CompressReader(reader, actualPartSize, wantEncryption)
 		idxCb = cb
 		defer s2c.Close()
@@ -488,7 +490,6 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	rawReader := srcInfo.Reader
 	pReader := NewPutObjReader(rawReader)
 
-	_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
 	var objectEncryptionKey crypto.ObjectKey
 	if isEncrypted {
 		if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) {
@@ -558,7 +559,8 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	}
 
 	if isEncrypted {
-		partInfo.ETag = tryDecryptETag(objectEncryptionKey[:], partInfo.ETag, crypto.S3.IsRequested(r.Header))
+		sseS3 := crypto.S3.IsRequested(r.Header) || crypto.S3.IsEncrypted(mi.UserDefined)
+		partInfo.ETag = tryDecryptETag(objectEncryptionKey[:], partInfo.ETag, sseS3)
 	}
 
 	response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified)

cmd/os-instrumented.go

@@ -23,7 +23,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/disk"
 	ioutilx "github.com/minio/minio/internal/ioutil"
 )

cmd/peer-rest-client.go

@@ -29,7 +29,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/event"
 	xhttp "github.com/minio/minio/internal/http"

cmd/peer-rest-server.go

@@ -30,7 +30,7 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	b "github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/logger"
@@ -480,7 +480,7 @@ func (s *peerRESTServer) GetSysConfigHandler(w http.ResponseWriter, r *http.Requ
 
 	info := madmin.GetSysConfig(ctx, r.Host)
 
-	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
+	logger.LogOnceIf(ctx, gob.NewEncoder(w).Encode(info), "get-sys-config")
 }
 
 // GetSysServicesHandler - returns system services information.
@@ -533,6 +533,7 @@ func (s *peerRESTServer) DeleteBucketMetadataHandler(w http.ResponseWriter, r *h
 	globalBucketTargetSys.Delete(bucketName)
 	globalEventNotifier.RemoveNotification(bucketName)
 	globalBucketConnStats.delete(bucketName)
+	globalBucketHTTPStats.delete(bucketName)
 	if localMetacacheMgr != nil {
 		localMetacacheMgr.deleteBucketCache(bucketName)
 	}

cmd/peer-s3-client.go

@@ -21,37 +21,81 @@ import (
 	"context"
 	"encoding/gob"
 	"errors"
-	"fmt"
 	"io"
 	"net/url"
 	"strconv"
 
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/rest"
-	xnet "github.com/minio/pkg/net"
 	"github.com/minio/pkg/sync/errgroup"
+	"golang.org/x/exp/slices"
 )
 
 var errPeerOffline = errors.New("peer is offline")
 
+type peerS3Client interface {
+	ListBuckets(ctx context.Context, opts BucketOptions) ([]BucketInfo, error)
+	GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error)
+	MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error
+	DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error
+
+	GetHost() string
+	SetPools([]int)
+	GetPools() []int
+}
+
+type localPeerS3Client struct {
+	host  string
+	pools []int
+}
+
+func (l *localPeerS3Client) GetHost() string {
+	return l.host
+}
+
+func (l *localPeerS3Client) SetPools(p []int) {
+	l.pools = make([]int, len(p))
+	copy(l.pools, p)
+}
+
+func (l localPeerS3Client) GetPools() []int {
+	return l.pools
+}
+
+func (l localPeerS3Client) ListBuckets(ctx context.Context, opts BucketOptions) ([]BucketInfo, error) {
+	return listBucketsLocal(ctx, opts)
+}
+
+func (l localPeerS3Client) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) {
+	return getBucketInfoLocal(ctx, bucket, opts)
+}
+
+func (l localPeerS3Client) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error {
+	return makeBucketLocal(ctx, bucket, opts)
+}
+
+func (l localPeerS3Client) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
+	return deleteBucketLocal(ctx, bucket, opts)
+}
+
 // client to talk to peer Nodes.
-type peerS3Client struct {
-	host       *xnet.Host
+type remotePeerS3Client struct {
+	host       string
+	pools      []int
 	restClient *rest.Client
 }
 
 // Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
 // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
-func (client *peerS3Client) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
+func (client *remotePeerS3Client) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
 	return client.callWithContext(GlobalContext, method, values, body, length)
 }
 
 // Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
 // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
-func (client *peerS3Client) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
+func (client *remotePeerS3Client) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
 	if values == nil {
 		values = make(url.Values)
 	}
@@ -67,16 +111,15 @@ func (client *peerS3Client) callWithContext(ctx context.Context, method string,
 
 // S3PeerSys - S3 peer call system.
 type S3PeerSys struct {
-	peerClients    []*peerS3Client // Excludes self
-	allPeerClients []*peerS3Client // Includes nil client for self
+	peerClients []peerS3Client // Excludes self
+	poolsCount  int
 }
 
 // NewS3PeerSys - creates new S3 peer calls.
 func NewS3PeerSys(endpoints EndpointServerPools) *S3PeerSys {
-	remote, all := newPeerS3Clients(endpoints)
 	return &S3PeerSys{
-		peerClients:    remote,
-		allPeerClients: all,
+		peerClients: newPeerS3Clients(endpoints.GetNodes()),
+		poolsCount:  len(endpoints),
 	}
 }
 
@@ -84,14 +127,8 @@ func NewS3PeerSys(endpoints EndpointServerPools) *S3PeerSys {
 func (sys *S3PeerSys) ListBuckets(ctx context.Context, opts BucketOptions) (result []BucketInfo, err error) {
 	g := errgroup.WithNErrs(len(sys.peerClients))
 
-	localBuckets, err := listBucketsLocal(ctx, opts)
-	if err != nil {
-		return nil, err
-	}
-
-	nodeBuckets := make([][]BucketInfo, len(sys.peerClients)+1)
+	nodeBuckets := make([][]BucketInfo, len(sys.peerClients))
 	errs := []error{nil}
-	nodeBuckets[0] = localBuckets
 
 	for idx, client := range sys.peerClients {
 		idx := idx
@@ -104,14 +141,14 @@ func (sys *S3PeerSys) ListBuckets(ctx context.Context, opts BucketOptions) (resu
 			if err != nil {
 				return err
 			}
-			nodeBuckets[idx+1] = localBuckets
+			nodeBuckets[idx] = localBuckets
 			return nil
 		}, idx)
 	}
 
 	errs = append(errs, g.Wait()...)
 
-	quorum := (len(sys.allPeerClients) / 2)
+	quorum := len(sys.peerClients)/2 + 1
 	if err = reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, quorum); err != nil {
 		return nil, err
 	}
@@ -156,11 +193,7 @@ func (sys *S3PeerSys) GetBucketInfo(ctx context.Context, bucket string, opts Buc
 
 	errs := g.Wait()
 
-	bucketInfo, err := getBucketInfoLocal(ctx, bucket, opts)
-	errs = append(errs, err)
-	bucketInfos = append(bucketInfos, bucketInfo)
-
-	quorum := (len(sys.allPeerClients) / 2)
+	quorum := len(sys.peerClients)/2 + 1
 	if err = reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, quorum); err != nil {
 		return BucketInfo{}, toObjectErr(err, bucket)
 	}
@@ -174,7 +207,7 @@ func (sys *S3PeerSys) GetBucketInfo(ctx context.Context, bucket string, opts Buc
 	return BucketInfo{}, toObjectErr(errVolumeNotFound, bucket)
 }
 
-func (client *peerS3Client) ListBuckets(ctx context.Context, opts BucketOptions) ([]BucketInfo, error) {
+func (client *remotePeerS3Client) ListBuckets(ctx context.Context, opts BucketOptions) ([]BucketInfo, error) {
 	v := url.Values{}
 	v.Set(peerS3BucketDeleted, strconv.FormatBool(opts.Deleted))
 
@@ -190,7 +223,7 @@ func (client *peerS3Client) ListBuckets(ctx context.Context, opts BucketOptions)
 }
 
 // GetBucketInfo returns bucket stat info from a peer
-func (client *peerS3Client) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) {
+func (client *remotePeerS3Client) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) {
 	v := url.Values{}
 	v.Set(peerS3Bucket, bucket)
 	v.Set(peerS3BucketDeleted, strconv.FormatBool(opts.Deleted))
@@ -209,7 +242,6 @@ func (client *peerS3Client) GetBucketInfo(ctx context.Context, bucket string, op
 // MakeBucket creates bucket across all peers
 func (sys *S3PeerSys) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	g := errgroup.WithNErrs(len(sys.peerClients))
-
 	for idx, client := range sys.peerClients {
 		client := client
 		g.Go(func() error {
@@ -219,30 +251,24 @@ func (sys *S3PeerSys) MakeBucket(ctx context.Context, bucket string, opts MakeBu
 			return client.MakeBucket(ctx, bucket, opts)
 		}, idx)
 	}
-
 	errs := g.Wait()
-	errs = append(errs, makeBucketLocal(ctx, bucket, opts))
 
-	quorum := (len(sys.allPeerClients) / 2) + 1
-	err := reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, quorum)
-
-	// Perform MRF on missing buckets for temporary errors.
-	for _, err := range errs {
-		if err == nil {
-			continue
+	for poolIdx := 0; poolIdx < sys.poolsCount; poolIdx++ {
+		perPoolErrs := make([]error, 0, len(sys.peerClients))
+		for i, client := range sys.peerClients {
+			if slices.Contains(client.GetPools(), poolIdx) {
+				perPoolErrs = append(perPoolErrs, errs[i])
+			}
 		}
-		if errors.Is(err, errPeerOffline) || errors.Is(err, errDiskNotFound) ||
-			isNetworkError(err) {
-			globalMRFState.addPartialOp(partialOperation{
-				bucket: bucket,
-			})
+		if poolErr := reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, len(perPoolErrs)/2+1); poolErr != nil {
+			return toObjectErr(poolErr, bucket)
 		}
 	}
-	return toObjectErr(err, bucket)
+	return nil
 }
 
 // MakeBucket creates a bucket on a peer
-func (client *peerS3Client) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error {
+func (client *remotePeerS3Client) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	v := url.Values{}
 	v.Set(peerS3Bucket, bucket)
 	v.Set(peerS3BucketForceCreate, strconv.FormatBool(opts.ForceCreate))
@@ -259,7 +285,6 @@ func (client *peerS3Client) MakeBucket(ctx context.Context, bucket string, opts
 // DeleteBucket deletes bucket across all peers
 func (sys *S3PeerSys) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
 	g := errgroup.WithNErrs(len(sys.peerClients))
-
 	for idx, client := range sys.peerClients {
 		client := client
 		g.Go(func() error {
@@ -269,32 +294,26 @@ func (sys *S3PeerSys) DeleteBucket(ctx context.Context, bucket string, opts Dele
 			return client.DeleteBucket(ctx, bucket, opts)
 		}, idx)
 	}
-
 	errs := g.Wait()
-	errs = append(errs, deleteBucketLocal(ctx, bucket, opts))
 
-	var errReturn error
-	for _, err := range errs {
-		if errReturn == nil && err != nil {
-			// always return first error
-			errReturn = toObjectErr(err, bucket)
-			break
+	for poolIdx := 0; poolIdx < sys.poolsCount; poolIdx++ {
+		perPoolErrs := make([]error, 0, len(sys.peerClients))
+		for i, client := range sys.peerClients {
+			if slices.Contains(client.GetPools(), poolIdx) {
+				perPoolErrs = append(perPoolErrs, errs[i])
+			}
 		}
-	}
-
-	for _, err := range errs {
-		if err == nil && errReturn != nil {
+		if poolErr := reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, len(perPoolErrs)/2+1); poolErr != nil {
 			// re-create successful deletes, since we are return an error.
 			sys.MakeBucket(ctx, bucket, MakeBucketOptions{})
-			break
+			return toObjectErr(poolErr, bucket)
 		}
 	}
-
-	return errReturn
+	return nil
 }
 
 // DeleteBucket deletes bucket on a peer
-func (client *peerS3Client) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
+func (client *remotePeerS3Client) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
 	v := url.Values{}
 	v.Set(peerS3Bucket, bucket)
 	v.Set(peerS3BucketForceDelete, strconv.FormatBool(opts.Force))
@@ -308,33 +327,35 @@ func (client *peerS3Client) DeleteBucket(ctx context.Context, bucket string, opt
 	return nil
 }
 
+func (client remotePeerS3Client) GetHost() string {
+	return client.host
+}
+
+func (client remotePeerS3Client) GetPools() []int {
+	return client.pools
+}
+
+func (client *remotePeerS3Client) SetPools(p []int) {
+	client.pools = make([]int, len(p))
+	copy(client.pools, p)
+}
+
 // newPeerS3Clients creates new peer clients.
-// The two slices will point to the same clients,
-// but 'all' will contain nil entry for local client.
-// The 'all' slice will be in the same order across the cluster.
-func newPeerS3Clients(endpoints EndpointServerPools) (remote, all []*peerS3Client) {
-	if !globalIsDistErasure {
-		// Only useful in distributed setups
-		return nil, nil
-	}
-	hosts := endpoints.hostsSorted()
-	remote = make([]*peerS3Client, 0, len(hosts))
-	all = make([]*peerS3Client, len(hosts))
-	for i, host := range hosts {
-		if host == nil {
-			continue
+func newPeerS3Clients(nodes []Node) (peers []peerS3Client) {
+	peers = make([]peerS3Client, len(nodes))
+	for i, node := range nodes {
+		if node.IsLocal {
+			peers[i] = &localPeerS3Client{host: node.Host}
+		} else {
+			peers[i] = newPeerS3Client(node.Host)
 		}
-		all[i] = newPeerS3Client(host)
-		remote = append(remote, all[i])
+		peers[i].SetPools(node.Pools)
 	}
-	if len(all) != len(remote)+1 {
-		logger.LogIf(context.Background(), fmt.Errorf("WARNING: Expected number of all hosts (%v) to be remote +1 (%v)", len(all), len(remote)))
-	}
-	return remote, all
+	return
 }
 
 // Returns a peer S3 client.
-func newPeerS3Client(peer *xnet.Host) *peerS3Client {
+func newPeerS3Client(peer string) peerS3Client {
 	scheme := "http"
 	if globalIsTLS {
 		scheme = "https"
@@ -342,7 +363,7 @@ func newPeerS3Client(peer *xnet.Host) *peerS3Client {
 
 	serverURL := &url.URL{
 		Scheme: scheme,
-		Host:   peer.String(),
+		Host:   peer,
 		Path:   peerS3Path,
 	}
 
@@ -360,5 +381,5 @@ func newPeerS3Client(peer *xnet.Host) *peerS3Client {
 		return !isNetworkError(err)
 	}
 
-	return &peerS3Client{host: peer, restClient: restClient}
+	return &remotePeerS3Client{host: peer, restClient: restClient}
 }

cmd/perf-tests.go

@@ -29,7 +29,7 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/pkg/randreader"
 )

cmd/server-main.go

@@ -36,7 +36,7 @@ import (
 
 	"github.com/coreos/go-systemd/v22/daemon"
 	"github.com/minio/cli"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -246,24 +246,11 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 
 	globalEndpoints, setupType, err = createServerEndpoints(globalMinioAddr, serverCmdArgs(ctx)...)
 	logger.FatalIf(err, "Invalid command line arguments")
+	globalNodes = globalEndpoints.GetNodes()
 
 	globalLocalNodeName = GetLocalPeer(globalEndpoints, globalMinioHost, globalMinioPort)
 	nodeNameSum := sha256.Sum256([]byte(globalLocalNodeName))
 	globalLocalNodeNameHex = hex.EncodeToString(nodeNameSum[:])
-	globalNodeNamesHex = make(map[string]struct{})
-	globalRemoteEndpoints = make(map[string]Endpoint)
-	for _, z := range globalEndpoints {
-		for _, ep := range z.Endpoints {
-			if ep.IsLocal {
-				globalRemoteEndpoints[globalLocalNodeName] = ep
-			} else {
-				globalRemoteEndpoints[ep.Host] = ep
-			}
-			nodeNameSum := sha256.Sum256([]byte(ep.Host))
-			globalNodeNamesHex[hex.EncodeToString(nodeNameSum[:])] = struct{}{}
-
-		}
-	}
 
 	// allow transport to be HTTP/1.1 for proxying.
 	globalProxyTransport = NewCustomHTTPProxyTransport()()
@@ -413,12 +400,9 @@ func bootstrapTrace(msg string) {
 	})
 }
 
-func initServer(ctx context.Context, newObject ObjectLayer) error {
+func initServerConfig(ctx context.Context, newObject ObjectLayer) error {
 	t1 := time.Now()
 
-	// Once the config is fully loaded, initialize the new object layer.
-	setObjectLayer(newObject)
-
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	for {
@@ -633,6 +617,12 @@ func serverMain(ctx *cli.Context) {
 		}
 	}
 
+	if !globalDisableFreezeOnBoot {
+		// Freeze the services until the bucket notification subsystem gets initialized.
+		bootstrapTrace("freezeServices")
+		freezeServices()
+	}
+
 	bootstrapTrace("newObjectLayer")
 	newObject, err := newObjectLayer(GlobalContext, globalEndpoints)
 	if err != nil {
@@ -642,34 +632,34 @@ func serverMain(ctx *cli.Context) {
 	xhttp.SetDeploymentID(globalDeploymentID)
 	xhttp.SetMinIOVersion(Version)
 
+	for _, n := range globalNodes {
+		nodeName := n.Host
+		if n.IsLocal {
+			nodeName = globalLocalNodeName
+		}
+		nodeNameSum := sha256.Sum256([]byte(nodeName + globalDeploymentID))
+		globalNodeNamesHex[hex.EncodeToString(nodeNameSum[:])] = struct{}{}
+	}
+
 	bootstrapTrace("newSharedLock")
 	globalLeaderLock = newSharedLock(GlobalContext, newObject, "leader.lock")
 
-	// Enable background operations for erasure coding
+	// Enable background operations on
+	//
+	// - Disk auto healing
+	// - MRF (most recently failed) healing
+	// - Background expiration routine for lifecycle policies
 	bootstrapTrace("initAutoHeal")
 	initAutoHeal(GlobalContext, newObject)
+
 	bootstrapTrace("initHealMRF")
 	initHealMRF(GlobalContext, newObject)
+
 	bootstrapTrace("initBackgroundExpiry")
 	initBackgroundExpiry(GlobalContext, newObject)
 
-	if !globalCLIContext.StrictS3Compat {
-		logger.Info(color.RedBold("WARNING: Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production"))
-	}
-
-	if globalActiveCred.Equal(auth.DefaultCredentials) {
-		msg := fmt.Sprintf("WARNING: Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables",
-			globalActiveCred)
-		logger.Info(color.RedBold(msg))
-	}
-
-	if !globalDisableFreezeOnBoot {
-		// Freeze the services until the bucket notification subsystem gets initialized.
-		freezeServices()
-	}
-
-	bootstrapTrace("initializing the server")
-	if err = initServer(GlobalContext, newObject); err != nil {
+	bootstrapTrace("initServerConfig")
+	if err = initServerConfig(GlobalContext, newObject); err != nil {
 		var cerr config.Err
 		// For any config error, we don't need to drop into safe-mode
 		// instead its a user error and should be fixed by user.
@@ -685,12 +675,22 @@ func serverMain(ctx *cli.Context) {
 		logger.LogIf(GlobalContext, err)
 	}
 
+	if !globalCLIContext.StrictS3Compat {
+		logger.Info(color.RedBold("WARNING: Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production"))
+	}
+
+	if globalActiveCred.Equal(auth.DefaultCredentials) {
+		msg := fmt.Sprintf("WARNING: Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables",
+			globalActiveCred)
+		logger.Info(color.RedBold(msg))
+	}
+
 	// Initialize users credentials and policies in background right after config has initialized.
 	go func() {
 		bootstrapTrace("globalIAMSys.Init")
 		globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalRefreshIAMInterval)
 
-		// Initialize
+		// Initialize Console UI
 		if globalBrowserEnabled {
 			bootstrapTrace("initConsoleServer")
 			srv, err := initConsoleServer()
@@ -721,7 +721,6 @@ func serverMain(ctx *cli.Context) {
 
 	go func() {
 		if !globalDisableFreezeOnBoot {
-			bootstrapTrace("freezeServices")
 			defer unfreezeServices()
 			defer bootstrapTrace("unfreezeServices")
 			t := time.AfterFunc(5*time.Minute, func() {

cmd/server-rlimit.go

@@ -21,7 +21,7 @@ import (
 	"runtime"
 	"runtime/debug"
 
-	"github.com/minio/madmin-go/v2/kernel"
+	"github.com/minio/madmin-go/v3/kernel"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/sys"
 )

cmd/server-startup-msg.go

@@ -25,7 +25,7 @@ import (
 	"strings"
 
 	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	color "github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/logger"
 	xnet "github.com/minio/pkg/net"

cmd/server-startup-msg_test.go

@@ -24,7 +24,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 )
 
 // Tests if we generate storage info.

cmd/sftp-server-driver.go

@@ -28,7 +28,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio/internal/auth"
@@ -97,18 +97,15 @@ func (f *sftpDriver) getMinIOClient() (*minio.Client, error) {
 		if err != nil {
 			return nil, err
 		}
-		ldapPolicies, _ := globalIAMSys.PolicyDBGet(targetUser, false, targetGroups...)
-		if len(ldapPolicies) == 0 {
-			return nil, errAuthentication
-		}
 		expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration("")
 		if err != nil {
 			return nil, err
 		}
 		claims := make(map[string]interface{})
 		claims[expClaim] = UTCNow().Add(expiryDur).Unix()
-		claims[ldapUser] = targetUser
-		claims[ldapUserN] = f.AccessKey()
+		for k, v := range f.permissions.CriticalOptions {
+			claims[k] = v
+		}
 
 		cred, err := auth.GetNewCredentialsWithMetadata(claims, globalActiveCred.SecretKey)
 		if err != nil {
@@ -165,6 +162,9 @@ func (f *sftpDriver) getMinIOClient() (*minio.Client, error) {
 }
 
 func (f *sftpDriver) AccessKey() string {
+	if _, ok := f.permissions.CriticalOptions["accessKey"]; !ok {
+		return f.permissions.CriticalOptions[ldapUserN]
+	}
 	return f.permissions.CriticalOptions["accessKey"]
 }
 
@@ -270,12 +270,20 @@ func (f *sftpDriver) Filecmd(r *sftp.Request) (err error) {
 		cctx, cancel := context.WithCancel(context.Background())
 		defer cancel()
 
+		if prefix == "" {
+			// if all objects are not deleted yet this call may fail.
+			return clnt.RemoveBucket(cctx, bucket)
+		}
+
 		objectsCh := make(chan minio.ObjectInfo)
 
 		// Send object names that are needed to be removed to objectsCh
 		go func() {
 			defer close(objectsCh)
-			opts := minio.ListObjectsOptions{Prefix: prefix, Recursive: true}
+			opts := minio.ListObjectsOptions{
+				Prefix:    prefix,
+				Recursive: true,
+			}
 			for object := range clnt.ListObjects(cctx, bucket, opts) {
 				if object.Err != nil {
 					return
@@ -305,6 +313,10 @@ func (f *sftpDriver) Filecmd(r *sftp.Request) (err error) {
 			return errors.New("bucket name cannot be empty")
 		}
 
+		if prefix == "" {
+			return clnt.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{Region: globalSite.Region})
+		}
+
 		dirPath := buildMinioDir(prefix)
 
 		_, err = clnt.PutObject(context.Background(), bucket, dirPath, bytes.NewReader([]byte("")), 0,

cmd/signals.go

@@ -51,10 +51,6 @@ func handleSignals() {
 		// send signal to various go-routines that they need to quit.
 		cancelGlobalContext()
 
-		if globalEventNotifier != nil {
-			globalEventNotifier.RemoveAllRemoteTargets()
-		}
-
 		if httpServer := newHTTPServerFn(); httpServer != nil {
 			err = httpServer.Shutdown()
 			if !errors.Is(err, http.ErrServerClosed) {
@@ -71,6 +67,10 @@ func handleSignals() {
 			logger.LogIf(context.Background(), srv.Shutdown())
 		}
 
+		if globalEventNotifier != nil {
+			globalEventNotifier.RemoveAllBucketTargets()
+		}
+
 		return (err == nil && oerr == nil)
 	}
 

cmd/signature-v4-utils_test.go

@@ -24,7 +24,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	xhttp "github.com/minio/minio/internal/http"
 )