xl-meta: Clean output (#18794)

* Sort files.
* Sort maps.
* Remove annoying stray ','
* Hide -ndjson - doesn't do what it claims.

Sample output:

```
{
	"/tmp/xl1/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":0,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":5,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]},
	"/tmp/xl10/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":0,"ModTime":"2022-03-27T07:40:36.271878663+02:00","Signature":"4ffe317b","Type":2,"VersionID":"afc0611a63e0403b8078bf21b9111a2b"},"Idx":0,"Metadata":{"DelObj":{"ID":"r8BhGmPgQDuAeL8huREaKw==","MTime":1648359636271878700,"MetaSys":{}},"Type":2}},{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":1,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":14,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]},
	"/tmp/xl11/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":0,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":15,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]},
	"/tmp/xl12/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":0,"ModTime":"2022-03-27T07:40:36.271878663+02:00","Signature":"4ffe317b","Type":2,"VersionID":"afc0611a63e0403b8078bf21b9111a2b"},"Idx":0,"Metadata":{"DelObj":{"ID":"r8BhGmPgQDuAeL8huREaKw==","MTime":1648359636271878700,"MetaSys":{}},"Type":2}},{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":1,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":16,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]},
	"/tmp/xl13/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":0,"ModTime":"2022-03-27T07:40:36.271878663+02:00","Signature":"4ffe317b","Type":2,"VersionID":"afc0611a63e0403b8078bf21b9111a2b"},"Idx":0,"Metadata":{"DelObj":{"ID":"r8BhGmPgQDuAeL8huREaKw==","MTime":1648359636271878700,"MetaSys":{}},"Type":2}},{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":1,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":1,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]},
	"/tmp/xl14/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":0,"ModTime":"2022-03-27T07:40:36.271878663+02:00","Signature":"4ffe317b","Type":2,"VersionID":"afc0611a63e0403b8078bf21b9111a2b"},"Idx":0,"Metadata":{"DelObj":{"ID":"r8BhGmPgQDuAeL8huREaKw==","MTime":1648359636271878700,"MetaSys":{}},"Type":2}},{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":1,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":2,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]},
	"/tmp/xl15/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":0,"ModTime":"2022-03-27T07:40:36.271878663+02:00","Signature":"4ffe317b","Type":2,"VersionID":"afc0611a63e0403b8078bf21b9111a2b"},"Idx":0,"Metadata":{"DelObj":{"ID":"r8BhGmPgQDuAeL8huREaKw==","MTime":1648359636271878700,"MetaSys":{}},"Type":2}},{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":1,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":3,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]},
	"/tmp/xl16/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":0,"ModTime":"2022-03-27T07:40:36.271878663+02:00","Signature":"4ffe317b","Type":2,"VersionID":"afc0611a63e0403b8078bf21b9111a2b"},"Idx":0,"Metadata":{"DelObj":{"ID":"r8BhGmPgQDuAeL8huREaKw==","MTime":1648359636271878700,"MetaSys":{}},"Type":2}},{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":1,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":4,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]},
	"/tmp/xl2/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":0,"ModTime":"2022-03-27T07:40:36.271878663+02:00","Signature":"4ffe317b","Type":2,"VersionID":"afc0611a63e0403b8078bf21b9111a2b"},"Idx":0,"Metadata":{"DelObj":{"ID":"r8BhGmPgQDuAeL8huREaKw==","MTime":1648359636271878700,"MetaSys":{}},"Type":2}},{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":1,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":6,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]},
	"/tmp/xl3/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":0,"ModTime":"2022-03-27T07:40:36.271878663+02:00","Signature":"4ffe317b","Type":2,"VersionID":"afc0611a63e0403b8078bf21b9111a2b"},"Idx":0,"Metadata":{"DelObj":{"ID":"r8BhGmPgQDuAeL8huREaKw==","MTime":1648359636271878700,"MetaSys":{}},"Type":2}},{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":1,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":7,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]},
	"/tmp/xl4/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":0,"ModTime":"2022-03-27T07:40:36.271878663+02:00","Signature":"4ffe317b","Type":2,"VersionID":"afc0611a63e0403b8078bf21b9111a2b"},"Idx":0,"Metadata":{"DelObj":{"ID":"r8BhGmPgQDuAeL8huREaKw==","MTime":1648359636271878700,"MetaSys":{}},"Type":2}},{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":1,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":8,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]},
	"/tmp/xl5/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":0,"ModTime":"2022-03-27T07:40:36.271878663+02:00","Signature":"4ffe317b","Type":2,"VersionID":"afc0611a63e0403b8078bf21b9111a2b"},"Idx":0,"Metadata":{"DelObj":{"ID":"r8BhGmPgQDuAeL8huREaKw==","MTime":1648359636271878700,"MetaSys":{}},"Type":2}},{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":1,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":9,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]},
	"/tmp/xl6/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":0,"ModTime":"2022-03-27T07:40:36.271878663+02:00","Signature":"4ffe317b","Type":2,"VersionID":"afc0611a63e0403b8078bf21b9111a2b"},"Idx":0,"Metadata":{"DelObj":{"ID":"r8BhGmPgQDuAeL8huREaKw==","MTime":1648359636271878700,"MetaSys":{}},"Type":2}},{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":1,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":10,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]},
	"/tmp/xl7/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":0,"ModTime":"2022-03-27T07:40:36.271878663+02:00","Signature":"4ffe317b","Type":2,"VersionID":"afc0611a63e0403b8078bf21b9111a2b"},"Idx":0,"Metadata":{"DelObj":{"ID":"r8BhGmPgQDuAeL8huREaKw==","MTime":1648359636271878700,"MetaSys":{}},"Type":2}},{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":1,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":11,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]},
	"/tmp/xl8/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":0,"ModTime":"2022-03-27T07:40:36.271878663+02:00","Signature":"4ffe317b","Type":2,"VersionID":"afc0611a63e0403b8078bf21b9111a2b"},"Idx":0,"Metadata":{"DelObj":{"ID":"r8BhGmPgQDuAeL8huREaKw==","MTime":1648359636271878700,"MetaSys":{}},"Type":2}},{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":1,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":12,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]},
	"/tmp/xl9/testbucket/hosts/xl.meta": {"Versions":[{"Header":{"Flags":0,"ModTime":"2022-03-27T07:40:36.271878663+02:00","Signature":"4ffe317b","Type":2,"VersionID":"afc0611a63e0403b8078bf21b9111a2b"},"Idx":0,"Metadata":{"DelObj":{"ID":"r8BhGmPgQDuAeL8huREaKw==","MTime":1648359636271878700,"MetaSys":{}},"Type":2}},{"Header":{"Flags":6,"ModTime":"2022-03-27T07:35:22.820280431+02:00","Signature":"ac150d34","Type":1,"VersionID":"12468eb5e6024ab581de7aa9a5220a4e"},"Idx":1,"Metadata":{"Type":1,"V2Obj":{"CSumAlgo":1,"DDir":"DvH10ieUSPmBLbKfCp3lVQ==","EcAlgo":1,"EcBSize":1048576,"EcDist":[5,6,7,8,9,10,11,12,13,14,15,16,1,2,3,4],"EcIndex":13,"EcM":12,"EcN":4,"ID":"EkaOteYCSrWB3nqppSIKTg==","MTime":1648359322820280300,"MetaSys":{"x-minio-internal-inline-data":"dHJ1ZQ=="},"MetaUsr":{"content-type":"application/octet-stream","etag":"ea210162eeb4adc31ff984cb744d6af6"},"PartASizes":[303],"PartETags":null,"PartNums":[1],"PartSizes":[303],"Size":303}}}]}
}
```

.github/workflows/issues.yaml

@@ -0,0 +1,18 @@
+# @format
+
+name: Issue Workflow
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/add-to-project@v0.5.0
+        with:
+          project-url: https://github.com/orgs/miniohq/projects/2
+          github-token: ${{ secrets.BOT_PAT }}

.github/workflows/replication.yaml

@@ -36,6 +36,12 @@ jobs:
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-decom
 
+      - name: Test Config File
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-configfile
+
       - name: Test Replication
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
@@ -48,3 +54,8 @@ jobs:
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-site-replication-minio
 
+      - name: Test Versioning
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-versioning

.github/workflows/vulncheck.yml

@@ -3,11 +3,10 @@ on:
   pull_request:
     branches:
       - master
-      - next
+
   push:
     branches:
       - master
-      - next
 
 permissions:
   contents: read # to fetch code (actions/checkout)
@@ -22,7 +21,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.21.4
+          go-version: 1.21.5
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

.gitignore

@@ -43,3 +43,4 @@ docs/debugging/inspect/inspect
 docs/debugging/pprofgoparser/pprofgoparser
 docs/debugging/reorder-disks/reorder-disks
 docs/debugging/populate-hard-links/populate-hardlinks
+docs/debugging/xattr/xattr

.golangci.yml

@@ -29,5 +29,8 @@ linters:
 issues:
   exclude-use-default: false
   exclude:
+    - "empty-block:"
+    - "unused-parameter:"
+    - "dot-imports:"
     - should have a package comment
     - error strings should not be capitalized or end with punctuation or a newline

Dockerfile.hotfix

@@ -9,7 +9,7 @@ ENV CGO_ENABLED 0
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
-    go install aead.dev/minisign/cmd/minisign@v0.2.0
+    go install aead.dev/minisign/cmd/minisign@v0.2.1
 
 # Download minio binary and signature file
 RUN curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \

Dockerfile.release

@@ -9,7 +9,7 @@ ENV CGO_ENABLED 0
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
-    go install aead.dev/minisign/cmd/minisign@v0.2.0
+    go install aead.dev/minisign/cmd/minisign@v0.2.1
 
 # Download minio binary and signature file
 RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \

Dockerfile.release.fips

@@ -9,7 +9,7 @@ ENV CGO_ENABLED 0
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
-    go install aead.dev/minisign/cmd/minisign@v0.2.0
+    go install aead.dev/minisign/cmd/minisign@v0.2.1
 
 # Download minio binary and signature file
 RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /go/bin/minio && \

Dockerfile.release.old_cpu

@@ -9,7 +9,7 @@ ENV CGO_ENABLED 0
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
-    go install aead.dev/minisign/cmd/minisign@v0.2.0
+    go install aead.dev/minisign/cmd/minisign@v0.2.1
 
 # Download minio binary and signature file
 RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \

Makefile

@@ -8,7 +8,6 @@ GOOS := $(shell go env GOOS)
 VERSION ?= $(shell git describe --tags)
 TAG ?= "quay.io/minio/minio:$(VERSION)"
 
-GOLANGCI_VERSION = v1.51.2
 GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)
 GOLANGCI = $(GOLANGCI_DIR)/golangci-lint
 
@@ -23,7 +22,7 @@ help: ## print this help
 
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
-	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR) $(GOLANGCI_VERSION)
+	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR)
 	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@6ac204f0b4d48d17ab4fa442134c7fba13127a4e
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
 
@@ -60,6 +59,13 @@ test-decom: install-race
 	@env bash $(PWD)/docs/distributed/decom-encrypted-sse-s3.sh
 	@env bash $(PWD)/docs/distributed/decom-compressed-sse-s3.sh
 
+test-versioning: install-race
+	@echo "Running minio versioning tests"
+	@env bash $(PWD)/docs/bucket/versioning/versioning-tests.sh
+
+test-configfile: install-race
+	@env bash $(PWD)/docs/distributed/distributed-from-config-file.sh
+
 test-upgrade: install-race
 	@echo "Running minio upgrade tests"
 	@(env bash $(PWD)/buildscripts/minio-upgrade.sh)

buildscripts/cross-compile.sh

@@ -9,7 +9,7 @@ function _init() {
 	export CGO_ENABLED=0
 
 	## List of architectures and OS to test coss compilation.
-	SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips"
+	SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64"
 }
 
 function _build() {

cmd/admin-bucket-handlers.go

@@ -94,7 +94,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		Quota:     data,
 		UpdatedAt: updatedAt,
 	}
-	if quotaConfig.Size == 0 || quotaConfig.Quota == 0 {
+	if quotaConfig.Size == 0 && quotaConfig.Quota == 0 {
 		bucketMeta.Quota = nil
 	}
 

cmd/admin-handlers-idp-ldap.go

@@ -19,10 +19,14 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
+	"fmt"
 	"io"
 	"net/http"
+	"strings"
 
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/policy"
@@ -175,3 +179,267 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 
 	writeSuccessResponseJSON(w, encryptedData)
 }
+
+// AddServiceAccountLDAP adds a new service account for provided LDAP username or DN
+//
+// PUT /minio/admin/v3/idp/ldap/add-service-account
+func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.Request) {
+	ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r)
+	if APIError.Code != "" {
+		writeErrorResponseJSON(ctx, w, APIError, r.URL)
+		return
+	}
+
+	// fail if ldap is not enabled
+	if !globalIAMSys.LDAPConfig.Enabled() {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errors.New("LDAP not enabled")), r.URL)
+	}
+
+	// Find the user for the request sender (as it may be sent via a service
+	// account or STS account):
+	requestorUser := cred.AccessKey
+	requestorParentUser := cred.AccessKey
+	requestorGroups := cred.Groups
+	requestorIsDerivedCredential := false
+	if cred.IsServiceAccount() || cred.IsTemp() {
+		requestorParentUser = cred.ParentUser
+		requestorIsDerivedCredential = true
+	}
+
+	// Check if we are creating svc account for request sender.
+	isSvcAccForRequestor := false
+	if targetUser == requestorUser || targetUser == requestorParentUser {
+		isSvcAccForRequestor = true
+	}
+
+	var (
+		targetGroups []string
+		err          error
+	)
+
+	// If we are creating svc account for request sender, ensure
+	// that targetUser is a real user (i.e. not derived
+	// credentials).
+	if isSvcAccForRequestor {
+		if requestorIsDerivedCredential {
+			if requestorParentUser == "" {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx,
+					errors.New("service accounts cannot be generated for temporary credentials without parent")), r.URL)
+				return
+			}
+			targetUser = requestorParentUser
+		}
+		targetGroups = requestorGroups
+
+		// Deny if the target user is not LDAP
+		isLDAP, err := globalIAMSys.LDAPConfig.DoesUsernameExist(targetUser)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		if isLDAP == "" {
+			err := errors.New("Specified user does not exist on LDAP server")
+			APIErr := errorCodes.ToAPIErrWithErr(ErrAdminNoSuchUser, err)
+			writeErrorResponseJSON(ctx, w, APIErr, r.URL)
+			return
+		}
+
+		// In case of LDAP/OIDC we need to set `opts.claims` to ensure
+		// it is associated with the LDAP/OIDC user properly.
+		for k, v := range cred.Claims {
+			if k == expClaim {
+				continue
+			}
+			opts.claims[k] = v
+		}
+	} else {
+		isDN := globalIAMSys.LDAPConfig.IsLDAPUserDN(targetUser)
+
+		opts.claims[ldapUserN] = targetUser // simple username
+		targetUser, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
+		if err != nil {
+			// if not found, check if DN
+			if strings.Contains(err.Error(), "not found") && isDN {
+				// warn user that DNs are not allowed
+				err = fmt.Errorf("Must use short username to add service account. %w", err)
+			}
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		opts.claims[ldapUser] = targetUser // DN
+	}
+
+	newCred, updatedAt, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	createResp := madmin.AddServiceAccountResp{
+		Credentials: madmin.Credentials{
+			AccessKey:  newCred.AccessKey,
+			SecretKey:  newCred.SecretKey,
+			Expiration: newCred.Expiration,
+		},
+	}
+
+	data, err := json.Marshal(createResp)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, encryptedData)
+
+	// Call hook for cluster-replication if the service account is not for a
+	// root user.
+	if newCred.ParentUser != globalActiveCred.AccessKey {
+		logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+			Type: madmin.SRIAMItemSvcAcc,
+			SvcAccChange: &madmin.SRSvcAccChange{
+				Create: &madmin.SRSvcAccCreate{
+					Parent:        newCred.ParentUser,
+					AccessKey:     newCred.AccessKey,
+					SecretKey:     newCred.SecretKey,
+					Groups:        newCred.Groups,
+					Name:          newCred.Name,
+					Description:   newCred.Description,
+					Claims:        opts.claims,
+					SessionPolicy: createReq.Policy,
+					Status:        auth.AccountOn,
+					Expiration:    createReq.Expiration,
+				},
+			},
+			UpdatedAt: updatedAt,
+		}))
+	}
+}
+
+// ListAccessKeysLDAP - GET /minio/admin/v3/idp/ldap/list-access-keys
+func (a adminAPIHandlers) ListAccessKeysLDAP(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+
+	userDN := r.Form.Get("userDN")
+
+	// If listing is requested for a specific user (who is not the request
+	// sender), check that the user has permissions.
+	if userDN != "" && userDN != cred.ParentUser {
+		if !globalIAMSys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          policy.ListServiceAccountsAdminAction,
+			ConditionValues: getConditionValues(r, "", cred),
+			IsOwner:         owner,
+			Claims:          cred.Claims,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+	} else {
+		if !globalIAMSys.IsAllowed(policy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          policy.ListServiceAccountsAdminAction,
+			ConditionValues: getConditionValues(r, "", cred),
+			IsOwner:         owner,
+			Claims:          cred.Claims,
+			DenyOnly:        true,
+		}) {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+		userDN = cred.AccessKey
+		if cred.ParentUser != "" {
+			userDN = cred.ParentUser
+		}
+	}
+
+	targetAccount, err := globalIAMSys.LDAPConfig.DoesUsernameExist(userDN)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	} else if userDN == "" {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
+		return
+	}
+
+	listType := r.Form.Get("listType")
+	if listType != "sts-only" && listType != "svcacc-only" && listType != "" {
+		// default to both
+		listType = ""
+	}
+
+	var serviceAccounts []auth.Credentials
+	var stsKeys []auth.Credentials
+
+	if listType == "" || listType == "sts-only" {
+		stsKeys, err = globalIAMSys.ListSTSAccounts(ctx, targetAccount)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+	if listType == "" || listType == "svcacc-only" {
+		serviceAccounts, err = globalIAMSys.ListServiceAccounts(ctx, targetAccount)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
+	var serviceAccountList []madmin.ServiceAccountInfo
+	var stsKeyList []madmin.ServiceAccountInfo
+
+	for _, svc := range serviceAccounts {
+		expiryTime := svc.Expiration
+		serviceAccountList = append(serviceAccountList, madmin.ServiceAccountInfo{
+			AccessKey:  svc.AccessKey,
+			Expiration: &expiryTime,
+		})
+	}
+	for _, sts := range stsKeys {
+		expiryTime := sts.Expiration
+		stsKeyList = append(stsKeyList, madmin.ServiceAccountInfo{
+			AccessKey:  sts.AccessKey,
+			Expiration: &expiryTime,
+		})
+	}
+
+	listResp := madmin.ListAccessKeysLDAPResp{
+		ServiceAccounts: serviceAccountList,
+		STSKeys:         stsKeyList,
+	}
+
+	data, err := json.Marshal(listResp)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, encryptedData)
+}

cmd/admin-handlers-pools.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 
 	"github.com/minio/minio/internal/logger"
@@ -66,16 +67,28 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 
 	vars := mux.Vars(r)
 	v := vars["pool"]
+	byID := vars["by-id"] == "true"
 
 	pools := strings.Split(v, ",")
 	poolIndices := make([]int, 0, len(pools))
 
 	for _, pool := range pools {
-		idx := globalEndpoints.GetPoolIdx(pool)
-		if idx == -1 {
-			// We didn't find any matching pools, invalid input
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
-			return
+		var idx int
+		if byID {
+			var err error
+			idx, err = strconv.Atoi(pool)
+			if err != nil {
+				// We didn't find any matching pools, invalid input
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+				return
+			}
+		} else {
+			idx = globalEndpoints.GetPoolIdx(pool)
+			if idx == -1 {
+				// We didn't find any matching pools, invalid input
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+				return
+			}
 		}
 		var pool *erasureSets
 		for pidx := range z.serverPools {
@@ -132,8 +145,17 @@ func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Requ
 
 	vars := mux.Vars(r)
 	v := vars["pool"]
+	byID := vars["by-id"] == "true"
+	idx := -1
+
+	if byID {
+		if i, err := strconv.Atoi(v); err == nil && i >= 0 && i < len(globalEndpoints) {
+			idx = i
+		}
+	} else {
+		idx = globalEndpoints.GetPoolIdx(v)
+	}
 
-	idx := globalEndpoints.GetPoolIdx(v)
 	if idx == -1 {
 		// We didn't find any matching pools, invalid input
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
@@ -178,8 +200,17 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 
 	vars := mux.Vars(r)
 	v := vars["pool"]
+	byID := vars["by-id"] == "true"
+	idx := -1
+
+	if byID {
+		if i, err := strconv.Atoi(v); err == nil && i >= 0 && i < len(globalEndpoints) {
+			idx = i
+		}
+	} else {
+		idx = globalEndpoints.GetPoolIdx(v)
+	}
 
-	idx := globalEndpoints.GetPoolIdx(v)
 	if idx == -1 {
 		apiErr := toAdminAPIErr(ctx, errInvalidArgument)
 		apiErr.Description = fmt.Sprintf("specified pool '%s' not found, please specify a valid pool", v)

cmd/admin-handlers-users.go

@@ -614,72 +614,17 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 
 // AddServiceAccount - PUT /minio/admin/v3/add-service-account
 func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
-
-	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil || globalNotificationSys == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
-		return
-	}
-
-	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
-	if s3Err != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
-		return
-	}
-
-	password := cred.SecretKey
-	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
-		return
-	}
-
-	var createReq madmin.AddServiceAccountReq
-	if err = json.Unmarshal(reqBytes, &createReq); err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
-		return
-	}
-
-	// service account access key cannot have space characters beginning and end of the string.
-	if hasSpaceBE(createReq.AccessKey) {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
-		return
-	}
-
-	if err := createReq.Validate(); err != nil {
-		// Since this validation would happen client side as well, we only send
-		// a generic error message here.
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
+	ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r)
+	if APIError.Code != "" {
+		writeErrorResponseJSON(ctx, w, APIError, r.URL)
 		return
 	}
 
 	var (
-		targetUser   string
 		targetGroups []string
+		err          error
 	)
 
-	// If the request did not set a TargetUser, the service account is
-	// created for the request sender.
-	targetUser = createReq.TargetUser
-	if targetUser == "" {
-		targetUser = cred.AccessKey
-	}
-
-	description := createReq.Description
-	if description == "" {
-		description = createReq.Comment
-	}
-	opts := newServiceAccountOpts{
-		accessKey:   createReq.AccessKey,
-		secretKey:   createReq.SecretKey,
-		name:        createReq.Name,
-		description: description,
-		expiration:  createReq.Expiration,
-		claims:      make(map[string]interface{}),
-	}
-
 	// Find the user for the request sender (as it may be sent via a service
 	// account or STS account):
 	requestorUser := cred.AccessKey
@@ -713,23 +658,6 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	// that targetUser is a real user (i.e. not derived
 	// credentials).
 	if isSvcAccForRequestor {
-		// Check if adding service account is explicitly denied.
-		//
-		// This allows turning off service accounts for request sender,
-		// if there is no deny statement this call is implicitly enabled.
-		if !globalIAMSys.IsAllowed(policy.Args{
-			AccountName:     requestorUser,
-			Groups:          requestorGroups,
-			Action:          policy.CreateServiceAccountAdminAction,
-			ConditionValues: getConditionValues(r, "", cred),
-			IsOwner:         owner,
-			Claims:          cred.Claims,
-			DenyOnly:        true,
-		}) {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
-
 		if requestorIsDerivedCredential {
 			if requestorParentUser == "" {
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx,
@@ -748,32 +676,16 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 			}
 			opts.claims[k] = v
 		}
-	} else {
-		// Need permission if we are creating a service account for a
-		// user <> to the request sender
-		if !globalIAMSys.IsAllowed(policy.Args{
-			AccountName:     requestorUser,
-			Groups:          requestorGroups,
-			Action:          policy.CreateServiceAccountAdminAction,
-			ConditionValues: getConditionValues(r, "", cred),
-			IsOwner:         owner,
-			Claims:          cred.Claims,
-		}) {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
-
+	} else if globalIAMSys.LDAPConfig.Enabled() {
 		// In case of LDAP we need to resolve the targetUser to a DN and
 		// query their groups:
-		if globalIAMSys.LDAPConfig.Enabled() {
-			opts.claims[ldapUserN] = targetUser // simple username
-			targetUser, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
-			if err != nil {
-				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-				return
-			}
-			opts.claims[ldapUser] = targetUser // username DN
+		opts.claims[ldapUserN] = targetUser // simple username
+		targetUser, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
 		}
+		opts.claims[ldapUser] = targetUser // username DN
 
 		// NOTE: if not using LDAP, then internal IDP or open ID is
 		// being used - in the former, group info is enforced when
@@ -781,19 +693,6 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		// latter, a group notion is not supported.
 	}
 
-	var sp *policy.Policy
-	if len(createReq.Policy) > 0 {
-		sp, err = policy.ParseConfig(bytes.NewReader(createReq.Policy))
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
-		if sp.Version == "" && len(sp.Statements) == 0 {
-			sp = nil
-		}
-	}
-
-	opts.sessionPolicy = sp
 	newCred, updatedAt, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -814,7 +713,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	encryptedData, err := madmin.EncryptData(password, data)
+	encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -2514,3 +2413,85 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 }
+
+func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) {
+	ctx := r.Context()
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrServerNotInitialized)
+	}
+
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(s3Err)
+	}
+
+	password := cred.SecretKey
+	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err)
+	}
+
+	var createReq madmin.AddServiceAccountReq
+	if err = json.Unmarshal(reqBytes, &createReq); err != nil {
+		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err)
+	}
+
+	// service account access key cannot have space characters beginning and end of the string.
+	if hasSpaceBE(createReq.AccessKey) {
+		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument)
+	}
+
+	if err := createReq.Validate(); err != nil {
+		// Since this validation would happen client side as well, we only send
+		// a generic error message here.
+		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument)
+	}
+	// If the request did not set a TargetUser, the service account is
+	// created for the request sender.
+	targetUser := createReq.TargetUser
+	if targetUser == "" {
+		targetUser = cred.AccessKey
+	}
+
+	description := createReq.Description
+	if description == "" {
+		description = createReq.Comment
+	}
+	opts := newServiceAccountOpts{
+		accessKey:   createReq.AccessKey,
+		secretKey:   createReq.SecretKey,
+		name:        createReq.Name,
+		description: description,
+		expiration:  createReq.Expiration,
+		claims:      make(map[string]interface{}),
+	}
+
+	// Check if action is allowed if creating access key for another user
+	// Check if action is explicitly denied if for self
+	if !globalIAMSys.IsAllowed(policy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          policy.CreateServiceAccountAdminAction,
+		ConditionValues: getConditionValues(r, "", cred),
+		IsOwner:         owner,
+		Claims:          cred.Claims,
+		DenyOnly:        (targetUser == cred.AccessKey || targetUser == cred.ParentUser),
+	}) {
+		return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAccessDenied)
+	}
+
+	var sp *policy.Policy
+	if len(createReq.Policy) > 0 {
+		sp, err = policy.ParseConfig(bytes.NewReader(createReq.Policy))
+		if err != nil {
+			return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
+		}
+	}
+
+	opts.sessionPolicy = sp
+
+	return ctx, cred, opts, createReq, targetUser, APIError{}
+}

cmd/admin-handlers.go

@@ -340,7 +340,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	storageInfo := objectAPI.StorageInfo(ctx)
+	storageInfo := objectAPI.StorageInfo(ctx, true)
 
 	// Collect any disk healing.
 	healing, _ := getAggregatedBackgroundHealState(ctx, nil)
@@ -1297,6 +1297,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 	customBucket := strings.TrimSpace(r.Form.Get(peerRESTBucket))
 	autotune := r.Form.Get("autotune") == "true"
 	noClear := r.Form.Get("noclear") == "true"
+	enableSha256 := r.Form.Get("enableSha256") == "true"
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -1313,7 +1314,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 		duration = time.Second * 10
 	}
 
-	storageInfo := objectAPI.StorageInfo(ctx)
+	storageInfo := objectAPI.StorageInfo(ctx, true)
 
 	sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune)
 	if !sufficientCapacity {
@@ -1366,6 +1367,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 		autotune:         autotune,
 		storageClass:     storageClass,
 		bucketName:       customBucket,
+		enableSha256:     enableSha256,
 	})
 	var prevResult madmin.SpeedTestResult
 	for {
@@ -1911,7 +1913,7 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 		}
 	}
 
-	log, audit := fetchLoggerInfo()
+	log, audit := fetchLoggerInfo(ctx)
 
 	// Get the notification target info
 	notifyTarget := fetchLambdaInfo()
@@ -2572,7 +2574,7 @@ func assignPoolNumbers(servers []madmin.ServerProperties) {
 func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
 
-	for _, tgt := range globalNotifyTargetList.Targets() {
+	for _, tgt := range globalEventNotifier.Targets() {
 		targetIDStatus := make(map[string]madmin.Status)
 		active, _ := tgt.IsActive()
 		targetID := tgt.ID()
@@ -2660,19 +2662,26 @@ func fetchKMSStatusV2(ctx context.Context) []madmin.KMS {
 	return stats
 }
 
+func targetStatus(ctx context.Context, h logger.Target) madmin.Status {
+	if h.IsOnline(ctx) {
+		return madmin.Status{Status: string(madmin.ItemOnline)}
+	}
+	return madmin.Status{Status: string(madmin.ItemOffline)}
+}
+
 // fetchLoggerDetails return log info
-func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
+func fetchLoggerInfo(ctx context.Context) ([]madmin.Logger, []madmin.Audit) {
 	var loggerInfo []madmin.Logger
 	var auditloggerInfo []madmin.Audit
 	for _, tgt := range logger.SystemTargets() {
 		if tgt.Endpoint() != "" {
-			loggerInfo = append(loggerInfo, madmin.Logger{tgt.String(): logger.TargetStatus(GlobalContext, tgt)})
+			loggerInfo = append(loggerInfo, madmin.Logger{tgt.String(): targetStatus(ctx, tgt)})
 		}
 	}
 
 	for _, tgt := range logger.AuditTargets() {
 		if tgt.Endpoint() != "" {
-			auditloggerInfo = append(auditloggerInfo, madmin.Audit{tgt.String(): logger.TargetStatus(GlobalContext, tgt)})
+			auditloggerInfo = append(auditloggerInfo, madmin.Audit{tgt.String(): targetStatus(ctx, tgt)})
 		}
 	}
 
@@ -2726,7 +2735,7 @@ func getClusterMetaInfo(ctx context.Context) []byte {
 		ci.Info.NoOfServers = len(globalEndpoints.Hostnames())
 		ci.Info.MinioVersion = Version
 
-		si := objectAPI.StorageInfo(ctx)
+		si := objectAPI.StorageInfo(ctx, true)
 
 		ci.Info.NoOfDrives = len(si.Disks)
 		for _, disk := range si.Disks {

cmd/admin-heal-ops.go

@@ -810,7 +810,7 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
 		// NOTE: Healing on meta is run regardless
 		// of any bucket being selected, this is to ensure that
 		// meta are always upto date and correct.
-		return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string) error {
+		return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string, scanMode madmin.HealScanMode) error {
 			if h.isQuitting() {
 				return errHealStopSignalled
 			}
@@ -867,7 +867,7 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
 
 	if !h.settings.Recursive {
 		if h.object != "" {
-			if err := h.healObject(bucket, h.object, ""); err != nil {
+			if err := h.healObject(bucket, h.object, "", h.settings.ScanMode); err != nil {
 				return err
 			}
 		}
@@ -882,7 +882,7 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
 }
 
 // healObject - heal the given object and record result
-func (h *healSequence) healObject(bucket, object, versionID string) error {
+func (h *healSequence) healObject(bucket, object, versionID string, scanMode madmin.HealScanMode) error {
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}

cmd/admin-router.go

@@ -300,6 +300,12 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.GetIdentityProviderCfg))
 		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.DeleteIdentityProviderCfg))
 
+		// LDAP specific service accounts ops
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp/ldap/add-service-account").HandlerFunc(adminMiddleware(adminAPI.AddServiceAccountLDAP))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp/ldap/list-access-keys").
+			HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysLDAP)).
+			Queries("userDN", "{userDN:.*}", "listType", "{listType:.*}")
+
 		// LDAP IAM operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(adminMiddleware(adminAPI.ListLDAPPolicyMappingEntities))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/ldap/policy/{operation}").HandlerFunc(adminMiddleware(adminAPI.AttachDetachPolicyLDAP))

cmd/admin-server-info.go

@@ -141,7 +141,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 
 	objLayer := newObjectLayerFn()
 	if objLayer != nil {
-		storageInfo := objLayer.LocalStorageInfo(GlobalContext)
+		storageInfo := objLayer.LocalStorageInfo(GlobalContext, true)
 		props.State = string(madmin.ItemOnline)
 		props.Disks = storageInfo.Disks
 	} else {

cmd/api-errors.go

@@ -430,6 +430,9 @@ const (
 	ErrLambdaARNInvalid
 	ErrLambdaARNNotFound
 
+	// New Codes for GetObjectAttributes and GetObjectVersionAttributes
+	ErrInvalidAttributeName
+
 	apiErrCodeEnd // This is used only for the testing code
 )
 
@@ -2063,6 +2066,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The specified policy is not found.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrInvalidAttributeName: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid attribute name specified.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	// Add your error structure here.
 }
 

cmd/api-router.go

@@ -23,7 +23,7 @@ import (
 	"net/http"
 
 	"github.com/klauspost/compress/gzhttp"
-	"github.com/minio/console/restapi"
+	consoleapi "github.com/minio/console/api"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
@@ -43,13 +43,13 @@ func setHTTPServer(h *xhttp.Server) {
 	globalObjLayerMutex.Unlock()
 }
 
-func newConsoleServerFn() *restapi.Server {
+func newConsoleServerFn() *consoleapi.Server {
 	globalObjLayerMutex.RLock()
 	defer globalObjLayerMutex.RUnlock()
 	return globalConsoleSrv
 }
 
-func setConsoleSrv(srv *restapi.Server) {
+func setConsoleSrv(srv *consoleapi.Server) {
 	globalObjLayerMutex.Lock()
 	globalConsoleSrv = srv
 	globalObjLayerMutex.Unlock()
@@ -227,6 +227,9 @@ func registerAPIRouter(router *mux.Router) {
 		// HeadObject
 		router.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
 			collectAPIStats("headobject", maxClients(gz(httpTraceAll(api.HeadObjectHandler)))))
+		// GetObjectAttribytes
+		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
+			collectAPIStats("getobjectattributes", maxClients(gz(httpTraceHdrs(api.GetObjectAttributesHandler))))).Queries("attributes", "")
 		// CopyObjectPart
 		router.Methods(http.MethodPut).Path("/{object:.+}").
 			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").

cmd/background-newdisks-heal-ops.go

@@ -345,18 +345,17 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 
 	initBackgroundHealing(ctx, objAPI) // start quick background healing
 
-	globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
-
 	if env.Get("_MINIO_AUTO_DRIVE_HEALING", config.EnableOn) == config.EnableOn || env.Get("_MINIO_AUTO_DISK_HEALING", config.EnableOn) == config.EnableOn {
+		globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
 		go monitorLocalDisksAndHeal(ctx, z)
 	}
 }
 
 func getLocalDisksToHeal() (disksToHeal Endpoints) {
 	globalLocalDrivesMu.RLock()
-	globalLocalDrives := globalLocalDrives
+	localDrives := globalLocalDrives
 	globalLocalDrivesMu.RUnlock()
-	for _, disk := range globalLocalDrives {
+	for _, disk := range localDrives {
 		_, err := disk.GetDiskID()
 		if errors.Is(err, errUnformattedDisk) {
 			disksToHeal = append(disksToHeal, disk.Endpoint())

cmd/batch-expire.go

@@ -33,9 +33,10 @@ import (
 	"github.com/minio/minio/internal/bucket/versioning"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/env"
-	"github.com/minio/pkg/wildcard"
-	"github.com/minio/pkg/workers"
+	"github.com/minio/pkg/v2/env"
+	"github.com/minio/pkg/v2/wildcard"
+	"github.com/minio/pkg/v2/workers"
+	"gopkg.in/yaml.v3"
 )
 
 // expire: # Expire objects that match a condition
@@ -80,19 +81,41 @@ import (
 
 // BatchJobExpirePurge type accepts non-negative versions to be retained
 type BatchJobExpirePurge struct {
+	line, col      int
 	RetainVersions int `yaml:"retainVersions" json:"retainVersions"`
 }
 
+var _ yaml.Unmarshaler = &BatchJobExpirePurge{}
+
+// UnmarshalYAML - BatchJobExpirePurge extends unmarshal to extract line, col
+func (p *BatchJobExpirePurge) UnmarshalYAML(val *yaml.Node) error {
+	type purge BatchJobExpirePurge
+	var tmp purge
+	err := val.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+
+	*p = BatchJobExpirePurge(tmp)
+	p.line, p.col = val.Line, val.Column
+	return nil
+}
+
 // Validate returns nil if value is valid, ie > 0.
 func (p BatchJobExpirePurge) Validate() error {
 	if p.RetainVersions < 0 {
-		return errors.New("retainVersions must be >= 0")
+		return BatchJobYamlErr{
+			line: p.line,
+			col:  p.col,
+			msg:  "retainVersions must be >= 0",
+		}
 	}
 	return nil
 }
 
 // BatchJobExpireFilter holds all the filters currently supported for batch replication
 type BatchJobExpireFilter struct {
+	line, col     int
 	OlderThan     time.Duration       `yaml:"olderThan,omitempty" json:"olderThan"`
 	CreatedBefore *time.Time          `yaml:"createdBefore,omitempty" json:"createdBefore"`
 	Tags          []BatchJobKV        `yaml:"tags,omitempty" json:"tags"`
@@ -103,6 +126,22 @@ type BatchJobExpireFilter struct {
 	Purge         BatchJobExpirePurge `yaml:"purge" json:"purge"`
 }
 
+var _ yaml.Unmarshaler = &BatchJobExpireFilter{}
+
+// UnmarshalYAML - BatchJobExpireFilter extends unmarshal to extract line, col
+// information
+func (ef *BatchJobExpireFilter) UnmarshalYAML(value *yaml.Node) error {
+	type expFilter BatchJobExpireFilter
+	var tmp expFilter
+	err := value.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+	*ef = BatchJobExpireFilter(tmp)
+	ef.line, ef.col = value.Line, value.Column
+	return err
+}
+
 // Matches returns true if obj matches the filter conditions specified in ef.
 func (ef BatchJobExpireFilter) Matches(obj ObjectInfo, now time.Time) bool {
 	switch ef.Type {
@@ -194,10 +233,18 @@ func (ef BatchJobExpireFilter) Validate() error {
 	case BatchJobExpireObject:
 	case BatchJobExpireDeleted:
 		if len(ef.Tags) > 0 || len(ef.Metadata) > 0 {
-			return errors.New("invalid batch-expire rule filter")
+			return BatchJobYamlErr{
+				line: ef.line,
+				col:  ef.col,
+				msg:  "delete type filter can't have tags or metadata",
+			}
 		}
 	default:
-		return errors.New("invalid batch-expire type")
+		return BatchJobYamlErr{
+			line: ef.line,
+			col:  ef.col,
+			msg:  "invalid batch-expire type",
+		}
 	}
 
 	for _, tag := range ef.Tags {
@@ -218,7 +265,11 @@ func (ef BatchJobExpireFilter) Validate() error {
 		return err
 	}
 	if ef.CreatedBefore != nil && !ef.CreatedBefore.Before(time.Now()) {
-		return errors.New("CreatedBefore is in the future")
+		return BatchJobYamlErr{
+			line: ef.line,
+			col:  ef.col,
+			msg:  "CreatedBefore is in the future",
+		}
 	}
 	return nil
 }
@@ -226,6 +277,7 @@ func (ef BatchJobExpireFilter) Validate() error {
 // BatchJobExpire represents configuration parameters for a batch expiration
 // job typically supplied in yaml form
 type BatchJobExpire struct {
+	line, col       int
 	APIVersion      string                 `yaml:"apiVersion" json:"apiVersion"`
 	Bucket          string                 `yaml:"bucket" json:"bucket"`
 	Prefix          string                 `yaml:"prefix" json:"prefix"`
@@ -234,6 +286,22 @@ type BatchJobExpire struct {
 	Rules           []BatchJobExpireFilter `yaml:"rules" json:"rules"`
 }
 
+var _ yaml.Unmarshaler = &BatchJobExpire{}
+
+// UnmarshalYAML - BatchJobExpire extends default unmarshal to extract line, col information.
+func (r *BatchJobExpire) UnmarshalYAML(val *yaml.Node) error {
+	type expireJob BatchJobExpire
+	var tmp expireJob
+	err := val.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+
+	*r = BatchJobExpire(tmp)
+	r.line, r.col = val.Line, val.Column
+	return nil
+}
+
 // Notify notifies notification endpoint if configured regarding job failure or success.
 func (r BatchJobExpire) Notify(ctx context.Context, body io.Reader) error {
 	if r.NotificationCfg.Endpoint == "" {

cmd/batch-handlers.go

@@ -52,7 +52,7 @@ import (
 	"github.com/minio/pkg/v2/env"
 	"github.com/minio/pkg/v2/policy"
 	"github.com/minio/pkg/v2/workers"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 var globalBatchConfig batch.Config
@@ -1007,12 +1007,9 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			// None of the provided metadata filters match skip the object.
 			return false
 		}
-		// if one of source or target is non MinIO, just replicate the top most version like `mc mirror`
-		if (r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3) && !info.IsLatest {
-			return false
-		}
 
-		return true
+		// if one of source or target is non MinIO, just replicate the top most version like `mc mirror`
+		return !((r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3) && !info.IsLatest)
 	}
 
 	u, err := url.Parse(r.Target.Endpoint)
@@ -1123,8 +1120,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		// one of source/target is s3, skip delete marker and all versions under the same object name.
 		s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
 
-		results := make(chan ObjectInfo, 100)
-		if err := api.Walk(ctx, r.Source.Bucket, r.Source.Prefix, results, WalkOptions{
+		if err := api.Walk(ctx, r.Source.Bucket, r.Source.Prefix, walkCh, WalkOptions{
 			Marker:   lastObject,
 			Filter:   selectObj,
 			AskDisks: walkQuorum,
@@ -1568,7 +1564,7 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 	}
 
 	job := &BatchJobRequest{}
-	if err = yaml.UnmarshalStrict(buf, job); err != nil {
+	if err = yaml.Unmarshal(buf, job); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/batch-job-common-types.go

@@ -18,26 +18,66 @@
 package cmd
 
 import (
-	"errors"
+	"fmt"
 	"strings"
 	"time"
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/pkg/v2/wildcard"
+	"gopkg.in/yaml.v3"
 )
 
 //go:generate msgp -file $GOFILE
+//msgp:ignore BatchJobYamlErr
+
+// BatchJobYamlErr can be used to return yaml validation errors with line,
+// column information guiding user to fix syntax errors
+type BatchJobYamlErr struct {
+	line, col int
+	msg       string
+}
+
+// message returns the error message excluding line, col information.
+// Intended to be used in unit tests.
+func (b BatchJobYamlErr) message() string {
+	return b.msg
+}
+
+// Error implements Error interface
+func (b BatchJobYamlErr) Error() string {
+	return fmt.Sprintf("%s\n Hint: error near line: %d, col: %d", b.msg, b.line, b.col)
+}
 
 // BatchJobKV is a key-value data type which supports wildcard matching
 type BatchJobKV struct {
-	Key   string `yaml:"key" json:"key"`
-	Value string `yaml:"value" json:"value"`
+	line, col int
+	Key       string `yaml:"key" json:"key"`
+	Value     string `yaml:"value" json:"value"`
+}
+
+var _ yaml.Unmarshaler = &BatchJobKV{}
+
+// UnmarshalYAML - BatchJobKV extends default unmarshal to extract line, col information.
+func (kv *BatchJobKV) UnmarshalYAML(val *yaml.Node) error {
+	type jobKV BatchJobKV
+	var tmp jobKV
+	err := val.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+	*kv = BatchJobKV(tmp)
+	kv.line, kv.col = val.Line, val.Column
+	return nil
 }
 
 // Validate returns an error if key is empty
 func (kv BatchJobKV) Validate() error {
 	if kv.Key == "" {
-		return errInvalidArgument
+		return BatchJobYamlErr{
+			line: kv.line,
+			col:  kv.col,
+			msg:  "key can't be empty",
+		}
 	}
 	return nil
 }
@@ -61,24 +101,66 @@ func (kv BatchJobKV) Match(ikv BatchJobKV) bool {
 // BatchJobNotification stores notification endpoint and token information.
 // Used by batch jobs to notify of their status.
 type BatchJobNotification struct {
-	Endpoint string `yaml:"endpoint" json:"endpoint"`
-	Token    string `yaml:"token" json:"token"`
+	line, col int
+	Endpoint  string `yaml:"endpoint" json:"endpoint"`
+	Token     string `yaml:"token" json:"token"`
+}
+
+var _ yaml.Unmarshaler = &BatchJobNotification{}
+
+// UnmarshalYAML - BatchJobNotification extends unmarshal to extract line, column information
+func (b *BatchJobNotification) UnmarshalYAML(val *yaml.Node) error {
+	type notification BatchJobNotification
+	var tmp notification
+	err := val.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+
+	*b = BatchJobNotification(tmp)
+	b.line, b.col = val.Line, val.Column
+	return nil
 }
 
 // BatchJobRetry stores retry configuration used in the event of failures.
 type BatchJobRetry struct {
-	Attempts int           `yaml:"attempts" json:"attempts"` // number of retry attempts
-	Delay    time.Duration `yaml:"delay" json:"delay"`       // delay between each retries
+	line, col int
+	Attempts  int           `yaml:"attempts" json:"attempts"` // number of retry attempts
+	Delay     time.Duration `yaml:"delay" json:"delay"`       // delay between each retries
+}
+
+var _ yaml.Unmarshaler = &BatchJobRetry{}
+
+// UnmarshalYAML - BatchJobRetry extends unmarshal to extract line, column information
+func (r *BatchJobRetry) UnmarshalYAML(val *yaml.Node) error {
+	type retry BatchJobRetry
+	var tmp retry
+	err := val.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+
+	*r = BatchJobRetry(tmp)
+	r.line, r.col = val.Line, val.Column
+	return nil
 }
 
 // Validate validates input replicate retries.
 func (r BatchJobRetry) Validate() error {
 	if r.Attempts < 0 {
-		return errInvalidArgument
+		return BatchJobYamlErr{
+			line: r.line,
+			col:  r.col,
+			msg:  "Invalid arguments specified",
+		}
 	}
 
 	if r.Delay < 0 {
-		return errInvalidArgument
+		return BatchJobYamlErr{
+			line: r.line,
+			col:  r.col,
+			msg:  "Invalid arguments specified",
+		}
 	}
 
 	return nil
@@ -96,6 +178,7 @@ func (r BatchJobRetry) Validate() error {
 
 // BatchJobSnowball describes the snowball feature when replicating objects from a local source to a remote target
 type BatchJobSnowball struct {
+	line, col   int
 	Disable     *bool   `yaml:"disable" json:"disable"`
 	Batch       *int    `yaml:"batch" json:"batch"`
 	InMemory    *bool   `yaml:"inmemory" json:"inmemory"`
@@ -104,21 +187,60 @@ type BatchJobSnowball struct {
 	SkipErrs    *bool   `yaml:"skipErrs" json:"skipErrs"`
 }
 
+var _ yaml.Unmarshaler = &BatchJobSnowball{}
+
+// UnmarshalYAML - BatchJobSnowball extends unmarshal to extract line, column information
+func (b *BatchJobSnowball) UnmarshalYAML(val *yaml.Node) error {
+	type snowball BatchJobSnowball
+	var tmp snowball
+	err := val.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+
+	*b = BatchJobSnowball(tmp)
+	b.line, b.col = val.Line, val.Column
+	return nil
+}
+
 // Validate the snowball parameters in the job description
 func (b BatchJobSnowball) Validate() error {
 	if *b.Batch <= 0 {
-		return errors.New("batch number should be non positive zero")
+		return BatchJobYamlErr{
+			line: b.line,
+			col:  b.col,
+			msg:  "batch number should be non positive zero",
+		}
 	}
 	_, err := humanize.ParseBytes(*b.SmallerThan)
-	return err
+	return BatchJobYamlErr{
+		line: b.line,
+		col:  b.col,
+		msg:  err.Error(),
+	}
 }
 
 // BatchJobSizeFilter supports size based filters - LesserThan and GreaterThan
 type BatchJobSizeFilter struct {
+	line, col  int
 	UpperBound BatchJobSize `yaml:"lessThan" json:"lessThan"`
 	LowerBound BatchJobSize `yaml:"greaterThan" json:"greaterThan"`
 }
 
+// UnmarshalYAML - BatchJobSizeFilter extends unmarshal to extract line, column information
+func (sf *BatchJobSizeFilter) UnmarshalYAML(val *yaml.Node) error {
+	type sizeFilter BatchJobSizeFilter
+	var tmp sizeFilter
+	err := val.Decode(&tmp)
+	if err != nil {
+		return err
+	}
+
+	*sf = BatchJobSizeFilter(tmp)
+	sf.line, sf.col = val.Line, val.Column
+	return nil
+}
+
 // InRange returns true in the following cases and false otherwise,
 // - sf.LowerBound < sz, when sf.LowerBound alone is specified
 // - sz < sf.UpperBound, when sf.UpperBound alone is specified
@@ -134,12 +256,14 @@ func (sf BatchJobSizeFilter) InRange(sz int64) bool {
 	return true
 }
 
-var errInvalidBatchJobSizeFilter = errors.New("invalid batch-job size filter")
-
 // Validate checks if sf is a valid batch-job size filter
 func (sf BatchJobSizeFilter) Validate() error {
 	if sf.LowerBound > 0 && sf.UpperBound > 0 && sf.LowerBound >= sf.UpperBound {
-		return errInvalidBatchJobSizeFilter
+		return BatchJobYamlErr{
+			line: sf.line,
+			col:  sf.col,
+			msg:  "invalid batch-job size filter",
+		}
 	}
 	return nil
 }

cmd/batch-job-common-types_test.go

@@ -83,6 +83,10 @@ func TestBatchJobSizeInRange(t *testing.T) {
 }
 
 func TestBatchJobSizeValidate(t *testing.T) {
+	errInvalidBatchJobSizeFilter := BatchJobYamlErr{
+		msg: "invalid batch-job size filter",
+	}
+
 	tests := []struct {
 		sizeFilter BatchJobSizeFilter
 		err        error
@@ -128,8 +132,16 @@ func TestBatchJobSizeValidate(t *testing.T) {
 	}
 	for i, test := range tests {
 		t.Run(fmt.Sprintf("test-%d", i+1), func(t *testing.T) {
-			if err := test.sizeFilter.Validate(); err != test.err {
-				t.Fatalf("Expected %v but got %v", test.err, err)
+			err := test.sizeFilter.Validate()
+			if err != nil {
+				gotErr := err.(BatchJobYamlErr)
+				testErr := test.err.(BatchJobYamlErr)
+				if gotErr.message() != testErr.message() {
+					t.Fatalf("Expected %v but got %v", test.err, err)
+				}
+			}
+			if err == nil && test.err != nil {
+				t.Fatalf("Expected %v but got nil", test.err)
 			}
 		})
 	}

cmd/bucket-handlers.go

@@ -1657,9 +1657,11 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Return an error if the bucket does not exist
-	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil && !forceDelete {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
+	if !forceDelete {
+		if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
 	}
 
 	// Attempt to delete bucket.

cmd/bucket-lifecycle.go

@@ -219,24 +219,23 @@ type transitionState struct {
 	numWorkers int
 	killCh     chan struct{}
 
-	activeTasks int32
+	activeTasks          atomic.Int64
+	missedImmediateTasks atomic.Int64
 
 	lastDayMu    sync.RWMutex
 	lastDayStats map[string]*lastDayTierStats
 }
 
-func (t *transitionState) queueTransitionTask(oi ObjectInfo, event lifecycle.Event, src lcEventSrc, blocking bool) {
+func (t *transitionState) queueTransitionTask(oi ObjectInfo, event lifecycle.Event, src lcEventSrc) {
 	task := transitionTask{objInfo: oi, event: event, src: src}
-	if blocking {
-		select {
-		case <-t.ctx.Done():
-		case t.transitionCh <- task:
-		}
-	} else {
-		select {
-		case <-t.ctx.Done():
-		case t.transitionCh <- task:
-		default:
+	select {
+	case <-t.ctx.Done():
+	case t.transitionCh <- task:
+	default:
+		switch src {
+		case lcEventSrc_s3PutObject, lcEventSrc_s3CopyObject, lcEventSrc_s3CompleteMultipartUpload:
+			// Update missed immediate tasks only for incoming requests.
+			t.missedImmediateTasks.Add(1)
 		}
 	}
 }
@@ -272,8 +271,14 @@ func (t *transitionState) PendingTasks() int {
 }
 
 // ActiveTasks returns the number of active (ongoing) ILM transition tasks.
-func (t *transitionState) ActiveTasks() int {
-	return int(atomic.LoadInt32(&t.activeTasks))
+func (t *transitionState) ActiveTasks() int64 {
+	return t.activeTasks.Load()
+}
+
+// MissedImmediateTasks returns the number of tasks - deferred to scanner due
+// to tasks channel being backlogged.
+func (t *transitionState) MissedImmediateTasks() int64 {
+	return t.missedImmediateTasks.Load()
 }
 
 // worker waits for transition tasks
@@ -288,7 +293,7 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 			if !ok {
 				return
 			}
-			atomic.AddInt32(&t.activeTasks, 1)
+			t.activeTasks.Add(1)
 			if err := transitionObject(t.ctx, objectAPI, task.objInfo, newLifecycleAuditEvent(task.src, task.event)); err != nil {
 				if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) && !xnet.IsNetworkOrHostDown(err, false) {
 					if !strings.Contains(err.Error(), "use of closed network connection") {
@@ -306,7 +311,7 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 				}
 				t.addLastDayStats(task.event.StorageClass, ts)
 			}
-			atomic.AddInt32(&t.activeTasks, -1)
+			t.activeTasks.Add(-1)
 		}
 	}
 }
@@ -379,7 +384,7 @@ func enqueueTransitionImmediate(obj ObjectInfo, src lcEventSrc) {
 	if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
 		switch event := lc.Eval(obj.ToLifecycleOpts()); event.Action {
 		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-			globalTransitionState.queueTransitionTask(obj, event, src, true)
+			globalTransitionState.queueTransitionTask(obj, event, src)
 		}
 	}
 }
@@ -502,9 +507,13 @@ func auditTierActions(ctx context.Context, tier string, bytes int64) func(err er
 		}
 
 		if err == nil {
-			op.TimeToResponseNS = time.Since(startTime).Nanoseconds()
+			since := time.Since(startTime)
+			op.TimeToResponseNS = since.Nanoseconds()
+			globalTierMetrics.Observe(tier, since)
+			globalTierMetrics.logSuccess(tier)
 		} else {
 			op.Error = err.Error()
+			globalTierMetrics.logFailure(tier)
 		}
 
 		logger.GetReqInfo(ctx).AppendTags("tierStats", op)

cmd/bucket-policy.go

@@ -121,7 +121,7 @@ func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[s
 		"CurrentTime":      {currTime.Format(time.RFC3339)},
 		"EpochTime":        {strconv.FormatInt(currTime.Unix(), 10)},
 		"SecureTransport":  {strconv.FormatBool(r.TLS != nil)},
-		"SourceIp":         {handlers.GetSourceIP(r)},
+		"SourceIp":         {handlers.GetSourceIPRaw(r)},
 		"UserAgent":        {r.UserAgent()},
 		"Referer":          {r.Referer()},
 		"principaltype":    {principalType},

cmd/bucket-replication-metrics.go

@@ -337,13 +337,13 @@ type SMA struct {
 	filledBuf bool
 }
 
-func newSMA(len int) *SMA {
-	if len <= 0 {
-		len = defaultWindowSize
+func newSMA(ln int) *SMA {
+	if ln <= 0 {
+		ln = defaultWindowSize
 	}
 	return &SMA{
-		buf:    make([]float64, len),
-		window: len,
+		buf:    make([]float64, ln),
+		window: ln,
 		idx:    0,
 	}
 }

cmd/bucket-replication.go

@@ -23,7 +23,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"math"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -373,15 +372,14 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 			if oi.DeleteMarker && (validReplStatus || replicate) {
 				dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync))
 				continue
-			} else {
-				// can be the case that other cluster is down and duplicate `mc rm --vid`
-				// is issued - this still needs to be replicated back to the other target
-				if !oi.VersionPurgeStatus.Empty() {
-					replicate = oi.VersionPurgeStatus == Pending || oi.VersionPurgeStatus == Failed
-					dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync))
-				}
-				continue
 			}
+			// can be the case that other cluster is down and duplicate `mc rm --vid`
+			// is issued - this still needs to be replicated back to the other target
+			if !oi.VersionPurgeStatus.Empty() {
+				replicate = oi.VersionPurgeStatus == Pending || oi.VersionPurgeStatus == Failed
+				dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync))
+			}
+			continue
 		}
 		tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, tgtArn)
 		// the target online status should not be used here while deciding
@@ -1469,6 +1467,24 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside
 			},
 		}
+		if tagTmStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]; ok {
+			ondiskTimestamp, err := time.Parse(time.RFC3339, tagTmStr)
+			if err == nil {
+				dstOpts.Internal.TaggingTimestamp = ondiskTimestamp
+			}
+		}
+		if retTmStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]; ok {
+			ondiskTimestamp, err := time.Parse(time.RFC3339, retTmStr)
+			if err == nil {
+				dstOpts.Internal.RetentionTimestamp = ondiskTimestamp
+			}
+		}
+		if lholdTmStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp]; ok {
+			ondiskTimestamp, err := time.Parse(time.RFC3339, lholdTmStr)
+			if err == nil {
+				dstOpts.Internal.LegalholdTimestamp = ondiskTimestamp
+			}
+		}
 		if _, rinfo.Err = c.CopyObject(ctx, tgt.Bucket, object, tgt.Bucket, object, getCopyObjMetadata(objInfo, tgt.StorageClass), srcOpts, dstOpts); rinfo.Err != nil {
 			rinfo.ReplicationStatus = replication.Failed
 			logger.LogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err))
@@ -1698,12 +1714,13 @@ type ReplicationPool struct {
 	activeWorkers    int32
 	activeMRFWorkers int32
 
-	objLayer ObjectLayer
-	ctx      context.Context
-	priority string
-	mu       sync.RWMutex
-	mrfMU    sync.Mutex
-	resyncer *replicationResyncer
+	objLayer   ObjectLayer
+	ctx        context.Context
+	priority   string
+	maxWorkers int
+	mu         sync.RWMutex
+	mrfMU      sync.Mutex
+	resyncer   *replicationResyncer
 
 	// workers:
 	workers    []chan ReplicationWorkerOperation
@@ -1749,9 +1766,13 @@ const (
 func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPoolOpts) *ReplicationPool {
 	var workers, failedWorkers int
 	priority := "auto"
+	maxWorkers := WorkerMaxLimit
 	if opts.Priority != "" {
 		priority = opts.Priority
 	}
+	if opts.MaxWorkers > 0 {
+		maxWorkers = opts.MaxWorkers
+	}
 	switch priority {
 	case "fast":
 		workers = WorkerMaxLimit
@@ -1763,7 +1784,13 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 		workers = WorkerAutoDefault
 		failedWorkers = MRFWorkerAutoDefault
 	}
+	if maxWorkers > 0 && workers > maxWorkers {
+		workers = maxWorkers
+	}
 
+	if maxWorkers > 0 && failedWorkers > maxWorkers {
+		failedWorkers = maxWorkers
+	}
 	pool := &ReplicationPool{
 		workers:         make([]chan ReplicationWorkerOperation, 0, workers),
 		lrgworkers:      make([]chan ReplicationWorkerOperation, 0, LargeWorkerCount),
@@ -1775,6 +1802,7 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 		ctx:             ctx,
 		objLayer:        o,
 		priority:        priority,
+		maxWorkers:      maxWorkers,
 	}
 
 	pool.AddLargeWorkers()
@@ -1930,7 +1958,7 @@ func (p *ReplicationPool) ResizeWorkers(n, checkOld int) {
 }
 
 // ResizeWorkerPriority sets replication failed workers pool size
-func (p *ReplicationPool) ResizeWorkerPriority(pri string) {
+func (p *ReplicationPool) ResizeWorkerPriority(pri string, maxWorkers int) {
 	var workers, mrfWorkers int
 	p.mu.Lock()
 	switch pri {
@@ -1944,13 +1972,21 @@ func (p *ReplicationPool) ResizeWorkerPriority(pri string) {
 		workers = WorkerAutoDefault
 		mrfWorkers = MRFWorkerAutoDefault
 		if len(p.workers) < WorkerAutoDefault {
-			workers = int(math.Min(float64(len(p.workers)+1), WorkerAutoDefault))
+			workers = min(len(p.workers)+1, WorkerAutoDefault)
 		}
 		if p.mrfWorkerSize < MRFWorkerAutoDefault {
-			mrfWorkers = int(math.Min(float64(p.mrfWorkerSize+1), MRFWorkerAutoDefault))
+			mrfWorkers = min(p.mrfWorkerSize+1, MRFWorkerAutoDefault)
 		}
 	}
+	if maxWorkers > 0 && workers > maxWorkers {
+		workers = maxWorkers
+	}
+
+	if maxWorkers > 0 && mrfWorkers > maxWorkers {
+		mrfWorkers = maxWorkers
+	}
 	p.priority = pri
+	p.maxWorkers = maxWorkers
 	p.mu.Unlock()
 	p.ResizeWorkers(workers, 0)
 	p.ResizeFailedWorkers(mrfWorkers)
@@ -2024,6 +2060,7 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
 		p.mu.RLock()
 		prio := p.priority
+		maxWorkers := p.maxWorkers
 		p.mu.RUnlock()
 		switch prio {
 		case "fast":
@@ -2031,16 +2068,18 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		case "slow":
 			logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic - we recommend increasing replication priority with `mc admin config set api replication_priority=auto`"), string(replicationSubsystem))
 		default:
-			if p.ActiveWorkers() < WorkerMaxLimit {
+			maxWorkers = min(maxWorkers, WorkerMaxLimit)
+			if p.ActiveWorkers() < maxWorkers {
 				p.mu.RLock()
-				workers := int(math.Min(float64(len(p.workers)+1), WorkerMaxLimit))
+				workers := min(len(p.workers)+1, maxWorkers)
 				existing := len(p.workers)
 				p.mu.RUnlock()
 				p.ResizeWorkers(workers, existing)
 			}
-			if p.ActiveMRFWorkers() < MRFWorkerMaxLimit {
+			maxMRFWorkers := min(maxWorkers, MRFWorkerMaxLimit)
+			if p.ActiveMRFWorkers() < maxMRFWorkers {
 				p.mu.RLock()
-				workers := int(math.Min(float64(p.mrfWorkerSize+1), MRFWorkerMaxLimit))
+				workers := min(p.mrfWorkerSize+1, maxMRFWorkers)
 				p.mu.RUnlock()
 				p.ResizeFailedWorkers(workers)
 			}
@@ -2078,6 +2117,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 		globalReplicationPool.queueMRFSave(doi.ToMRFEntry())
 		p.mu.RLock()
 		prio := p.priority
+		maxWorkers := p.maxWorkers
 		p.mu.RUnlock()
 		switch prio {
 		case "fast":
@@ -2085,9 +2125,10 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 		case "slow":
 			logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming deletes - we recommend increasing replication priority with `mc admin config set api replication_priority=auto`"), string(replicationSubsystem))
 		default:
-			if p.ActiveWorkers() < WorkerMaxLimit {
+			maxWorkers = min(maxWorkers, WorkerMaxLimit)
+			if p.ActiveWorkers() < maxWorkers {
 				p.mu.RLock()
-				workers := int(math.Min(float64(len(p.workers)+1), WorkerMaxLimit))
+				workers := min(len(p.workers)+1, maxWorkers)
 				existing := len(p.workers)
 				p.mu.RUnlock()
 				p.ResizeWorkers(workers, existing)
@@ -2097,13 +2138,12 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 }
 
 type replicationPoolOpts struct {
-	Priority string
+	Priority   string
+	MaxWorkers int
 }
 
 func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
-	globalReplicationPool = NewReplicationPool(ctx, objectAPI, replicationPoolOpts{
-		Priority: globalAPIConfig.getReplicationPriority(),
-	})
+	globalReplicationPool = NewReplicationPool(ctx, objectAPI, globalAPIConfig.getReplicationOpts())
 	globalReplicationStats = NewReplicationStats(ctx, objectAPI)
 	go globalReplicationStats.trackEWMA()
 }
@@ -2215,7 +2255,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
 		if rs != nil {
 			h, err := rs.ToHeader()
 			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Invalid range header for %s/%s(%s) - %w", bucket, object, opts.VersionID, err))
+				logger.LogIf(ctx, fmt.Errorf("invalid range header for %s/%s(%s) - %w", bucket, object, opts.VersionID, err))
 				continue
 			}
 			gopts.Set(xhttp.Range, h)
@@ -2308,6 +2348,127 @@ func scheduleReplication(ctx context.Context, oi ObjectInfo, o ObjectLayer, dsc
 	}
 }
 
+// proxyTaggingToRepTarget proxies tagging requests to remote targets for
+// active-active replicated setups
+func proxyTaggingToRepTarget(ctx context.Context, bucket, object string, tags *tags.Tags, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (proxy proxyResult) {
+	// this option is set when active-active replication is in place between site A -> B,
+	// and request hits site B that does not have the object yet.
+	if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
+		return proxy
+	}
+	var wg sync.WaitGroup
+	errs := make([]error, len(proxyTargets.Targets))
+	for idx, t := range proxyTargets.Targets {
+		tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, t.Arn)
+		if tgt == nil || globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
+			continue
+		}
+		// if proxying explicitly disabled on remote target
+		if tgt.disableProxy {
+			continue
+		}
+		idx := idx
+		wg.Add(1)
+		go func(idx int, tgt *TargetClient) {
+			defer wg.Done()
+			var err error
+			if tags != nil {
+				popts := minio.PutObjectTaggingOptions{
+					VersionID: opts.VersionID,
+					Internal: minio.AdvancedObjectTaggingOptions{
+						ReplicationProxyRequest: "true",
+					},
+				}
+				err = tgt.PutObjectTagging(ctx, tgt.Bucket, object, tags, popts)
+			} else {
+				dopts := minio.RemoveObjectTaggingOptions{
+					VersionID: opts.VersionID,
+					Internal: minio.AdvancedObjectTaggingOptions{
+						ReplicationProxyRequest: "true",
+					},
+				}
+				err = tgt.RemoveObjectTagging(ctx, tgt.Bucket, object, dopts)
+			}
+			if err != nil {
+				errs[idx] = err
+			}
+		}(idx, tgt)
+	}
+	wg.Wait()
+
+	var (
+		terr        error
+		taggedCount int
+	)
+	for _, err := range errs {
+		if err == nil {
+			taggedCount++
+			continue
+		}
+		errCode := minio.ToErrorResponse(err).Code
+		if errCode != "NoSuchKey" && errCode != "NoSuchVersion" {
+			terr = err
+		}
+	}
+	// don't return error if at least one target was tagged successfully
+	if taggedCount == 0 && terr != nil {
+		proxy.Err = terr
+	}
+	return proxy
+}
+
+// proxyGetTaggingToRepTarget proxies get tagging requests to remote targets for
+// active-active replicated setups
+func proxyGetTaggingToRepTarget(ctx context.Context, bucket, object string, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (tgs *tags.Tags, proxy proxyResult) {
+	// this option is set when active-active replication is in place between site A -> B,
+	// and request hits site B that does not have the object yet.
+	if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
+		return nil, proxy
+	}
+	var wg sync.WaitGroup
+	errs := make([]error, len(proxyTargets.Targets))
+	tagSlc := make([]map[string]string, len(proxyTargets.Targets))
+	for idx, t := range proxyTargets.Targets {
+		tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, t.Arn)
+		if tgt == nil || globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
+			continue
+		}
+		// if proxying explicitly disabled on remote target
+		if tgt.disableProxy {
+			continue
+		}
+		idx := idx
+		wg.Add(1)
+		go func(idx int, tgt *TargetClient) {
+			defer wg.Done()
+			var err error
+			gopts := minio.GetObjectTaggingOptions{
+				VersionID: opts.VersionID,
+				Internal: minio.AdvancedObjectTaggingOptions{
+					ReplicationProxyRequest: "true",
+				},
+			}
+			tgs, err = tgt.GetObjectTagging(ctx, tgt.Bucket, object, gopts)
+			if err != nil {
+				errs[idx] = err
+			} else {
+				tagSlc[idx] = tgs.ToMap()
+			}
+		}(idx, tgt)
+	}
+	wg.Wait()
+	for idx, err := range errs {
+		errCode := minio.ToErrorResponse(err).Code
+		if err != nil && errCode != "NoSuchKey" && errCode != "NoSuchVersion" {
+			return nil, proxyResult{Err: err}
+		}
+		if err == nil {
+			tgs, _ = tags.MapToObjectTags(tagSlc[idx])
+		}
+	}
+	return tgs, proxy
+}
+
 func scheduleReplicationDelete(ctx context.Context, dv DeletedObjectReplicationInfo, o ObjectLayer) {
 	globalReplicationPool.queueReplicaDeleteTask(dv)
 	for arn := range dv.ReplicationState.Targets {
@@ -3222,7 +3383,11 @@ func (p *ReplicationPool) persistToDrive(ctx context.Context, v MRFReplicateEntr
 		return r
 	}
 
-	for _, localDrive := range globalLocalDrives {
+	globalLocalDrivesMu.RLock()
+	localDrives := globalLocalDrives
+	globalLocalDrivesMu.RUnlock()
+
+	for _, localDrive := range localDrives {
 		r := newReader()
 		err := localDrive.CreateFile(ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), -1, r)
 		r.Close()
@@ -3284,7 +3449,12 @@ func (p *ReplicationPool) loadMRF() (mrfRec MRFReplicateEntries, err error) {
 
 		return re, nil
 	}
-	for _, localDrive := range globalLocalDrives {
+
+	globalLocalDrivesMu.RLock()
+	localDrives := globalLocalDrives
+	globalLocalDrivesMu.RUnlock()
+
+	for _, localDrive := range localDrives {
 		rc, err := localDrive.ReadFileStream(p.ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), 0, -1)
 		if err != nil {
 			continue

cmd/bucket-targets.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/url"
 	"sync"
@@ -351,6 +352,27 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 			return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
 		}
 	}
+
+	// Check if target is a MinIO server and alive
+	hcCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	scheme := "http"
+	if tgt.Secure {
+		scheme = "https"
+	}
+	result := <-sys.hcClient.Alive(hcCtx, madmin.AliveOpts{}, madmin.ServerProperties{
+		Endpoint: tgt.Endpoint,
+		Scheme:   scheme,
+	})
+
+	cancel()
+	if result.Error != nil {
+		return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: result.Error, AccessKey: tgt.Credentials.AccessKey}
+	}
+	if !result.Online {
+		err := errors.New("Health check timed out after 3 seconds")
+		return RemoteTargetConnectionErr{Err: err}
+	}
+
 	sys.Lock()
 	defer sys.Unlock()
 

cmd/common-main.go

@@ -46,10 +46,10 @@ import (
 	"github.com/inconshreveable/mousetrap"
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
+	consoleapi "github.com/minio/console/api"
+	"github.com/minio/console/api/operations"
 	consoleoauth2 "github.com/minio/console/pkg/auth/idp/oauth2"
 	consoleCerts "github.com/minio/console/pkg/certs"
-	"github.com/minio/console/restapi"
-	"github.com/minio/console/restapi/operations"
 	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
@@ -178,6 +178,23 @@ func minioConfigToConsoleFeatures() {
 	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
 	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
 
+	// This section sets Browser (console) stored config
+	if valueSCP := globalBrowserConfig.GetCSPolicy(); valueSCP != "" {
+		os.Setenv("CONSOLE_SECURE_CONTENT_SECURITY_POLICY", valueSCP)
+	}
+
+	if hstsSeconds := globalBrowserConfig.GetHSTSSeconds(); hstsSeconds > 0 {
+		isubdom := globalBrowserConfig.IsHSTSIncludeSubdomains()
+		isprel := globalBrowserConfig.IsHSTSPreload()
+		os.Setenv("CONSOLE_SECURE_STS_SECONDS", strconv.Itoa(hstsSeconds))
+		os.Setenv("CONSOLE_SECURE_STS_INCLUDE_SUB_DOMAINS", isubdom)
+		os.Setenv("CONSOLE_SECURE_STS_PRELOAD", isprel)
+	}
+
+	if valueRefer := globalBrowserConfig.GetReferPolicy(); valueRefer != "" {
+		os.Setenv("CONSOLE_SECURE_REFERRER_POLICY", valueRefer)
+	}
+
 	globalSubnetConfig.ApplyEnv()
 }
 
@@ -207,7 +224,7 @@ func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
 	return m
 }
 
-func initConsoleServer() (*restapi.Server, error) {
+func initConsoleServer() (*consoleapi.Server, error) {
 	// unset all console_ environment variables.
 	for _, cenv := range env.List(consolePrefix) {
 		os.Unsetenv(cenv)
@@ -225,9 +242,9 @@ func initConsoleServer() (*restapi.Server, error) {
 	}
 
 	// set certs before other console initialization
-	restapi.GlobalRootCAs, restapi.GlobalPublicCerts, restapi.GlobalTLSCertsManager = globalRootCAs, globalPublicCerts, globalTLSCerts
+	consoleapi.GlobalRootCAs, consoleapi.GlobalPublicCerts, consoleapi.GlobalTLSCertsManager = globalRootCAs, globalPublicCerts, globalTLSCerts
 
-	swaggerSpec, err := loads.Embedded(restapi.SwaggerJSON, restapi.FlatSwaggerJSON)
+	swaggerSpec, err := loads.Embedded(consoleapi.SwaggerJSON, consoleapi.FlatSwaggerJSON)
 	if err != nil {
 		return nil, err
 	}
@@ -238,18 +255,18 @@ func initConsoleServer() (*restapi.Server, error) {
 		// Disable console logging if server debug log is not enabled
 		noLog := func(string, ...interface{}) {}
 
-		restapi.LogInfo = noLog
-		restapi.LogError = noLog
+		consoleapi.LogInfo = noLog
+		consoleapi.LogError = noLog
 		api.Logger = noLog
 	}
 
 	// Pass in console application config. This needs to happen before the
 	// ConfigureAPI() call.
-	restapi.GlobalMinIOConfig = restapi.MinIOConfig{
+	consoleapi.GlobalMinIOConfig = consoleapi.MinIOConfig{
 		OpenIDProviders: buildOpenIDConsoleConfig(),
 	}
 
-	server := restapi.NewServer(api)
+	server := consoleapi.NewServer(api)
 	// register all APIs
 	server.ConfigureAPI()
 
@@ -257,16 +274,16 @@ func initConsoleServer() (*restapi.Server, error) {
 
 	server.Host = globalMinioConsoleHost
 	server.Port = consolePort
-	restapi.Port = globalMinioConsolePort
-	restapi.Hostname = globalMinioConsoleHost
+	consoleapi.Port = globalMinioConsolePort
+	consoleapi.Hostname = globalMinioConsoleHost
 
 	if globalIsTLS {
 		// If TLS certificates are provided enforce the HTTPS.
 		server.EnabledListeners = []string{"https"}
 		server.TLSPort = consolePort
 		// Need to store tls-port, tls-host un config variables so secure.middleware can read from there
-		restapi.TLSPort = globalMinioConsolePort
-		restapi.Hostname = globalMinioConsoleHost
+		consoleapi.TLSPort = globalMinioConsolePort
+		consoleapi.Hostname = globalMinioConsoleHost
 	}
 
 	return server, nil
@@ -310,84 +327,121 @@ func checkUpdate(mode string) {
 	logger.Info(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
 }
 
-func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
-	var dir string
-	var dirSet bool
-
-	switch {
-	case ctx.IsSet(option):
-		dir = ctx.String(option)
-		dirSet = true
-	case ctx.GlobalIsSet(option):
-		dir = ctx.GlobalString(option)
-		dirSet = true
-		// cli package does not expose parent's option option.  Below code is workaround.
-		if dir == "" || dir == getDefaultDir() {
-			dirSet = false // Unset to false since GlobalIsSet() true is a false positive.
-			if ctx.Parent().GlobalIsSet(option) {
-				dir = ctx.Parent().GlobalString(option)
-				dirSet = true
-			}
-		}
-	default:
-		// Neither local nor global option is provided.  In this case, try to use
-		// default directory.
+func newConfigDir(dir string, dirSet bool, getDefaultDir func() string) (*ConfigDir, error) {
+	if dir == "" {
 		dir = getDefaultDir()
-		if dir == "" {
-			logger.FatalIf(errInvalidArgument, "%s option must be provided", option)
-		}
 	}
 
 	if dir == "" {
-		logger.FatalIf(errors.New("empty directory"), "%s directory cannot be empty", option)
+		if !dirSet {
+			return nil, fmt.Errorf("missing option must be provided")
+		}
+		return nil, fmt.Errorf("provided option cannot be empty")
 	}
 
 	// Disallow relative paths, figure out absolute paths.
 	dirAbs, err := filepath.Abs(dir)
-	logger.FatalIf(err, "Unable to fetch absolute path for %s=%s", option, dir)
+	if err != nil {
+		return nil, err
+	}
+	err = mkdirAllIgnorePerm(dirAbs)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create the directory `%s`: %w", dirAbs, err)
+	}
 
-	logger.FatalIf(mkdirAllIgnorePerm(dirAbs), "Unable to create directory specified %s=%s", option, dir)
-
-	return &ConfigDir{path: dirAbs}, dirSet
+	return &ConfigDir{path: dirAbs}, nil
 }
 
-func handleCommonCmdArgs(ctx *cli.Context) {
+func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 	// Get "json" flag from command line argument and
-	// enable json and quite modes if json flag is turned on.
-	globalCLIContext.JSON = ctx.IsSet("json") || ctx.GlobalIsSet("json")
-	if globalCLIContext.JSON {
-		logger.EnableJSON()
-	}
-
+	ctxt.JSON = ctx.IsSet("json") || ctx.GlobalIsSet("json")
 	// Get quiet flag from command line argument.
-	globalCLIContext.Quiet = ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
-	if globalCLIContext.Quiet {
-		logger.EnableQuiet()
-	}
-
+	ctxt.Quiet = ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
 	// Get anonymous flag from command line argument.
-	globalCLIContext.Anonymous = ctx.IsSet("anonymous") || ctx.GlobalIsSet("anonymous")
-	if globalCLIContext.Anonymous {
-		logger.EnableAnonymous()
-	}
-
+	ctxt.Anonymous = ctx.IsSet("anonymous") || ctx.GlobalIsSet("anonymous")
 	// Fetch address option
-	addr := ctx.GlobalString("address")
-	if addr == "" || addr == ":"+GlobalMinioDefaultPort {
-		addr = ctx.String("address")
+	ctxt.Addr = ctx.GlobalString("address")
+	if ctxt.Addr == "" || ctxt.Addr == ":"+GlobalMinioDefaultPort {
+		ctxt.Addr = ctx.String("address")
 	}
 
 	// Fetch console address option
-	consoleAddr := ctx.GlobalString("console-address")
-	if consoleAddr == "" {
-		consoleAddr = ctx.String("console-address")
+	ctxt.ConsoleAddr = ctx.GlobalString("console-address")
+	if ctxt.ConsoleAddr == "" {
+		ctxt.ConsoleAddr = ctx.String("console-address")
 	}
 
+	// Check "no-compat" flag from command line argument.
+	ctxt.StrictS3Compat = !(ctx.IsSet("no-compat") || ctx.GlobalIsSet("no-compat"))
+	ctxt.PreAllocate = ctx.IsSet("pre-allocate") || ctx.GlobalIsSet("pre-allocate")
+
+	switch {
+	case ctx.IsSet("config-dir"):
+		ctxt.ConfigDir = ctx.String("config-dir")
+		ctxt.configDirSet = true
+	case ctx.GlobalIsSet("config-dir"):
+		ctxt.ConfigDir = ctx.GlobalString("config-dir")
+		ctxt.configDirSet = true
+	}
+
+	switch {
+	case ctx.IsSet("certs-dir"):
+		ctxt.CertsDir = ctx.String("certs-dir")
+		ctxt.certsDirSet = true
+	case ctx.GlobalIsSet("certs-dir"):
+		ctxt.CertsDir = ctx.GlobalString("certs-dir")
+		ctxt.certsDirSet = true
+	}
+
+	ctxt.FTP = ctx.StringSlice("ftp")
+	ctxt.SFTP = ctx.StringSlice("sftp")
+
+	ctxt.Interface = ctx.String("interface")
+	ctxt.UserTimeout = ctx.Duration("conn-user-timeout")
+	ctxt.ConnReadDeadline = ctx.Duration("conn-read-deadline")
+	ctxt.ConnWriteDeadline = ctx.Duration("conn-write-deadline")
+
+	ctxt.ShutdownTimeout = ctx.Duration("shutdown-timeout")
+	ctxt.IdleTimeout = ctx.Duration("idle-timeout")
+	ctxt.ReadHeaderTimeout = ctx.Duration("read-header-timeout")
+
+	if conf := ctx.String("config"); len(conf) > 0 {
+		err = mergeServerCtxtFromConfigFile(conf, ctxt)
+	} else {
+		err = mergeDisksLayoutFromArgs(serverCmdArgs(ctx), ctxt)
+	}
+
+	return err
+}
+
+func handleCommonArgs(ctxt serverCtxt) {
+	if ctxt.JSON {
+		logger.EnableJSON()
+	}
+	if ctxt.Quiet {
+		logger.EnableQuiet()
+	}
+	if ctxt.Anonymous {
+		logger.EnableAnonymous()
+	}
+
+	consoleAddr := ctxt.ConsoleAddr
+	addr := ctxt.Addr
+	configDir := ctxt.ConfigDir
+	configSet := ctxt.configDirSet
+	certsDir := ctxt.CertsDir
+	certsSet := ctxt.certsDirSet
+
 	if consoleAddr == "" {
 		p, err := xnet.GetFreePort()
 		if err != nil {
 			logger.FatalIf(err, "Unable to get free port for Console UI on the host")
 		}
+		// hold the port
+		l, err := net.Listen("TCP", fmt.Sprintf(":%s", p.String()))
+		if err == nil {
+			defer l.Close()
+		}
 		consoleAddr = net.JoinHostPort("", p.String())
 	}
 
@@ -417,16 +471,12 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 
 	globalMinioAddr = addr
 
-	// Check "no-compat" flag from command line argument.
-	globalCLIContext.StrictS3Compat = true
-	if ctx.IsSet("no-compat") || ctx.GlobalIsSet("no-compat") {
-		globalCLIContext.StrictS3Compat = false
-	}
-
 	// Set all config, certs and CAs directories.
-	var configSet, certsSet bool
-	globalConfigDir, configSet = newConfigDirFromCtx(ctx, "config-dir", defaultConfigDir.Get)
-	globalCertsDir, certsSet = newConfigDirFromCtx(ctx, "certs-dir", defaultCertsDir.Get)
+	var err error
+	globalConfigDir, err = newConfigDir(configDir, configSet, defaultConfigDir.Get)
+	logger.FatalIf(err, "Unable to initialize the (deprecated) config directory")
+	globalCertsDir, err = newConfigDir(certsDir, certsSet, defaultCertsDir.Get)
+	logger.FatalIf(err, "Unable to initialize the certs directory")
 
 	// Remove this code when we deprecate and remove config-dir.
 	// This code is to make sure we inherit from the config-dir
@@ -438,9 +488,11 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	globalCertsCADir = &ConfigDir{path: filepath.Join(globalCertsDir.Get(), certsCADir)}
 
 	logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get())
+}
 
-	// Check if we have configured a custom DNS cache TTL.
+func runDNSCache(ctx *cli.Context) {
 	dnsTTL := ctx.Duration("dns-cache-ttl")
+	// Check if we have configured a custom DNS cache TTL.
 	if dnsTTL <= 0 {
 		dnsTTL = 10 * time.Minute
 	}
@@ -621,7 +673,7 @@ func loadEnvVarsFromFiles() {
 	}
 }
 
-func handleCommonEnvVars() {
+func serverHandleEnvVars() {
 	var err error
 	globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.EnableOn))
 	if err != nil {
@@ -749,6 +801,10 @@ func handleCommonEnvVars() {
 		}
 	}
 
+	globalDisableFreezeOnBoot = env.Get("_MINIO_DISABLE_API_FREEZE_ON_BOOT", "") == "true" || serverDebugLog
+}
+
+func loadRootCredentials() {
 	// At this point, either both environment variables
 	// are defined or both are not defined.
 	// Check both cases and authenticate them if correctly defined
@@ -763,6 +819,9 @@ func handleCommonEnvVars() {
 		user = env.Get(config.EnvAccessKey, "")
 		password = env.Get(config.EnvSecretKey, "")
 		hasCredentials = true
+	} else if globalServerCtxt.RootUser != "" && globalServerCtxt.RootPwd != "" {
+		user, password = globalServerCtxt.RootUser, globalServerCtxt.RootPwd
+		hasCredentials = true
 	}
 	if hasCredentials {
 		cred, err := auth.CreateCredentials(user, password)
@@ -782,8 +841,6 @@ func handleCommonEnvVars() {
 	} else {
 		globalActiveCred = auth.DefaultCredentials
 	}
-
-	globalDisableFreezeOnBoot = env.Get("_MINIO_DISABLE_API_FREEZE_ON_BOOT", "") == "true" || serverDebugLog
 }
 
 // Initialize KMS global variable after valiadating and loading the configuration.

cmd/config-current.go

@@ -24,6 +24,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/minio/minio/internal/config/browser"
+
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/api"
@@ -74,6 +76,7 @@ func initHelp() {
 		config.DriveSubSys:          drive.DefaultKVS,
 		config.CacheSubSys:          cache.DefaultKVS,
 		config.BatchSubSys:          batch.DefaultKVS,
+		config.BrowserSubSys:        browser.DefaultKVS,
 	}
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
@@ -223,7 +226,12 @@ func initHelp() {
 		config.HelpKV{
 			Key:         config.CacheSubSys,
 			Type:        "string",
-			Description: "enable various cache optimizations on MinIO for reads",
+			Description: "enable cache plugin on MinIO for GET/HEAD requests",
+			Optional:    true,
+		},
+		config.HelpKV{
+			Key:         config.BrowserSubSys,
+			Description: "manage Browser HTTP specific features, such as Security headers, etc.",
 			Optional:    true,
 		},
 	}
@@ -273,6 +281,7 @@ func initHelp() {
 		config.CallhomeSubSys:       callhome.HelpCallhome,
 		config.DriveSubSys:          drive.HelpDrive,
 		config.CacheSubSys:          cache.Help,
+		config.BrowserSubSys:        browser.Help,
 	}
 
 	config.RegisterHelpSubSys(helpMap)
@@ -407,6 +416,10 @@ func validateSubSysConfig(ctx context.Context, s config.Config, subSys string, o
 				return err
 			}
 		}
+	case config.BrowserSubSys:
+		if _, err := browser.LookupConfig(s[config.BrowserSubSys][config.Default]); err != nil {
+			return err
+		}
 	default:
 		if config.LoggerSubSystems.Contains(subSys) {
 			if err := logger.ValidateSubSysConfig(ctx, s, subSys); err != nil {
@@ -587,6 +600,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			return fmt.Errorf("Unable to apply scanner config: %w", err)
 		}
 		// update dynamic scanner values.
+		scannerIdleMode.Store(scannerCfg.IdleMode)
 		scannerCycle.Store(scannerCfg.Cycle)
 		logger.LogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait))
 	case config.LoggerWebhookSubSys:
@@ -689,6 +703,12 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		} else {
 			globalCacheConfig.Update(cacheCfg)
 		}
+	case config.BrowserSubSys:
+		browserCfg, err := browser.LookupConfig(s[config.BrowserSubSys][config.Default])
+		if err != nil {
+			return fmt.Errorf("Unable to apply browser config: %w", err)
+		}
+		globalBrowserConfig.Update(browserCfg)
 	}
 	globalServerConfigMu.Lock()
 	defer globalServerConfigMu.Unlock()

cmd/data-scanner.go

@@ -65,8 +65,9 @@ var (
 	globalHealConfig heal.Config
 
 	// Sleeper values are updated when config is loaded.
-	scannerSleeper = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults
-	scannerCycle   = uatomic.NewDuration(dataScannerStartDelay)
+	scannerSleeper  = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults
+	scannerCycle    = uatomic.NewDuration(dataScannerStartDelay)
+	scannerIdleMode = uatomic.NewInt32(0) // default is throttled when idle
 )
 
 // initDataScanner will start the scanner in the background.
@@ -121,13 +122,13 @@ func readBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer) backgroundH
 	buf, err := readConfig(ctx, objAPI, backgroundHealInfoPath)
 	if err != nil {
 		if !errors.Is(err, errConfigNotFound) {
-			logger.LogIf(ctx, err)
+			logger.LogOnceIf(ctx, err, backgroundHealInfoPath)
 		}
 		return backgroundHealInfo{}
 	}
 	var info backgroundHealInfo
 	if err = json.Unmarshal(buf, &info); err != nil {
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, backgroundHealInfoPath)
 	}
 	return info
 }
@@ -203,7 +204,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
 			err := objAPI.NSScanner(ctx, results, uint32(cycleInfo.current), scanMode)
-			logger.LogIf(ctx, err)
+			logger.LogOnceIf(ctx, err, "ns-scanner")
 			res := map[string]string{"cycle": strconv.FormatUint(cycleInfo.current, 10)}
 			if err != nil {
 				res["error"] = err.Error()
@@ -223,7 +224,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 				binary.LittleEndian.PutUint64(tmp, cycleInfo.next)
 				tmp, _ = cycleInfo.MarshalMsg(tmp)
 				err = saveConfig(ctx, objAPI, dataUsageBloomNamePath, tmp)
-				logger.LogIf(ctx, err)
+				logger.LogOnceIf(ctx, err, dataUsageBloomNamePath)
 			}
 		}
 	}
@@ -246,6 +247,8 @@ type folderScanner struct {
 	healObjectSelect      uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
 	scanMode              madmin.HealScanMode
 
+	weSleep func() bool
+
 	disks       []StorageAPI
 	disksQuorum int
 
@@ -299,7 +302,7 @@ type folderScanner struct {
 // The returned cache will always be valid, but may not be updated from the existing.
 // Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
 // If the supplied context is canceled the function will return at the first chance.
-func scanDataFolder(ctx context.Context, disks []StorageAPI, basePath string, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode) (dataUsageCache, error) {
+func scanDataFolder(ctx context.Context, disks []StorageAPI, basePath string, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode, weSleep func() bool) (dataUsageCache, error) {
 	switch cache.Info.Name {
 	case "", dataUsageRoot:
 		return cache, errors.New("internal error: root scan attempted")
@@ -316,6 +319,7 @@ func scanDataFolder(ctx context.Context, disks []StorageAPI, basePath string, ca
 		dataUsageScannerDebug: false,
 		healObjectSelect:      0,
 		scanMode:              scanMode,
+		weSleep:               weSleep,
 		updates:               cache.Info.updates,
 		updateCurrentPath:     updatePath,
 		disks:                 disks,
@@ -372,6 +376,8 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 	done := ctx.Done()
 	scannerLogPrefix := color.Green("folder-scanner:")
 
+	noWait := func() {}
+
 	thisHash := hashPath(folder.name)
 	// Store initial compaction state.
 	wasCompacted := into.Compacted
@@ -401,8 +407,10 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		if !f.oldCache.Info.replication.Empty() && f.oldCache.Info.replication.Config.HasActiveRules(prefix, true) {
 			replicationCfg = f.oldCache.Info.replication
 		}
-		// Check if we can skip it due to bloom filter...
-		scannerSleeper.Sleep(ctx, dataScannerSleepPerFolder)
+
+		if f.weSleep() {
+			scannerSleeper.Sleep(ctx, dataScannerSleepPerFolder)
+		}
 
 		var existingFolders, newFolders []cachedFolder
 		var foundObjects bool
@@ -453,8 +461,11 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				return nil
 			}
 
-			// Dynamic time delay.
-			wait := scannerSleeper.Timer(ctx)
+			wait := noWait
+			if f.weSleep() {
+				// Dynamic time delay.
+				wait = scannerSleeper.Timer(ctx)
+			}
 
 			// Get file size, ignore errors.
 			item := scannerItem{
@@ -581,12 +592,12 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						foundAny = true
 						break
 					}
-					if next := f.updateCache.searchParent(parent); next == nil {
+					next := f.updateCache.searchParent(parent)
+					if next == nil {
 						foundAny = true
 						break
-					} else {
-						parent = *next
 					}
+					parent = *next
 				}
 				if !foundAny {
 					// Add non-compacted empty entry.
@@ -704,8 +715,11 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						// this object might be dangling.
 						entry, _ = entries.firstFound()
 					}
-					// wait timer per object.
-					wait := scannerSleeper.Timer(ctx)
+					wait := noWait
+					if f.weSleep() {
+						// wait timer per object.
+						wait = scannerSleeper.Timer(ctx)
+					}
 					defer wait()
 					f.updateCurrentPath(entry.name)
 					stopFn := globalScannerMetrics.log(scannerMetricHealAbandonedObject, f.root, entry.name)
@@ -729,7 +743,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 							versionID: "",
 						}, madmin.HealItemObject)
 						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-							logger.LogIf(ctx, err)
+							logger.LogOnceIf(ctx, err, entry.name)
 						}
 						foundObjs = foundObjs || err == nil
 						return
@@ -746,7 +760,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						}, madmin.HealItemObject)
 						stopFn(int(ver.Size))
 						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-							logger.LogIf(ctx, err)
+							logger.LogOnceIf(ctx, err, fiv.Name)
 						}
 						if err == nil {
 							successVersions++
@@ -1133,7 +1147,7 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 				err := o.CheckAbandonedParts(ctx, i.bucket, i.objectPath(), madmin.HealOpts{Remove: healDeleteDangling})
 				done()
 				if err != nil {
-					logger.LogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err))
+					logger.LogOnceIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err), i.objectPath())
 				}
 			}
 		}
@@ -1190,7 +1204,7 @@ func applyTransitionRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo)
 	if obj.DeleteMarker {
 		return false
 	}
-	globalTransitionState.queueTransitionTask(obj, event, src, false)
+	globalTransitionState.queueTransitionTask(obj, event, src)
 	return true
 }
 
@@ -1199,7 +1213,7 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return false
 		}
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, obj.Name)
 		return false
 	}
 	// Notification already sent in *expireTransitionedObject*, just return 'true' here.

cmd/data-usage-cache.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -32,6 +32,7 @@ import (
 	"time"
 
 	"github.com/cespare/xxhash/v2"
+	"github.com/dustin/go-humanize"
 	"github.com/klauspost/compress/zstd"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -46,6 +47,10 @@ import (
 // dataUsageHash is the hash type used.
 type dataUsageHash string
 
+// sizeHistogramV1 is size histogram V1, which has fewer intervals esp. between
+// 1024B and 1MiB.
+type sizeHistogramV1 [dataUsageBucketLenV1]uint64
+
 // sizeHistogram is a size histogram.
 type sizeHistogram [dataUsageBucketLen]uint64
 
@@ -204,8 +209,8 @@ func (r *replicationAllStats) clone() *replicationAllStats {
 	return &dst
 }
 
-//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6
-//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6
+//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
+//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
 
 //msgp:tuple dataUsageEntryV2
 type dataUsageEntryV2 struct {
@@ -263,14 +268,28 @@ type dataUsageEntryV6 struct {
 	Compacted        bool
 }
 
+type dataUsageEntryV7 struct {
+	Children dataUsageHashMap `msg:"ch"`
+	// These fields do no include any children.
+	Size             int64                `msg:"sz"`
+	Objects          uint64               `msg:"os"`
+	Versions         uint64               `msg:"vs"` // Versions that are not delete markers.
+	DeleteMarkers    uint64               `msg:"dms"`
+	ObjSizes         sizeHistogramV1      `msg:"szs"`
+	ObjVersions      versionsHistogram    `msg:"vh"`
+	ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
+	AllTierStats     *allTierStats        `msg:"ats,omitempty"`
+	Compacted        bool                 `msg:"c"`
+}
+
 // dataUsageCache contains a cache of data usage entries latest version.
 type dataUsageCache struct {
 	Info  dataUsageCacheInfo
 	Cache map[string]dataUsageEntry
 }
 
-//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6
-//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6
+//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7
+//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7
 
 // dataUsageCacheV2 contains a cache of data usage entries version 2.
 type dataUsageCacheV2 struct {
@@ -302,6 +321,12 @@ type dataUsageCacheV6 struct {
 	Cache map[string]dataUsageEntryV6
 }
 
+// dataUsageCacheV7 contains a cache of data usage entries version 7.
+type dataUsageCacheV7 struct {
+	Info  dataUsageCacheInfo
+	Cache map[string]dataUsageEntryV7
+}
+
 //msgp:ignore dataUsageEntryInfo
 type dataUsageEntryInfo struct {
 	Name   string
@@ -750,11 +775,42 @@ func (h *sizeHistogram) add(size int64) {
 	}
 }
 
+// mergeV1 is used to migrate data usage cache from sizeHistogramV1 to
+// sizeHistogram
+func (h *sizeHistogram) mergeV1(v sizeHistogramV1) {
+	var oidx, nidx int
+	for oidx < len(v) {
+		intOld, intNew := ObjectsHistogramIntervalsV1[oidx], ObjectsHistogramIntervals[nidx]
+		// skip intervals that aren't common to both histograms
+		if intOld.start != intNew.start || intOld.end != intNew.end {
+			nidx++
+			continue
+		}
+		h[nidx] += v[oidx]
+		oidx++
+		nidx++
+	}
+}
+
 // toMap returns the map to a map[string]uint64.
 func (h *sizeHistogram) toMap() map[string]uint64 {
 	res := make(map[string]uint64, dataUsageBucketLen)
+	var splCount uint64
 	for i, count := range h {
-		res[ObjectsHistogramIntervals[i].name] = count
+		szInt := ObjectsHistogramIntervals[i]
+		switch {
+		case humanize.KiByte == szInt.start && szInt.end == humanize.MiByte-1:
+			// spl interval: [1024B, 1MiB)
+			res[szInt.name] = splCount
+		case humanize.KiByte <= szInt.start && szInt.end <= humanize.MiByte-1:
+			// intervals that fall within the spl interval above; they
+			// appear earlier in this array of intervals, see
+			// ObjectsHistogramIntervals
+			splCount += count
+			fallthrough
+		default:
+			res[szInt.name] = count
+		}
 	}
 	return res
 }
@@ -1027,7 +1083,8 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 // Bumping the cache version will drop data from previous versions
 // and write new data with the new version.
 const (
-	dataUsageCacheVerCurrent = 7
+	dataUsageCacheVerCurrent = 8
+	dataUsageCacheVerV7      = 7
 	dataUsageCacheVerV6      = 6
 	dataUsageCacheVerV5      = 5
 	dataUsageCacheVerV4      = 4
@@ -1275,6 +1332,34 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 			}
 			d.Cache[k] = due
 		}
+		return nil
+	case dataUsageCacheVerV7:
+		// Zstd compressed.
+		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
+		if err != nil {
+			return err
+		}
+		defer dec.Close()
+		dold := &dataUsageCacheV7{}
+		if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil {
+			return err
+		}
+		d.Info = dold.Info
+		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
+		for k, v := range dold.Cache {
+			var szHist sizeHistogram
+			szHist.mergeV1(v.ObjSizes)
+			d.Cache[k] = dataUsageEntry{
+				Children:         v.Children,
+				Size:             v.Size,
+				Objects:          v.Objects,
+				Versions:         v.Versions,
+				ObjSizes:         szHist,
+				ReplicationStats: v.ReplicationStats,
+				Compacted:        v.Compacted,
+			}
+		}
+
 		return nil
 	case dataUsageCacheVerCurrent:
 		// Zstd compressed.

cmd/data-usage-cache_gen.go

@@ -1480,6 +1480,149 @@ func (z *dataUsageCacheV6) Msgsize() (s int) {
 	return
 }
 
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageCacheV7) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Info":
+			err = z.Info.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "Info")
+				return
+			}
+		case "Cache":
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Cache")
+				return
+			}
+			if z.Cache == nil {
+				z.Cache = make(map[string]dataUsageEntryV7, zb0002)
+			} else if len(z.Cache) > 0 {
+				for key := range z.Cache {
+					delete(z.Cache, key)
+				}
+			}
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 dataUsageEntryV7
+				za0001, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Cache")
+					return
+				}
+				err = za0002.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache", za0001)
+					return
+				}
+				z.Cache[za0001] = za0002
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageCacheV7) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Info":
+			bts, err = z.Info.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Info")
+				return
+			}
+		case "Cache":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Cache")
+				return
+			}
+			if z.Cache == nil {
+				z.Cache = make(map[string]dataUsageEntryV7, zb0002)
+			} else if len(z.Cache) > 0 {
+				for key := range z.Cache {
+					delete(z.Cache, key)
+				}
+			}
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 dataUsageEntryV7
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache")
+					return
+				}
+				bts, err = za0002.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache", za0001)
+					return
+				}
+				z.Cache[za0001] = za0002
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *dataUsageCacheV7) Msgsize() (s int) {
+	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize
+	if z.Cache != nil {
+		for za0001, za0002 := range z.Cache {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
+		}
+	}
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
@@ -2684,6 +2827,297 @@ func (z *dataUsageEntryV6) Msgsize() (s int) {
 	return
 }
 
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageEntryV7) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "ch":
+			err = z.Children.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "Children")
+				return
+			}
+		case "sz":
+			z.Size, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "Size")
+				return
+			}
+		case "os":
+			z.Objects, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "Objects")
+				return
+			}
+		case "vs":
+			z.Versions, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "Versions")
+				return
+			}
+		case "dms":
+			z.DeleteMarkers, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "DeleteMarkers")
+				return
+			}
+		case "szs":
+			var zb0002 uint32
+			zb0002, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "ObjSizes")
+				return
+			}
+			if zb0002 != uint32(dataUsageBucketLenV1) {
+				err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLenV1), Got: zb0002}
+				return
+			}
+			for za0001 := range z.ObjSizes {
+				z.ObjSizes[za0001], err = dc.ReadUint64()
+				if err != nil {
+					err = msgp.WrapError(err, "ObjSizes", za0001)
+					return
+				}
+			}
+		case "vh":
+			var zb0003 uint32
+			zb0003, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "ObjVersions")
+				return
+			}
+			if zb0003 != uint32(dataUsageVersionLen) {
+				err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
+				return
+			}
+			for za0002 := range z.ObjVersions {
+				z.ObjVersions[za0002], err = dc.ReadUint64()
+				if err != nil {
+					err = msgp.WrapError(err, "ObjVersions", za0002)
+					return
+				}
+			}
+		case "rs":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "ReplicationStats")
+					return
+				}
+				z.ReplicationStats = nil
+			} else {
+				if z.ReplicationStats == nil {
+					z.ReplicationStats = new(replicationAllStats)
+				}
+				err = z.ReplicationStats.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "ReplicationStats")
+					return
+				}
+			}
+		case "ats":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "AllTierStats")
+					return
+				}
+				z.AllTierStats = nil
+			} else {
+				if z.AllTierStats == nil {
+					z.AllTierStats = new(allTierStats)
+				}
+				err = z.AllTierStats.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "AllTierStats")
+					return
+				}
+			}
+		case "c":
+			z.Compacted, err = dc.ReadBool()
+			if err != nil {
+				err = msgp.WrapError(err, "Compacted")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageEntryV7) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "ch":
+			bts, err = z.Children.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Children")
+				return
+			}
+		case "sz":
+			z.Size, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Size")
+				return
+			}
+		case "os":
+			z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Objects")
+				return
+			}
+		case "vs":
+			z.Versions, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Versions")
+				return
+			}
+		case "dms":
+			z.DeleteMarkers, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "DeleteMarkers")
+				return
+			}
+		case "szs":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ObjSizes")
+				return
+			}
+			if zb0002 != uint32(dataUsageBucketLenV1) {
+				err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLenV1), Got: zb0002}
+				return
+			}
+			for za0001 := range z.ObjSizes {
+				z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "ObjSizes", za0001)
+					return
+				}
+			}
+		case "vh":
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ObjVersions")
+				return
+			}
+			if zb0003 != uint32(dataUsageVersionLen) {
+				err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
+				return
+			}
+			for za0002 := range z.ObjVersions {
+				z.ObjVersions[za0002], bts, err = msgp.ReadUint64Bytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "ObjVersions", za0002)
+					return
+				}
+			}
+		case "rs":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.ReplicationStats = nil
+			} else {
+				if z.ReplicationStats == nil {
+					z.ReplicationStats = new(replicationAllStats)
+				}
+				bts, err = z.ReplicationStats.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "ReplicationStats")
+					return
+				}
+			}
+		case "ats":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.AllTierStats = nil
+			} else {
+				if z.AllTierStats == nil {
+					z.AllTierStats = new(allTierStats)
+				}
+				bts, err = z.AllTierStats.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "AllTierStats")
+					return
+				}
+			}
+		case "c":
+			z.Compacted, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Compacted")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *dataUsageEntryV7) Msgsize() (s int) {
+	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLenV1 * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 3
+	if z.ReplicationStats == nil {
+		s += msgp.NilSize
+	} else {
+		s += z.ReplicationStats.Msgsize()
+	}
+	s += 4
+	if z.AllTierStats == nil {
+		s += msgp.NilSize
+	} else {
+		s += z.AllTierStats.Msgsize()
+	}
+	s += 2 + msgp.BoolSize
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
 	{
@@ -3705,6 +4139,84 @@ func (z *sizeHistogram) Msgsize() (s int) {
 	return
 }
 
+// DecodeMsg implements msgp.Decodable
+func (z *sizeHistogramV1) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageBucketLenV1) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLenV1), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *sizeHistogramV1) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteArrayHeader(uint32(dataUsageBucketLenV1))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for za0001 := range z {
+		err = en.WriteUint64(z[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *sizeHistogramV1) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLenV1))
+	for za0001 := range z {
+		o = msgp.AppendUint64(o, z[za0001])
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *sizeHistogramV1) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageBucketLenV1) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLenV1), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *sizeHistogramV1) Msgsize() (s int) {
+	s = msgp.ArrayHeaderSize + (dataUsageBucketLenV1 * (msgp.Uint64Size))
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *tierStats) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte

cmd/data-usage-cache_gen_test.go

@@ -1084,6 +1084,119 @@ func BenchmarkDecodesizeHistogram(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalsizeHistogramV1(t *testing.T) {
+	v := sizeHistogramV1{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgsizeHistogramV1(b *testing.B) {
+	v := sizeHistogramV1{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgsizeHistogramV1(b *testing.B) {
+	v := sizeHistogramV1{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalsizeHistogramV1(b *testing.B) {
+	v := sizeHistogramV1{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodesizeHistogramV1(t *testing.T) {
+	v := sizeHistogramV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodesizeHistogramV1 Msgsize() is inaccurate")
+	}
+
+	vn := sizeHistogramV1{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodesizeHistogramV1(b *testing.B) {
+	v := sizeHistogramV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodesizeHistogramV1(b *testing.B) {
+	v := sizeHistogramV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshaltierStats(t *testing.T) {
 	v := tierStats{}
 	bts, err := v.MarshalMsg(nil)

cmd/data-usage-cache_test.go

@@ -0,0 +1,98 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/dustin/go-humanize"
+)
+
+func TestSizeHistogramToMap(t *testing.T) {
+	tests := []struct {
+		sizes []int64
+		want  map[string]uint64
+	}{
+		{
+			sizes: []int64{100, 1000, 72_000, 100_000},
+			want: map[string]uint64{
+				"LESS_THAN_1024_B":         2,
+				"BETWEEN_64_KB_AND_256_KB": 2,
+				"BETWEEN_1024B_AND_1_MB":   2,
+			},
+		},
+		{
+			sizes: []int64{100, 1000, 2000, 100_000, 13 * humanize.MiByte},
+			want: map[string]uint64{
+				"LESS_THAN_1024_B":         2,
+				"BETWEEN_1024_B_AND_64_KB": 1,
+				"BETWEEN_64_KB_AND_256_KB": 1,
+				"BETWEEN_1024B_AND_1_MB":   2,
+				"BETWEEN_10_MB_AND_64_MB":  1,
+			},
+		},
+	}
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("Test-%d", i), func(t *testing.T) {
+			var h sizeHistogram
+			for _, sz := range test.sizes {
+				h.add(sz)
+			}
+			got := h.toMap()
+			exp := test.want
+			// what is in exp is in got
+			for k := range exp {
+				if exp[k] != got[k] {
+					t.Fatalf("interval %s: Expected %d values but got %d values\n", k, exp[k], got[k])
+				}
+			}
+			// what is absent in exp is absent in got too
+			for k := range got {
+				if _, ok := exp[k]; !ok && got[k] > 0 {
+					t.Fatalf("Unexpected interval: %s has value %d\n", k, got[k])
+				}
+			}
+		})
+	}
+}
+
+func TestMigrateSizeHistogramFromV1(t *testing.T) {
+	tests := []struct {
+		v    sizeHistogramV1
+		want sizeHistogram
+	}{
+		{
+			v:    sizeHistogramV1{0: 10, 1: 20, 2: 3},
+			want: sizeHistogram{0: 10, 5: 20, 6: 3},
+		},
+		{
+			v:    sizeHistogramV1{0: 10, 1: 20, 2: 3, 3: 4, 4: 5, 5: 6, 6: 7},
+			want: sizeHistogram{0: 10, 5: 20, 6: 3, 7: 4, 8: 5, 9: 6, 10: 7},
+		},
+	}
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+			var got sizeHistogram
+			got.mergeV1(test.v)
+			if got != test.want {
+				t.Fatalf("Expected %v but got %v", test.want, got)
+			}
+		})
+	}
+}

cmd/data-usage.go

@@ -56,7 +56,7 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
 			attempts = 1
 		}
 		if err = saveConfig(ctx, objAPI, dataUsageObjNamePath, dataUsageJSON); err != nil {
-			logger.LogIf(ctx, err)
+			logger.LogOnceIf(ctx, err, dataUsageObjNamePath)
 		}
 		attempts++
 	}

cmd/data-usage_test.go

@@ -62,7 +62,9 @@ func TestDataUsageUpdate(t *testing.T) {
 		return
 	}
 
-	got, err := scanDataFolder(context.Background(), nil, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0)
+	weSleep := func() bool { return false }
+
+	got, err := scanDataFolder(context.Background(), nil, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0, weSleep)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -80,7 +82,7 @@ func TestDataUsageUpdate(t *testing.T) {
 			size:    1322310,
 			flatten: true,
 			objs:    8,
-			oSizes:  sizeHistogram{0: 2, 1: 6},
+			oSizes:  sizeHistogram{0: 2, 1: 3, 2: 2, 4: 1},
 		},
 		{
 			path:   "/",
@@ -92,7 +94,7 @@ func TestDataUsageUpdate(t *testing.T) {
 			path:   "/dir1",
 			size:   1302010,
 			objs:   5,
-			oSizes: sizeHistogram{0: 1, 1: 4},
+			oSizes: sizeHistogram{0: 1, 1: 1, 2: 2, 4: 1},
 		},
 		{
 			path:  "/dir1/dira",
@@ -165,7 +167,6 @@ func TestDataUsageUpdate(t *testing.T) {
 			size: 200,
 		},
 	}
-
 	createUsageTestFiles(t, base, bucket, files)
 	err = os.RemoveAll(filepath.Join(base, bucket, "dir1/dira/dirasub/dcfile"))
 	if err != nil {
@@ -173,7 +174,7 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = scanDataFolder(context.Background(), nil, base, got, getSize, 0)
+		got, err = scanDataFolder(context.Background(), nil, base, got, getSize, 0, weSleep)
 		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
@@ -192,14 +193,14 @@ func TestDataUsageUpdate(t *testing.T) {
 			size:    363515,
 			flatten: true,
 			objs:    14,
-			oSizes:  sizeHistogram{0: 7, 1: 7},
+			oSizes:  sizeHistogram{0: 7, 1: 5, 2: 2},
 		},
 		{
 			path:    "/dir1",
 			size:    342210,
 			objs:    7,
 			flatten: false,
-			oSizes:  sizeHistogram{0: 2, 1: 5},
+			oSizes:  sizeHistogram{0: 2, 1: 3, 2: 2},
 		},
 		{
 			path:   "/newfolder",
@@ -280,7 +281,10 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		}
 		return
 	}
-	got, err := scanDataFolder(context.Background(), nil, base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize, 0)
+
+	weSleep := func() bool { return false }
+
+	got, err := scanDataFolder(context.Background(), nil, base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize, 0, weSleep)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -303,7 +307,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 			path:   "flat",
 			size:   1322310 + expectSize,
 			objs:   8 + expectSize,
-			oSizes: sizeHistogram{0: 2 + expectSize, 1: 6},
+			oSizes: sizeHistogram{0: 2 + expectSize, 1: 3, 2: 2, 4: 1},
 		},
 		{
 			path:   "bucket/",
@@ -316,7 +320,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 			path:   "bucket/dir1",
 			size:   1302010,
 			objs:   5,
-			oSizes: sizeHistogram{0: 1, 1: 4},
+			oSizes: sizeHistogram{0: 1, 1: 1, 2: 2, 4: 1},
 		},
 		{
 			// Gets compacted at this level...
@@ -414,7 +418,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = scanDataFolder(context.Background(), nil, base, got, getSize, 0)
+		got, err = scanDataFolder(context.Background(), nil, base, got, getSize, 0, weSleep)
 		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
@@ -431,13 +435,13 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 			path:   "flat",
 			size:   363515 + expectSize,
 			objs:   14 + expectSize,
-			oSizes: sizeHistogram{0: 7 + expectSize, 1: 7},
+			oSizes: sizeHistogram{0: 7 + expectSize, 1: 5, 2: 2},
 		},
 		{
 			path:   "bucket/dir1",
 			size:   342210,
 			objs:   7,
-			oSizes: sizeHistogram{0: 2, 1: 5},
+			oSizes: sizeHistogram{0: 2, 1: 3, 2: 2},
 		},
 		{
 			path:   "bucket/",
@@ -562,7 +566,8 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 		}
 		return
 	}
-	want, err := scanDataFolder(context.Background(), nil, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0)
+	weSleep := func() bool { return false }
+	want, err := scanDataFolder(context.Background(), nil, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0, weSleep)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/encryption-v1.go

@@ -209,9 +209,6 @@ func DecryptETags(ctx context.Context, k kms.KMS, objects []ObjectInfo) error {
 // uploaded by the user using multipart mechanism:
 // initiate new multipart, upload part, complete upload
 func (o *ObjectInfo) isMultipart() bool {
-	if len(o.Parts) == 0 {
-		return false
-	}
 	_, encrypted := crypto.IsEncrypted(o.UserDefined)
 	if encrypted {
 		if !crypto.IsMultiPart(o.UserDefined) {
@@ -228,7 +225,7 @@ func (o *ObjectInfo) isMultipart() bool {
 	// Further check if this object is uploaded using multipart mechanism
 	// by the user and it is not about Erasure internally splitting the
 	// object into parts in PutObject()
-	return !(o.backendType == BackendErasure && len(o.ETag) == 32)
+	return len(o.ETag) != 32
 }
 
 // ParseSSECopyCustomerRequest parses the SSE-C header fields of the provided request.
@@ -1082,6 +1079,30 @@ func (o *ObjectInfo) metadataDecrypter() objectMetaDecryptFn {
 	}
 }
 
+// decryptChecksums will attempt to decode checksums and return it/them if set.
+// if part > 0, and we have the checksum for the part that will be returned.
+func (o *ObjectInfo) decryptPartsChecksums() {
+	data := o.Checksum
+	if len(data) == 0 {
+		return
+	}
+	if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted {
+		decrypted, err := o.metadataDecrypter()("object-checksum", data)
+		if err != nil {
+			logger.LogIf(GlobalContext, err)
+			return
+		}
+		data = decrypted
+	}
+	cs := hash.ReadPartCheckSums(data)
+	if len(cs) == len(o.Parts) {
+		for i := range o.Parts {
+			o.Parts[i].Checksums = cs[i]
+		}
+	}
+	return
+}
+
 // metadataEncryptFn provides an encryption function for metadata.
 // Will return nil, nil if unencrypted.
 func (o *ObjectInfo) metadataEncryptFn(headers http.Header) (objectMetaEncryptFn, error) {

cmd/endpoint-ellipses.go

@@ -18,12 +18,15 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
+	"net/url"
 	"runtime"
 	"sort"
 	"strconv"
 	"strings"
 
+	"github.com/cespare/xxhash/v2"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/pkg/v2/ellipses"
@@ -335,13 +338,128 @@ const (
 
 var globalCustomErasureDriveCount = false
 
-// CreateServerEndpoints - validates and creates new endpoints from input args, supports
-// both ellipses and without ellipses transparently.
-func createServerEndpoints(serverAddr string, args ...string) (
-	endpointServerPools EndpointServerPools, setupType SetupType, err error,
-) {
+type node struct {
+	nodeName string
+	disks    []string
+}
+
+type endpointsList []node
+
+func (el *endpointsList) add(arg string) error {
+	u, err := url.Parse(arg)
+	if err != nil {
+		return err
+	}
+	found := false
+	list := *el
+	for i := range list {
+		if list[i].nodeName == u.Host {
+			list[i].disks = append(list[i].disks, u.String())
+			found = true
+			break
+		}
+	}
+	if !found {
+		list = append(list, node{nodeName: u.Host, disks: []string{u.String()}})
+	}
+	*el = list
+	return nil
+}
+
+// buildDisksLayoutFromConfFile supports with and without ellipses transparently.
+func buildDisksLayoutFromConfFile(pools [][]string) (layout disksLayout, err error) {
+	if len(pools) == 0 {
+		return layout, errInvalidArgument
+	}
+
+	for _, list := range pools {
+		var endpointsList endpointsList
+
+		for _, arg := range list {
+			switch {
+			case ellipses.HasList(arg):
+				patterns, err := ellipses.FindListPatterns(arg)
+				if err != nil {
+					return layout, err
+				}
+				for _, exp := range patterns.Expand() {
+					for _, ep := range exp {
+						if err := endpointsList.add(ep); err != nil {
+							return layout, err
+						}
+					}
+				}
+			case ellipses.HasEllipses(arg):
+				patterns, err := ellipses.FindEllipsesPatterns(arg)
+				if err != nil {
+					return layout, err
+				}
+				for _, exp := range patterns.Expand() {
+					if err := endpointsList.add(strings.Join(exp, "")); err != nil {
+						return layout, err
+					}
+				}
+			default:
+				if err := endpointsList.add(arg); err != nil {
+					return layout, err
+				}
+			}
+		}
+
+		var stopping bool
+		var singleNode bool
+		var eps []string
+
+		for i := 0; ; i++ {
+			for _, node := range endpointsList {
+				if node.nodeName == "" {
+					singleNode = true
+				}
+
+				if len(node.disks) <= i {
+					stopping = true
+					continue
+				}
+				if stopping {
+					return layout, errors.New("number of disks per node does not match")
+				}
+				eps = append(eps, node.disks[i])
+			}
+			if stopping {
+				break
+			}
+		}
+
+		for _, node := range endpointsList {
+			if node.nodeName != "" && singleNode {
+				return layout, errors.New("all arguments must but either single node or distributed")
+			}
+		}
+
+		setArgs, err := GetAllSets(eps...)
+		if err != nil {
+			return layout, err
+		}
+
+		h := xxhash.New()
+		for _, s := range setArgs {
+			for _, d := range s {
+				h.WriteString(d)
+			}
+		}
+
+		layout.pools = append(layout.pools, poolDisksLayout{
+			cmdline: fmt.Sprintf("hash:%x", h.Sum(nil)),
+			layout:  setArgs,
+		})
+	}
+	return
+}
+
+// mergeDisksLayoutFromArgs supports with and without ellipses transparently.
+func mergeDisksLayoutFromArgs(args []string, ctxt *serverCtxt) (err error) {
 	if len(args) == 0 {
-		return nil, -1, errInvalidArgument
+		return errInvalidArgument
 	}
 
 	ok := true
@@ -349,44 +467,42 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		ok = ok && !ellipses.HasEllipses(arg)
 	}
 
+	var setArgs [][]string
+
 	// None of the args have ellipses use the old style.
 	if ok {
-		setArgs, err := GetAllSets(args...)
+		setArgs, err = GetAllSets(args...)
 		if err != nil {
-			return nil, -1, err
+			return err
 		}
-		endpointList, newSetupType, err := CreateEndpoints(serverAddr, setArgs...)
-		if err != nil {
-			return nil, -1, err
+		ctxt.Layout = disksLayout{
+			legacy: true,
+			pools:  []poolDisksLayout{{layout: setArgs}},
 		}
-		for i := range endpointList {
-			endpointList[i].SetPoolIndex(0)
-		}
-		endpointServerPools = append(endpointServerPools, PoolEndpoints{
-			Legacy:       true,
-			SetCount:     len(setArgs),
-			DrivesPerSet: len(setArgs[0]),
-			Endpoints:    endpointList,
-			CmdLine:      strings.Join(args, " "),
-			Platform:     fmt.Sprintf("OS: %s | Arch: %s", runtime.GOOS, runtime.GOARCH),
-		})
-		setupType = newSetupType
-		return endpointServerPools, setupType, nil
+		return
 	}
 
-	var poolArgs [][][]string
 	for _, arg := range args {
 		if !ellipses.HasEllipses(arg) && len(args) > 1 {
 			// TODO: support SNSD deployments to be decommissioned in future
-			return nil, -1, fmt.Errorf("all args must have ellipses for pool expansion (%w) args: %s", errInvalidArgument, args)
+			return fmt.Errorf("all args must have ellipses for pool expansion (%w) args: %s", errInvalidArgument, args)
 		}
-
-		setArgs, err := GetAllSets(arg)
+		setArgs, err = GetAllSets(arg)
 		if err != nil {
-			return nil, -1, err
+			return err
 		}
+		ctxt.Layout.pools = append(ctxt.Layout.pools, poolDisksLayout{cmdline: arg, layout: setArgs})
+	}
+	return
+}
 
-		poolArgs = append(poolArgs, setArgs)
+// CreateServerEndpoints - validates and creates new endpoints from input args, supports
+// both ellipses and without ellipses transparently.
+func createServerEndpoints(serverAddr string, poolArgs []poolDisksLayout, legacy bool) (
+	endpointServerPools EndpointServerPools, setupType SetupType, err error,
+) {
+	if len(poolArgs) == 0 {
+		return nil, -1, errInvalidArgument
 	}
 
 	poolEndpoints, setupType, err := CreatePoolEndpoints(serverAddr, poolArgs...)
@@ -396,11 +512,12 @@ func createServerEndpoints(serverAddr string, args ...string) (
 
 	for i, endpointList := range poolEndpoints {
 		if err = endpointServerPools.Add(PoolEndpoints{
-			SetCount:     len(poolArgs[i]),
-			DrivesPerSet: len(poolArgs[i][0]),
+			Legacy:       legacy,
+			SetCount:     len(poolArgs[i].layout),
+			DrivesPerSet: len(poolArgs[i].layout[0]),
 			Endpoints:    endpointList,
 			Platform:     fmt.Sprintf("OS: %s | Arch: %s", runtime.GOOS, runtime.GOARCH),
-			CmdLine:      args[i],
+			CmdLine:      poolArgs[i].cmdline,
 		}); err != nil {
 			return nil, -1, err
 		}

cmd/endpoint-ellipses_test.go

@@ -54,15 +54,20 @@ func TestCreateServerEndpoints(t *testing.T) {
 		{":9001", []string{"http://localhost:9001/export{01...64}"}, true},
 	}
 
-	for _, testCase := range testCases {
+	for i, testCase := range testCases {
 		testCase := testCase
 		t.Run("", func(t *testing.T) {
-			_, _, err := createServerEndpoints(testCase.serverAddr, testCase.args...)
+			srvCtxt := serverCtxt{}
+			err := mergeDisksLayoutFromArgs(testCase.args, &srvCtxt)
 			if err != nil && testCase.success {
-				t.Errorf("Expected success but failed instead %s", err)
+				t.Fatalf("Test %d: unexpected error: %v", i+1, err)
+			}
+			_, _, err = createServerEndpoints(testCase.serverAddr, srvCtxt.Layout.pools, srvCtxt.Layout.legacy)
+			if err != nil && testCase.success {
+				t.Errorf("Test %d: Expected success but failed instead %s", i+1, err)
 			}
 			if err == nil && !testCase.success {
-				t.Errorf("Expected failure but passed instead")
+				t.Errorf("Test %d: Expected failure but passed instead", i+1)
 			}
 		})
 	}

cmd/endpoint.go

@@ -103,7 +103,7 @@ func (endpoint Endpoint) GridHost() string {
 
 // UpdateIsLocal - resolves the host and updates if it is local or not.
 func (endpoint *Endpoint) UpdateIsLocal() (err error) {
-	if !endpoint.IsLocal {
+	if endpoint.Host != "" {
 		endpoint.IsLocal, err = isLocalHost(endpoint.Hostname(), endpoint.Port(), globalMinioPort)
 		if err != nil {
 			return err
@@ -522,6 +522,16 @@ func (endpoints Endpoints) UpdateIsLocal() error {
 					continue
 				}
 
+				if endpoints[i].Host == "" {
+					resolvedList[i] = true
+					endpoints[i].IsLocal = true
+					epsResolved++
+					if !foundLocal {
+						foundLocal = true
+					}
+					continue
+				}
+
 				// Log the message to console about the host resolving
 				reqInfo := (&logger.ReqInfo{}).AppendTags(
 					"host",
@@ -705,6 +715,17 @@ func (p PoolEndpointList) UpdateIsLocal() error {
 						continue
 					}
 
+					if endpoint.Host == "" {
+						if !foundLocal {
+							foundLocal = true
+						}
+						endpoint.IsLocal = true
+						endpoints[j] = endpoint
+						epsResolved++
+						resolvedList[endpoint] = true
+						continue
+					}
+
 					// Log the message to console about the host resolving
 					reqInfo := (&logger.ReqInfo{}).AppendTags(
 						"host",
@@ -803,11 +824,23 @@ func (p PoolEndpointList) UpdateIsLocal() error {
 	return nil
 }
 
+func isEmptyLayout(poolsLayout ...poolDisksLayout) bool {
+	return len(poolsLayout) == 0 || len(poolsLayout[0].layout) == 0 || len(poolsLayout[0].layout[0]) == 0 || len(poolsLayout[0].layout[0][0]) == 0
+}
+
+func isSingleDriveLayout(poolsLayout ...poolDisksLayout) bool {
+	return len(poolsLayout) == 1 && len(poolsLayout[0].layout) == 1 && len(poolsLayout[0].layout[0]) == 1
+}
+
 // CreatePoolEndpoints creates a list of endpoints per pool, resolves their relevant hostnames and
 // discovers those are local or remote.
-func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints, SetupType, error) {
+func CreatePoolEndpoints(serverAddr string, poolsLayout ...poolDisksLayout) ([]Endpoints, SetupType, error) {
 	var setupType SetupType
 
+	if isEmptyLayout(poolsLayout...) {
+		return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg("invalid number of endpoints")
+	}
+
 	// Check whether serverAddr is valid for this host.
 	if err := CheckLocalServerAddr(serverAddr); err != nil {
 		return nil, setupType, err
@@ -815,11 +848,11 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 
 	_, serverAddrPort := mustSplitHostPort(serverAddr)
 
-	poolEndpoints := make(PoolEndpointList, len(poolArgs))
+	poolEndpoints := make(PoolEndpointList, len(poolsLayout))
 
 	// For single arg, return single drive EC setup.
-	if len(poolArgs) == 1 && len(poolArgs[0]) == 1 && len(poolArgs[0][0]) == 1 && len(poolArgs[0][0][0]) == 1 {
-		endpoint, err := NewEndpoint(poolArgs[0][0][0])
+	if isSingleDriveLayout(poolsLayout...) {
+		endpoint, err := NewEndpoint(poolsLayout[0].layout[0][0])
 		if err != nil {
 			return nil, setupType, err
 		}
@@ -847,11 +880,13 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 		return poolEndpoints, setupType, nil
 	}
 
-	for poolIdx, args := range poolArgs {
+	uniqueArgs := set.NewStringSet()
+
+	for poolIdx, pool := range poolsLayout {
 		var endpoints Endpoints
-		for setIdx, iargs := range args {
+		for setIdx, setLayout := range pool.layout {
 			// Convert args to endpoints
-			eps, err := NewEndpoints(iargs...)
+			eps, err := NewEndpoints(setLayout...)
 			if err != nil {
 				return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 			}
@@ -877,26 +912,10 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 		poolEndpoints[poolIdx] = endpoints
 	}
 
-	for _, endpoints := range poolEndpoints {
-		// Return Erasure setup when all endpoints are path style.
-		if endpoints[0].Type() == PathEndpointType {
-			setupType = ErasureSetupType
-		}
-		if endpoints[0].Type() == URLEndpointType && setupType != DistErasureSetupType {
-			setupType = DistErasureSetupType
-		}
-	}
-
-	if setupType == ErasureSetupType {
-		return poolEndpoints, setupType, nil
-	}
-
 	if err := poolEndpoints.UpdateIsLocal(); err != nil {
 		return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 	}
 
-	uniqueArgs := set.NewStringSet()
-
 	for i, endpoints := range poolEndpoints {
 		// Here all endpoints are URL style.
 		endpointPathSet := set.NewStringSet()
@@ -906,7 +925,7 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 
 		for _, endpoint := range endpoints {
 			endpointPathSet.Add(endpoint.Path)
-			if endpoint.IsLocal {
+			if endpoint.IsLocal && endpoint.Host != "" {
 				localServerHostSet.Add(endpoint.Hostname())
 
 				_, port, err := net.SplitHostPort(endpoint.Host)
@@ -922,9 +941,8 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 		orchestrated := IsKubernetes() || IsDocker()
 		reverseProxy := (env.Get("_MINIO_REVERSE_PROXY", "") != "") && ((env.Get("MINIO_CI_CD", "") != "") || (env.Get("CI", "") != ""))
 		// If not orchestrated
-		if !orchestrated &&
-			// and not setup in reverse proxy
-			!reverseProxy {
+		// and not setup in reverse proxy
+		if !orchestrated && !reverseProxy {
 			// Check whether same path is not used in endpoints of a host on different port.
 			// Only verify this on baremetal setups, DNS is not available in orchestrated
 			// environments so we can't do much here.
@@ -932,14 +950,18 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 			hostIPCache := make(map[string]set.StringSet)
 			for _, endpoint := range endpoints {
 				host := endpoint.Hostname()
-				hostIPSet, ok := hostIPCache[host]
-				if !ok {
-					var err error
-					hostIPSet, err = getHostIP(host)
-					if err != nil {
-						return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
+				var hostIPSet set.StringSet
+				if host != "" {
+					var ok bool
+					hostIPSet, ok = hostIPCache[host]
+					if !ok {
+						var err error
+						hostIPSet, err = getHostIP(host)
+						if err != nil {
+							return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
+						}
+						hostIPCache[host] = hostIPSet
 					}
-					hostIPCache[host] = hostIPSet
 				}
 				if IPSet, ok := pathIPMap[endpoint.Path]; ok {
 					if !IPSet.Intersection(hostIPSet).IsEmpty() {
@@ -970,12 +992,14 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 
 		// Add missing port in all endpoints.
 		for i := range endpoints {
-			_, port, err := net.SplitHostPort(endpoints[i].Host)
-			if err != nil {
-				endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort)
-			} else if endpoints[i].IsLocal && serverAddrPort != port {
-				// If endpoint is local, but port is different than serverAddrPort, then make it as remote.
-				endpoints[i].IsLocal = false
+			if endpoints[i].Host != "" {
+				_, port, err := net.SplitHostPort(endpoints[i].Host)
+				if err != nil {
+					endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort)
+				} else if endpoints[i].IsLocal && serverAddrPort != port {
+					// If endpoint is local, but port is different than serverAddrPort, then make it as remote.
+					endpoints[i].IsLocal = false
+				}
 			}
 		}
 
@@ -994,10 +1018,12 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 			// This means it is DistErasure setup.
 		}
 
-		poolEndpoints[i] = endpoints
-
 		for _, endpoint := range endpoints {
-			uniqueArgs.Add(endpoint.Host)
+			if endpoint.Host != "" {
+				uniqueArgs.Add(endpoint.Host)
+			} else {
+				uniqueArgs.Add(net.JoinHostPort("localhost", serverAddrPort))
+			}
 		}
 
 		poolEndpoints[i] = endpoints
@@ -1008,212 +1034,27 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 		updateDomainIPs(uniqueArgs)
 	}
 
+	erasureType := len(uniqueArgs.ToSlice()) == 1
+
 	for _, endpoints := range poolEndpoints {
 		// Return Erasure setup when all endpoints are path style.
 		if endpoints[0].Type() == PathEndpointType {
 			setupType = ErasureSetupType
+			break
 		}
-		if endpoints[0].Type() == URLEndpointType && setupType != DistErasureSetupType {
-			setupType = DistErasureSetupType
+		if endpoints[0].Type() == URLEndpointType {
+			if erasureType {
+				setupType = ErasureSetupType
+			} else {
+				setupType = DistErasureSetupType
+			}
+			break
 		}
 	}
 
 	return poolEndpoints, setupType, nil
 }
 
-// CreateEndpoints - validates and creates new endpoints for given args.
-func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType, error) {
-	var endpoints Endpoints
-	var setupType SetupType
-	var err error
-
-	// Check whether serverAddr is valid for this host.
-	if err = CheckLocalServerAddr(serverAddr); err != nil {
-		return endpoints, setupType, err
-	}
-
-	_, serverAddrPort := mustSplitHostPort(serverAddr)
-
-	// For single arg, return single drive setup.
-	if len(args) == 1 && len(args[0]) == 1 {
-		var endpoint Endpoint
-		endpoint, err = NewEndpoint(args[0][0])
-		if err != nil {
-			return endpoints, setupType, err
-		}
-		if err := endpoint.UpdateIsLocal(); err != nil {
-			return endpoints, setupType, err
-		}
-		if endpoint.Type() != PathEndpointType {
-			return endpoints, setupType, config.ErrInvalidEndpoint(nil).Msg("use path style endpoint for single node setup")
-		}
-
-		endpoint.SetPoolIndex(0)
-		endpoint.SetSetIndex(0)
-		endpoint.SetDiskIndex(0)
-
-		endpoints = append(endpoints, endpoint)
-		setupType = ErasureSDSetupType
-
-		// Check for cross device mounts if any.
-		if err = checkCrossDeviceMounts(endpoints); err != nil {
-			return endpoints, setupType, config.ErrInvalidEndpoint(nil).Msg(err.Error())
-		}
-
-		return endpoints, setupType, nil
-	}
-
-	for setIdx, iargs := range args {
-		// Convert args to endpoints
-		eps, err := NewEndpoints(iargs...)
-		if err != nil {
-			return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
-		}
-
-		// Check for cross device mounts if any.
-		if err = checkCrossDeviceMounts(eps); err != nil {
-			return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
-		}
-
-		for diskIdx := range eps {
-			eps[diskIdx].SetSetIndex(setIdx)
-			eps[diskIdx].SetDiskIndex(diskIdx)
-		}
-
-		endpoints = append(endpoints, eps...)
-	}
-
-	if len(endpoints) == 0 {
-		return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg("invalid number of endpoints")
-	}
-
-	// Return Erasure setup when all endpoints are path style.
-	if endpoints[0].Type() == PathEndpointType {
-		setupType = ErasureSetupType
-		return endpoints, setupType, nil
-	}
-
-	if err = endpoints.UpdateIsLocal(); err != nil {
-		return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
-	}
-
-	// Here all endpoints are URL style.
-	endpointPathSet := set.NewStringSet()
-	localEndpointCount := 0
-	localServerHostSet := set.NewStringSet()
-	localPortSet := set.NewStringSet()
-
-	for _, endpoint := range endpoints {
-		endpointPathSet.Add(endpoint.Path)
-		if endpoint.IsLocal {
-			localServerHostSet.Add(endpoint.Hostname())
-
-			var port string
-			_, port, err = net.SplitHostPort(endpoint.Host)
-			if err != nil {
-				port = serverAddrPort
-			}
-			localPortSet.Add(port)
-
-			localEndpointCount++
-		}
-	}
-
-	orchestrated := IsKubernetes() || IsDocker()
-	reverseProxy := (env.Get("_MINIO_REVERSE_PROXY", "") != "") && ((env.Get("MINIO_CI_CD", "") != "") || (env.Get("CI", "") != ""))
-	// If not orchestrated
-	if !orchestrated &&
-		// and not setup in reverse proxy
-		!reverseProxy {
-		// Check whether same path is not used in endpoints of a host on different port.
-		// Only verify this on baremetal setups, DNS is not available in orchestrated
-		// environments so we can't do much here.
-		pathIPMap := make(map[string]set.StringSet)
-		hostIPCache := make(map[string]set.StringSet)
-		for _, endpoint := range endpoints {
-			host := endpoint.Hostname()
-			hostIPSet, ok := hostIPCache[host]
-			if !ok {
-				hostIPSet, err = getHostIP(host)
-				if err != nil {
-					return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
-				}
-				hostIPCache[host] = hostIPSet
-			}
-			if IPSet, ok := pathIPMap[endpoint.Path]; ok {
-				if !IPSet.Intersection(hostIPSet).IsEmpty() {
-					return endpoints, setupType,
-						config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
-				}
-				pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
-			} else {
-				pathIPMap[endpoint.Path] = hostIPSet
-			}
-		}
-	}
-
-	// Check whether same path is used for more than 1 local endpoints.
-	{
-		localPathSet := set.CreateStringSet()
-		for _, endpoint := range endpoints {
-			if !endpoint.IsLocal {
-				continue
-			}
-			if localPathSet.Contains(endpoint.Path) {
-				return endpoints, setupType,
-					config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' cannot be served by different address on same server", endpoint.Path))
-			}
-			localPathSet.Add(endpoint.Path)
-		}
-	}
-
-	// Add missing port in all endpoints.
-	for i := range endpoints {
-		_, port, err := net.SplitHostPort(endpoints[i].Host)
-		if err != nil {
-			endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort)
-		} else if endpoints[i].IsLocal && serverAddrPort != port {
-			// If endpoint is local, but port is different than serverAddrPort, then make it as remote.
-			endpoints[i].IsLocal = false
-		}
-	}
-
-	// All endpoints are pointing to local host
-	if len(endpoints) == localEndpointCount {
-		// If all endpoints have same port number, Just treat it as local erasure setup
-		// using URL style endpoints.
-		if len(localPortSet) == 1 {
-			if len(localServerHostSet) > 1 {
-				return endpoints, setupType,
-					config.ErrInvalidErasureEndpoints(nil).Msg("all local endpoints should not have different hostnames/ips")
-			}
-			return endpoints, ErasureSetupType, nil
-		}
-
-		// Even though all endpoints are local, but those endpoints use different ports.
-		// This means it is DistErasure setup.
-	}
-
-	uniqueArgs := set.NewStringSet()
-	for _, endpoint := range endpoints {
-		uniqueArgs.Add(endpoint.Host)
-	}
-
-	// Error out if we have less than 2 unique servers.
-	if len(uniqueArgs.ToSlice()) < 2 && setupType == DistErasureSetupType {
-		err := fmt.Errorf("Unsupported number of endpoints (%s), minimum number of servers cannot be less than 2 in distributed setup", endpoints)
-		return endpoints, setupType, err
-	}
-
-	publicIPs := env.Get(config.EnvPublicIPs, "")
-	if len(publicIPs) == 0 {
-		updateDomainIPs(uniqueArgs)
-	}
-
-	setupType = DistErasureSetupType
-	return endpoints, setupType, nil
-}
-
 // GetLocalPeer - returns local peer value, returns globalMinioAddr
 // for FS and Erasure mode. In case of distributed server return
 // the first element from the set of peers which indicate that

cmd/endpoint_test.go

@@ -225,24 +225,24 @@ func TestCreateEndpoints(t *testing.T) {
 
 	testCases := []struct {
 		serverAddr         string
-		args               [][]string
+		args               []string
 		expectedServerAddr string
 		expectedEndpoints  Endpoints
 		expectedSetupType  SetupType
 		expectedErr        error
 	}{
-		{"localhost", [][]string{}, "", Endpoints{}, -1, fmt.Errorf("address localhost: missing port in address")},
+		{"localhost", []string{}, "", Endpoints{}, -1, fmt.Errorf("address localhost: missing port in address")},
 
 		// Erasure Single Drive
-		{"localhost:9000", [][]string{{"http://localhost/d1"}}, "", Endpoints{}, -1, fmt.Errorf("use path style endpoint for FS setup")},
-		{":443", [][]string{{"/d1"}}, ":443", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, ErasureSDSetupType, nil},
-		{"localhost:10000", [][]string{{"/d1"}}, "localhost:10000", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, ErasureSDSetupType, nil},
-		{"localhost:9000", [][]string{{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}}, "", Endpoints{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
+		{"localhost:9000", []string{"http://localhost/d1"}, "", Endpoints{}, -1, fmt.Errorf("use path style endpoint for SD setup")},
+		{":443", []string{"/d1"}, ":443", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, ErasureSDSetupType, nil},
+		{"localhost:10000", []string{"/d1"}, "localhost:10000", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, ErasureSDSetupType, nil},
+		{"localhost:9000", []string{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}, "", Endpoints{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
 
 		// Erasure Setup with PathEndpointType
 		{
 			":1234",
-			[][]string{{"/d1", "/d2", "/d3", "/d4"}},
+			[]string{"/d1", "/d2", "/d3", "/d4"},
 			":1234",
 			Endpoints{
 				Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true},
@@ -253,49 +253,49 @@ func TestCreateEndpoints(t *testing.T) {
 			ErasureSetupType, nil,
 		},
 		// DistErasure Setup with URLEndpointType
-		{":9000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}}, ":9000", Endpoints{
+		{":9000", []string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}, ":9000", Endpoints{
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d1"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d2"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d3"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d4"}, IsLocal: true},
 		}, ErasureSetupType, nil},
 		// DistErasure Setup with URLEndpointType having mixed naming to local host.
-		{"127.0.0.1:10000", [][]string{{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}}, "", Endpoints{}, -1, fmt.Errorf("all local endpoints should not have different hostnames/ips")},
+		{"127.0.0.1:10000", []string{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}, "", Endpoints{}, -1, fmt.Errorf("all local endpoints should not have different hostnames/ips")},
 
-		{":9001", [][]string{{"http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export", "http://" + nonLoopBackIP + ":9001/export", "http://10.0.0.2:9001/export"}}, "", Endpoints{}, -1, fmt.Errorf("path '/export' can not be served by different port on same address")},
+		{":9001", []string{"http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export", "http://" + nonLoopBackIP + ":9001/export", "http://10.0.0.2:9001/export"}, "", Endpoints{}, -1, fmt.Errorf("path '/export' can not be served by different port on same address")},
 
-		{":9000", [][]string{{"http://127.0.0.1:9000/export", "http://" + nonLoopBackIP + ":9000/export", "http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export"}}, "", Endpoints{}, -1, fmt.Errorf("path '/export' cannot be served by different address on same server")},
+		{":9000", []string{"http://127.0.0.1:9000/export", "http://" + nonLoopBackIP + ":9000/export", "http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export"}, "", Endpoints{}, -1, fmt.Errorf("path '/export' cannot be served by different address on same server")},
 
 		// DistErasure type
-		{"127.0.0.1:10000", [][]string{{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", Endpoints{
+		{"127.0.0.1:10000", []string{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", Endpoints{
 			Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0]},
 			Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1]},
 			Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2]},
 			Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3]},
 		}, DistErasureSetupType, nil},
 
-		{"127.0.0.1:10000", [][]string{{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}}, "127.0.0.1:10000", Endpoints{
+		{"127.0.0.1:10000", []string{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", Endpoints{
 			Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0]},
 			Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1]},
 			Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2]},
 			Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3]},
 		}, DistErasureSetupType, nil},
 
-		{":80", [][]string{{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}}, ":80", Endpoints{
+		{":80", []string{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}, ":80", Endpoints{
 			Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0]},
 			Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1]},
 			Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2]},
 			Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3]},
 		}, DistErasureSetupType, nil},
 
-		{":9000", [][]string{{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}}, ":9000", Endpoints{
+		{":9000", []string{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}, ":9000", Endpoints{
 			Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0]},
 			Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1]},
 			Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2]},
 			Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3]},
 		}, DistErasureSetupType, nil},
 
-		{":9000", [][]string{{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}}, ":9000", Endpoints{
+		{":9000", []string{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}, ":9000", Endpoints{
 			Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0]},
 			Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1]},
 			Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2]},
@@ -303,7 +303,7 @@ func TestCreateEndpoints(t *testing.T) {
 		}, DistErasureSetupType, nil},
 
 		// DistErasure Setup using only local host.
-		{":9003", [][]string{{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}}, ":9003", Endpoints{
+		{":9003", []string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}, ":9003", Endpoints{
 			Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0]},
 			Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1]},
 			Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2]},
@@ -311,24 +311,32 @@ func TestCreateEndpoints(t *testing.T) {
 		}, DistErasureSetupType, nil},
 	}
 
-	for _, testCase := range testCases {
+	for i, testCase := range testCases {
+		i := i
 		testCase := testCase
 		t.Run("", func(t *testing.T) {
-			endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
+			var srvCtxt serverCtxt
+			err := mergeDisksLayoutFromArgs(testCase.args, &srvCtxt)
+			if err != nil && testCase.expectedErr == nil {
+				t.Errorf("Test %d: unexpected error: %v", i+1, err)
+			}
+			pools, setupType, err := CreatePoolEndpoints(testCase.serverAddr, srvCtxt.Layout.pools...)
 			if err == nil && testCase.expectedErr != nil {
-				t.Errorf("error: expected = %v, got = <nil>", testCase.expectedErr)
+				t.Errorf("Test %d: expected = %v, got = <nil>", i+1, testCase.expectedErr)
 			}
 			if err == nil {
 				if setupType != testCase.expectedSetupType {
-					t.Errorf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType)
+					t.Errorf("Test %d: setupType: expected = %v, got = %v", i+1, testCase.expectedSetupType, setupType)
 				}
+				endpoints := pools[0]
 				if len(endpoints) != len(testCase.expectedEndpoints) {
-					t.Errorf("endpoints: expected = %d, got = %d", len(testCase.expectedEndpoints),
+					t.Errorf("Test %d: endpoints: expected = %d, got = %d", i+1, len(testCase.expectedEndpoints),
 						len(endpoints))
 				} else {
 					for i, endpoint := range endpoints {
 						if testCase.expectedEndpoints[i].String() != endpoint.String() {
-							t.Errorf("endpoints: expected = %s, got = %s",
+							t.Errorf("Test %d: endpoints: expected = %s, got = %s",
+								i+1,
 								testCase.expectedEndpoints[i],
 								endpoint)
 						}
@@ -336,7 +344,7 @@ func TestCreateEndpoints(t *testing.T) {
 				}
 			}
 			if err != nil && testCase.expectedErr == nil {
-				t.Errorf("error: expected = <nil>, got = %v, testCase: %v", err, testCase)
+				t.Errorf("Test %d: error: expected = <nil>, got = %v, testCase: %v", i+1, err, testCase)
 			}
 		})
 	}

cmd/erasure-healing.go

@@ -68,7 +68,7 @@ func (fi FileInfo) DataShardFixed() bool {
 	return fi.Metadata[reservedMetadataPrefixLowerDataShardFix] == "true"
 }
 
-func (er erasureObjects) listAndHeal(bucket, prefix string, healEntry func(string, metaCacheEntry) error) error {
+func (er erasureObjects) listAndHeal(bucket, prefix string, scanMode madmin.HealScanMode, healEntry func(string, metaCacheEntry, madmin.HealScanMode) error) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -101,7 +101,7 @@ func (er erasureObjects) listAndHeal(bucket, prefix string, healEntry func(strin
 		minDisks:       1,
 		reportNotFound: false,
 		agreed: func(entry metaCacheEntry) {
-			if err := healEntry(bucket, entry); err != nil {
+			if err := healEntry(bucket, entry, scanMode); err != nil {
 				cancel()
 			}
 		},
@@ -113,7 +113,7 @@ func (er erasureObjects) listAndHeal(bucket, prefix string, healEntry func(strin
 				entry, _ = entries.firstFound()
 			}
 
-			if err := healEntry(bucket, *entry); err != nil {
+			if err := healEntry(bucket, *entry, scanMode); err != nil {
 				cancel()
 				return
 			}
@@ -128,159 +128,6 @@ func (er erasureObjects) listAndHeal(bucket, prefix string, healEntry func(strin
 	return nil
 }
 
-// HealBucket heals a bucket if it doesn't exist on one of the disks, additionally
-// also heals the missing entries for bucket metadata files
-// `policy.json, notification.xml, listeners.json`.
-func (er erasureObjects) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (
-	result madmin.HealResultItem, err error,
-) {
-	storageDisks := er.getDisks()
-	storageEndpoints := er.getEndpoints()
-
-	// Heal bucket.
-	return er.healBucket(ctx, storageDisks, storageEndpoints, bucket, opts)
-}
-
-// Heal bucket - create buckets on disks where it does not exist.
-func (er erasureObjects) healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints []Endpoint, bucket string, opts madmin.HealOpts) (res madmin.HealResultItem, err error) {
-	// get write quorum for an object
-	writeQuorum := len(storageDisks) - er.defaultParityCount
-	if writeQuorum == er.defaultParityCount {
-		writeQuorum++
-	}
-
-	if globalTrace.NumSubscribers(madmin.TraceHealing) > 0 {
-		startTime := time.Now()
-		defer func() {
-			healTrace(healingMetricBucket, startTime, bucket, "", &opts, err, &res)
-		}()
-	}
-
-	// Initialize sync waitgroup.
-	g := errgroup.WithNErrs(len(storageDisks))
-
-	// Disk states slices
-	beforeState := make([]string, len(storageDisks))
-	afterState := make([]string, len(storageDisks))
-
-	// Make a volume entry on all underlying storage disks.
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if storageDisks[index] == nil {
-				beforeState[index] = madmin.DriveStateOffline
-				afterState[index] = madmin.DriveStateOffline
-				return errDiskNotFound
-			}
-
-			beforeState[index] = madmin.DriveStateOk
-			afterState[index] = madmin.DriveStateOk
-
-			if bucket == minioReservedBucket {
-				return nil
-			}
-
-			if _, serr := storageDisks[index].StatVol(ctx, bucket); serr != nil {
-				if serr == errDiskNotFound {
-					beforeState[index] = madmin.DriveStateOffline
-					afterState[index] = madmin.DriveStateOffline
-					return serr
-				}
-				if serr != errVolumeNotFound {
-					beforeState[index] = madmin.DriveStateCorrupt
-					afterState[index] = madmin.DriveStateCorrupt
-					return serr
-				}
-
-				beforeState[index] = madmin.DriveStateMissing
-				afterState[index] = madmin.DriveStateMissing
-
-				// mutate only if not a dry-run
-				if opts.DryRun {
-					return nil
-				}
-
-				return serr
-			}
-			return nil
-		}, index)
-	}
-
-	errs := g.Wait()
-
-	// Initialize heal result info
-	res = madmin.HealResultItem{
-		Type:         madmin.HealItemBucket,
-		Bucket:       bucket,
-		DiskCount:    len(storageDisks),
-		ParityBlocks: er.defaultParityCount,
-		DataBlocks:   len(storageDisks) - er.defaultParityCount,
-	}
-
-	for i := range beforeState {
-		res.Before.Drives = append(res.Before.Drives, madmin.HealDriveInfo{
-			UUID:     "",
-			Endpoint: storageEndpoints[i].String(),
-			State:    beforeState[i],
-		})
-	}
-
-	reducedErr := reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, res.DataBlocks)
-	if errors.Is(reducedErr, errVolumeNotFound) && !opts.Recreate {
-		for i := range beforeState {
-			res.After.Drives = append(res.After.Drives, madmin.HealDriveInfo{
-				UUID:     "",
-				Endpoint: storageEndpoints[i].String(),
-				State:    madmin.DriveStateOk,
-			})
-		}
-		return res, nil
-	}
-
-	// Initialize sync waitgroup.
-	g = errgroup.WithNErrs(len(storageDisks))
-
-	// Make a volume entry on all underlying storage disks.
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if beforeState[index] == madmin.DriveStateMissing {
-				makeErr := storageDisks[index].MakeVol(ctx, bucket)
-				if makeErr == nil {
-					afterState[index] = madmin.DriveStateOk
-				}
-				return makeErr
-			}
-			return errs[index]
-		}, index)
-	}
-
-	errs = g.Wait()
-
-	reducedErr = reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, writeQuorum)
-	if reducedErr != nil {
-		// If we have exactly half the drives not available,
-		// we should still allow HealBucket to not return error.
-		// this is necessary for starting the server.
-		readQuorum := res.DataBlocks
-		switch reduceReadQuorumErrs(ctx, errs, nil, readQuorum) {
-		case nil:
-		case errDiskNotFound:
-		default:
-			return res, reducedErr
-		}
-	}
-
-	for i := range afterState {
-		res.After.Drives = append(res.After.Drives, madmin.HealDriveInfo{
-			UUID:     "",
-			Endpoint: storageEndpoints[i].String(),
-			State:    afterState[i],
-		})
-	}
-	return res, nil
-}
-
 // listAllBuckets lists all buckets from all disks. It also
 // returns the occurrence of each buckets in all disks
 func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets map[string]VolInfo, readQuorum int) error {
@@ -370,10 +217,8 @@ func (fi *FileInfo) SetHealing() {
 // Healing returns true if object is being healed (i.e fi is being passed down
 // from healObject)
 func (fi FileInfo) Healing() bool {
-	if _, ok := fi.Metadata[xMinIOHealing]; ok {
-		return true
-	}
-	return false
+	_, ok := fi.Metadata[xMinIOHealing]
+	return ok
 }
 
 // Heals an object by re-writing corrupt/missing erasure blocks.
@@ -489,7 +334,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			erasure.ShardFileSize(latestMeta.Parts[0].ActualSize) < smallFileThreshold)
 	}
 
-	result.ObjectSize, err = latestMeta.GetActualSize()
+	result.ObjectSize, err = latestMeta.ToObjectInfo(bucket, object, true).GetActualSize()
 	if err != nil {
 		return result, err
 	}
@@ -760,7 +605,8 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 
 		// Attempt a rename now from healed data to final location.
 		partsMetadata[i].SetHealing()
-		if _, err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i], bucket, object); err != nil {
+
+		if _, err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i], bucket, object, RenameOptions{}); err != nil {
 			return result, err
 		}
 
@@ -1020,6 +866,22 @@ func isAllNotFound(errs []error) bool {
 	return len(errs) > 0
 }
 
+// isAllBucketsNotFound will return true if all the errors are either errFileNotFound
+// or errFileCorrupt
+// A 0 length slice will always return false.
+func isAllBucketsNotFound(errs []error) bool {
+	if len(errs) == 0 {
+		return false
+	}
+	notFoundCount := 0
+	for _, err := range errs {
+		if err != nil && errors.Is(err, errVolumeNotFound) {
+			notFoundCount++
+		}
+	}
+	return len(errs) == notFoundCount
+}
+
 // ObjectDir is considered dangling/corrupted if any only
 // if total disks - a combination of corrupted and missing
 // files is lesser than N/2+1 number of disks.
@@ -1045,7 +907,7 @@ func isObjectDirDangling(errs []error) (ok bool) {
 	return found < notFound && found > 0
 }
 
-// Object is considered dangling/corrupted if any only
+// Object is considered dangling/corrupted if and only
 // if total disks - a combination of corrupted and missing
 // files is lesser than number of data blocks.
 func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (validMeta FileInfo, ok bool) {

cmd/erasure-healing_test.go

@@ -362,7 +362,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 	// This would create the bucket.
-	_, err = er.HealBucket(ctx, bucket, madmin.HealOpts{
+	_, err = obj.HealBucket(ctx, bucket, madmin.HealOpts{
 		DryRun: false,
 		Remove: false,
 	})
@@ -543,7 +543,7 @@ func TestHealingVersioned(t *testing.T) {
 		t.Fatal(err)
 	}
 	// This would create the bucket.
-	_, err = er.HealBucket(ctx, bucket, madmin.HealOpts{
+	_, err = obj.HealBucket(ctx, bucket, madmin.HealOpts{
 		DryRun: false,
 		Remove: false,
 	})
@@ -647,8 +647,8 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 
 	if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true},
-		func(bucket, object, vid string) error {
-			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{Remove: true})
+		func(bucket, object, vid string, scanMode madmin.HealScanMode) error {
+			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{ScanMode: scanMode, Remove: true})
 			return err
 		}); err != nil {
 		t.Fatal(err)
@@ -694,8 +694,8 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 
 	if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true},
-		func(bucket, object, vid string) error {
-			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{Remove: true})
+		func(bucket, object, vid string, scanMode madmin.HealScanMode) error {
+			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{ScanMode: scanMode, Remove: true})
 			return err
 		}); err != nil {
 		t.Fatal(err)
@@ -743,8 +743,8 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 
 	if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Remove: true},
-		func(bucket, object, vid string) error {
-			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{Remove: true})
+		func(bucket, object, vid string, scanMode madmin.HealScanMode) error {
+			_, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{ScanMode: scanMode, Remove: true})
 			return err
 		}); err != nil {
 		t.Fatal(err)

cmd/erasure-metadata.go

@@ -22,18 +22,15 @@ import (
 	"encoding/hex"
 	"fmt"
 	"sort"
-	"strconv"
 	"strings"
 	"time"
 
 	"github.com/minio/minio/internal/amztime"
 	"github.com/minio/minio/internal/bucket/replication"
-	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/sync/errgroup"
-	"github.com/minio/sio"
 )
 
 // Object was stored with additional erasure codes due to degraded system at upload time
@@ -88,52 +85,6 @@ func (fi FileInfo) IsValid() bool {
 		correctIndexes)
 }
 
-func (fi FileInfo) checkMultipart() (int64, bool) {
-	if len(fi.Parts) == 0 {
-		return 0, false
-	}
-	if !crypto.IsMultiPart(fi.Metadata) {
-		return 0, false
-	}
-	var size int64
-	for _, part := range fi.Parts {
-		psize, err := sio.DecryptedSize(uint64(part.Size))
-		if err != nil {
-			return 0, false
-		}
-		size += int64(psize)
-	}
-
-	return size, len(extractETag(fi.Metadata)) != 32
-}
-
-// GetActualSize - returns the actual size of the stored object
-func (fi FileInfo) GetActualSize() (int64, error) {
-	if _, ok := fi.Metadata[ReservedMetadataPrefix+"compression"]; ok {
-		sizeStr, ok := fi.Metadata[ReservedMetadataPrefix+"actual-size"]
-		if !ok {
-			return -1, errInvalidDecompressedSize
-		}
-		size, err := strconv.ParseInt(sizeStr, 10, 64)
-		if err != nil {
-			return -1, errInvalidDecompressedSize
-		}
-		return size, nil
-	}
-	if _, ok := crypto.IsEncrypted(fi.Metadata); ok {
-		size, ok := fi.checkMultipart()
-		if !ok {
-			size, err := sio.DecryptedSize(uint64(fi.Size))
-			if err != nil {
-				err = errObjectTampered // assign correct error type
-			}
-			return int64(size), err
-		}
-		return size, nil
-	}
-	return fi.Size, nil
-}
-
 // ToObjectInfo - Converts metadata to object info.
 func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInfo {
 	object = decodeDirObject(object)
@@ -166,7 +117,6 @@ func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInf
 			objInfo.Expires = t.UTC()
 		}
 	}
-	objInfo.backendType = BackendErasure
 
 	// Extract etag from metadata.
 	objInfo.ETag = extractETag(fi.Metadata)

cmd/erasure-multipart.go

@@ -662,12 +662,12 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 			// Account for padding and forced compression overhead and encryption.
 			buffer = make([]byte, data.ActualSize()+256+32+32, data.ActualSize()*2+512)
 		} else {
-			buffer = er.bp.Get()
-			defer er.bp.Put(buffer)
+			buffer = globalBytePoolCap.Get()
+			defer globalBytePoolCap.Put(buffer)
 		}
 	case size >= fi.Erasure.BlockSize:
-		buffer = er.bp.Get()
-		defer er.bp.Put(buffer)
+		buffer = globalBytePoolCap.Get()
+		defer globalBytePoolCap.Put(buffer)
 	case size < fi.Erasure.BlockSize:
 		// No need to allocate fully fi.Erasure.BlockSize buffer if the incoming data is smaller.
 		buffer = make([]byte, size, 2*size+int64(fi.Erasure.ParityBlocks+fi.Erasure.DataBlocks-1))
@@ -688,10 +688,10 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	if data.Size() > bigFileThreshold {
 		// Add input readahead.
 		// We use 2 buffers, so we always have a full buffer of input.
-		bufA := er.bp.Get()
-		bufB := er.bp.Get()
-		defer er.bp.Put(bufA)
-		defer er.bp.Put(bufB)
+		bufA := globalBytePoolCap.Get()
+		bufB := globalBytePoolCap.Get()
+		defer globalBytePoolCap.Put(bufA)
+		defer globalBytePoolCap.Put(bufB)
 		ra, err := readahead.NewReaderBuffer(data, [][]byte{bufA[:fi.Erasure.BlockSize], bufB[:fi.Erasure.BlockSize]})
 		if err == nil {
 			toEncode = ra
@@ -1290,13 +1290,15 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		})
 	}
 
-	// Check if there is any offline disk and add it to the MRF list
-	for _, disk := range onlineDisks {
-		if disk != nil && disk.IsOnline() {
-			continue
+	if !opts.Speedtest && !versionsDisparity {
+		// Check if there is any offline disk and add it to the MRF list
+		for _, disk := range onlineDisks {
+			if disk != nil && disk.IsOnline() {
+				continue
+			}
+			er.addPartial(bucket, object, fi.VersionID)
+			break
 		}
-		er.addPartial(bucket, object, fi.VersionID)
-		break
 	}
 
 	for i := 0; i < len(onlineDisks); i++ {

cmd/erasure-object.go

@@ -99,7 +99,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
 		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(srcBucket, minioMetaBucket) {
-			_, derr := er.deleteIfDangling(ctx, srcBucket, srcObject, metaArr, errs, nil, srcOpts)
+			_, derr := er.deleteIfDangling(context.Background(), srcBucket, srcObject, metaArr, errs, nil, srcOpts)
 			if derr != nil {
 				err = derr
 			}
@@ -413,6 +413,7 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 							queued:    time.Now(),
 							setIndex:  er.setIndex,
 							poolIndex: er.poolIndex,
+							scanMode:  scan,
 						})
 					})
 					// Healing is triggered and we have written
@@ -475,40 +476,50 @@ func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID
 	auditLogInternal(ctx, opts)
 }
 
-func joinErrors(errs ...error) error {
-	s := make([]string, 0, len(errs))
-	nonNilErrs := make([]any, 0, len(errs))
-	for _, err := range errs {
-		if err == nil {
-			continue
+func joinErrs(errs []error) []string {
+	s := make([]string, len(errs))
+	for i := range s {
+		if errs[i] == nil {
+			s[i] = "<nil>"
+		} else {
+			s[i] = errs[i].Error()
 		}
-		s = append(s, "[%w]")
-		nonNilErrs = append(nonNilErrs, err)
 	}
-	// If all the errors were nil, return nil.
-	if len(nonNilErrs) == 0 {
-		return nil
-	}
-	allErrs := strings.Join(s, "\n")
-	return fmt.Errorf(allErrs, nonNilErrs...)
+	return s
 }
 
 func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrs []error, opts ObjectOptions) (FileInfo, error) {
-	_, file, line, cok := runtime.Caller(1)
 	var err error
 	m, ok := isObjectDangling(metaArr, errs, dataErrs)
 	if ok {
 		tags := make(map[string]interface{}, 4)
 		tags["set"] = er.setIndex
 		tags["pool"] = er.poolIndex
-		tags["merrs"] = joinErrors(errs...)     // errors.Join(errs...)
-		tags["derrs"] = joinErrors(dataErrs...) // errors.Join(dataErrs...)
+		tags["merrs"] = joinErrs(errs)
+		tags["derrs"] = joinErrs(dataErrs)
 		if m.IsValid() {
 			tags["size"] = m.Size
 			tags["mtime"] = m.ModTime.Format(http.TimeFormat)
+			tags["data"] = m.Erasure.DataBlocks
 			tags["parity"] = m.Erasure.ParityBlocks
+		} else {
+			tags["invalid-meta"] = true
+			tags["data"] = er.setDriveCount - er.defaultParityCount
+			tags["parity"] = er.defaultParityCount
 		}
 
+		// count the number of offline disks
+		offline := 0
+		for i := 0; i < max(len(errs), len(dataErrs)); i++ {
+			if i < len(errs) && errs[i] == errDiskNotFound || i < len(dataErrs) && dataErrs[i] == errDiskNotFound {
+				offline++
+			}
+		}
+		if offline > 0 {
+			tags["offline"] = offline
+		}
+
+		_, file, line, cok := runtime.Caller(1)
 		if cok {
 			tags["caller"] = fmt.Sprintf("%s:%d", file, line)
 		}
@@ -535,11 +546,26 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 				if disks[index] == nil {
 					return errDiskNotFound
 				}
-				return disks[index].DeleteVersion(ctx, bucket, object, fi, false)
+				return disks[index].DeleteVersion(ctx, bucket, object, fi, false, DeleteOptions{})
 			}, index)
 		}
 
-		g.Wait()
+		rmDisks := make(map[string]string, len(disks))
+		for index, err := range g.Wait() {
+			var errStr, diskName string
+			if err != nil {
+				errStr = err.Error()
+			} else {
+				errStr = "<nil>"
+			}
+			if disks[index] != nil {
+				diskName = disks[index].String()
+			} else {
+				diskName = fmt.Sprintf("disk-%d", index)
+			}
+			rmDisks[diskName] = errStr
+		}
+		tags["cleanupResult"] = rmDisks
 	}
 	return m, err
 }
@@ -910,7 +936,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 
 	if err != nil {
 		if shouldCheckForDangling(err, errs, bucket) {
-			_, derr := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
+			_, derr := er.deleteIfDangling(context.Background(), bucket, object, metaArr, errs, nil, opts)
 			if derr != nil {
 				err = derr
 			}
@@ -1021,7 +1047,7 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str
 			if !fi.IsValid() {
 				return errFileCorrupt
 			}
-			sign, err := disks[index].RenameData(ctx, srcBucket, srcEntry, fi, dstBucket, dstEntry)
+			sign, err := disks[index].RenameData(ctx, srcBucket, srcEntry, fi, dstBucket, dstEntry, RenameOptions{})
 			if err != nil {
 				return err
 			}
@@ -1036,6 +1062,23 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str
 	var versionsDisparity bool
 
 	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
+	if err != nil {
+		dg := errgroup.WithNErrs(len(disks))
+		for index, nerr := range errs {
+			if nerr != nil {
+				continue
+			}
+			index := index
+			// When we are going to return error, attempt to delete success
+			// on some of the drives, if we cannot we do not have to notify
+			// caller this dangling object will be now scheduled to be removed
+			// via active healing.
+			dg.Go(func() error {
+				return disks[index].DeleteVersion(context.Background(), dstBucket, dstEntry, metadata[index], false, DeleteOptions{UndoWrite: true})
+			}, index)
+		}
+		dg.Wait()
+	}
 	if err == nil {
 		versions := reduceCommonVersions(diskVersions, writeQuorum)
 		for index, dversions := range diskVersions {
@@ -1109,8 +1152,8 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 	case size == 0:
 		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
 	case size >= fi.Erasure.BlockSize:
-		buffer = er.bp.Get()
-		defer er.bp.Put(buffer)
+		buffer = globalBytePoolCap.Get()
+		defer globalBytePoolCap.Put(buffer)
 	case size < fi.Erasure.BlockSize:
 		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
 		buffer = make([]byte, size, 2*size+int64(fi.Erasure.ParityBlocks+fi.Erasure.DataBlocks-1))
@@ -1200,7 +1243,7 @@ func (er erasureObjects) PutObject(ctx context.Context, bucket string, object st
 }
 
 // Heal up to two versions of one object when there is disparity between disks
-func healObjectVersionsDisparity(bucket string, entry metaCacheEntry) error {
+func healObjectVersionsDisparity(bucket string, entry metaCacheEntry, scanMode madmin.HealScanMode) error {
 	if entry.isDir() {
 		return nil
 	}
@@ -1224,13 +1267,13 @@ func healObjectVersionsDisparity(bucket string, entry metaCacheEntry) error {
 
 	fivs, err := entry.fileInfoVersions(bucket)
 	if err != nil {
-		go healObject(bucket, entry.name, "", madmin.HealDeepScan)
+		healObject(bucket, entry.name, "", madmin.HealDeepScan)
 		return err
 	}
 
 	if len(fivs.Versions) <= 2 {
 		for _, version := range fivs.Versions {
-			go healObject(bucket, entry.name, version.VersionID, madmin.HealNormalScan)
+			healObject(bucket, entry.name, version.VersionID, scanMode)
 		}
 	}
 
@@ -1360,8 +1403,8 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	case size == 0:
 		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
 	case size >= fi.Erasure.BlockSize || size == -1:
-		buffer = er.bp.Get()
-		defer er.bp.Put(buffer)
+		buffer = globalBytePoolCap.Get()
+		defer globalBytePoolCap.Put(buffer)
 	case size < fi.Erasure.BlockSize:
 		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
 		buffer = make([]byte, size, 2*size+int64(fi.Erasure.ParityBlocks+fi.Erasure.DataBlocks-1))
@@ -1420,10 +1463,10 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	toEncode := io.Reader(data)
 	if data.Size() > bigFileThreshold {
 		// We use 2 buffers, so we always have a full buffer of input.
-		bufA := er.bp.Get()
-		bufB := er.bp.Get()
-		defer er.bp.Put(bufA)
-		defer er.bp.Put(bufB)
+		bufA := globalBytePoolCap.Get()
+		bufB := globalBytePoolCap.Get()
+		defer globalBytePoolCap.Put(bufA)
+		defer globalBytePoolCap.Put(bufB)
 		ra, err := readahead.NewReaderBuffer(data, [][]byte{bufA[:fi.Erasure.BlockSize], bufB[:fi.Erasure.BlockSize]})
 		if err == nil {
 			toEncode = ra
@@ -1516,7 +1559,6 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		if errors.Is(err, errFileNotFound) {
 			return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object)
 		}
-		logger.LogOnceIf(ctx, err, "erasure-object-rename-"+bucket+"-"+object)
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
@@ -1531,18 +1573,21 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	// For speedtest objects do not attempt to heal them.
 	if !opts.Speedtest {
-		// Whether a disk was initially or becomes offline
-		// during this upload, send it to the MRF list.
-		for i := 0; i < len(onlineDisks); i++ {
-			if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
-				continue
+		// When there is versions disparity we are healing
+		// the content implicitly for all versions, we can
+		// avoid triggering another MRF heal for offline drives.
+		if !versionsDisparity {
+			// Whether a disk was initially or becomes offline
+			// during this upload, send it to the MRF list.
+			for i := 0; i < len(onlineDisks); i++ {
+				if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
+					continue
+				}
+
+				er.addPartial(bucket, object, fi.VersionID)
+				break
 			}
-
-			er.addPartial(bucket, object, fi.VersionID)
-			break
-		}
-
-		if versionsDisparity {
+		} else {
 			globalMRFState.addPartialOp(partialOperation{
 				bucket:      bucket,
 				object:      object,
@@ -1579,7 +1624,7 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			return disks[index].DeleteVersion(ctx, bucket, object, fi, forceDelMarker)
+			return disks[index].DeleteVersion(ctx, bucket, object, fi, forceDelMarker, DeleteOptions{})
 		}, index)
 	}
 	// return errors if any during deletion
@@ -1692,7 +1737,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 				}
 				return
 			}
-			errs := disk.DeleteVersions(ctx, bucket, dedupVersions)
+			errs := disk.DeleteVersions(ctx, bucket, dedupVersions, DeleteOptions{})
 			for i, err := range errs {
 				if err == nil {
 					continue
@@ -2028,7 +2073,7 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
 		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
-			_, derr := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
+			_, derr := er.deleteIfDangling(context.Background(), bucket, object, metaArr, errs, nil, opts)
 			if derr != nil {
 				err = derr
 			}
@@ -2101,7 +2146,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 	if err != nil {
 		if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
-			_, derr := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
+			_, derr := er.deleteIfDangling(context.Background(), bucket, object, metaArr, errs, nil, opts)
 			if derr != nil {
 				err = derr
 			}

cmd/erasure-server-pool-decom.go

@@ -716,7 +716,7 @@ func (set *erasureObjects) listObjectsToDecommission(ctx context.Context, bi dec
 		path:           bi.Prefix,
 		recursive:      true,
 		forwardTo:      "",
-		minDisks:       len(disks) / 2,
+		minDisks:       listQuorum,
 		reportNotFound: false,
 		agreed:         fn,
 		partial: func(entries metaCacheEntries, _ []error) {

cmd/erasure-server-pool-rebalance.go

@@ -129,7 +129,7 @@ func (z *erasureServerPools) initRebalanceMeta(ctx context.Context, buckets []st
 	}
 
 	// Fetch disk capacity and available space.
-	si := z.StorageInfo(ctx)
+	si := z.StorageInfo(ctx, true)
 	diskStats := make([]struct {
 		AvailableSpace uint64
 		TotalSpace     uint64
@@ -624,10 +624,12 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 		go func() {
 			defer wg.Done()
 
+			listQuorum := (len(disks) + 1) / 2
+
 			// How to resolve partial results.
 			resolver := metadataResolutionParams{
-				dirQuorum: len(disks) / 2, // make sure to capture all quorum ratios
-				objQuorum: len(disks) / 2, // make sure to capture all quorum ratios
+				dirQuorum: listQuorum, // make sure to capture all quorum ratios
+				objQuorum: listQuorum, // make sure to capture all quorum ratios
 				bucket:    bucket,
 			}
 			err := listPathRaw(ctx, listPathRawOptions{
@@ -635,7 +637,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 				bucket:         bucket,
 				recursive:      true,
 				forwardTo:      "",
-				minDisks:       len(disks) / 2, // to capture all quorum ratios
+				minDisks:       listQuorum,
 				reportNotFound: false,
 				agreed: func(entry metaCacheEntry) {
 					workers <- struct{}{}

cmd/erasure-server-pool.go

@@ -38,6 +38,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/bpool"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/sync/errgroup"
@@ -82,6 +83,21 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		}
 	)
 
+	// Maximum number of reusable buffers per node at any given point in time.
+	n := 1024 // single node single/multiple drives set this to 1024 entries
+
+	if globalIsDistErasure {
+		n = 2048
+	}
+
+	// Initialize byte pool once for all sets, bpool size is set to
+	// setCount * setDriveCount with each memory upto blockSizeV2.
+	globalBytePoolCap = bpool.NewBytePoolCap(n, blockSizeV2, blockSizeV2*2)
+
+	if globalServerCtxt.PreAllocate {
+		globalBytePoolCap.Populate()
+	}
+
 	var localDrives []StorageAPI
 	local := endpointServerPools.FirstLocal()
 	for i, ep := range endpointServerPools {
@@ -107,12 +123,6 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 			return nil, err
 		}
 
-		for _, storageDisk := range storageDisks[i] {
-			if storageDisk != nil && storageDisk.IsLocal() {
-				localDrives = append(localDrives, storageDisk)
-			}
-		}
-
 		if deploymentID == "" {
 			// all pools should have same deployment ID
 			deploymentID = formats[i].ID
@@ -139,6 +149,18 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		if distributionAlgo != "" && z.distributionAlgo == "" {
 			z.distributionAlgo = distributionAlgo
 		}
+
+		for _, storageDisk := range storageDisks[i] {
+			if storageDisk != nil && storageDisk.IsLocal() {
+				localDrives = append(localDrives, storageDisk)
+			}
+		}
+	}
+
+	if !globalIsDistErasure {
+		globalLocalDrivesMu.Lock()
+		globalLocalDrives = localDrives
+		globalLocalDrivesMu.Unlock()
 	}
 
 	z.decommissionCancelers = make([]context.CancelFunc, len(z.serverPools))
@@ -167,10 +189,6 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		break
 	}
 
-	globalLocalDrivesMu.Lock()
-	globalLocalDrives = localDrives
-	defer globalLocalDrivesMu.Unlock()
-
 	return z, nil
 }
 
@@ -595,7 +613,7 @@ func (z *erasureServerPools) BackendInfo() (b madmin.BackendInfo) {
 	return
 }
 
-func (z *erasureServerPools) LocalStorageInfo(ctx context.Context) StorageInfo {
+func (z *erasureServerPools) LocalStorageInfo(ctx context.Context, metrics bool) StorageInfo {
 	var storageInfo StorageInfo
 
 	storageInfos := make([]StorageInfo, len(z.serverPools))
@@ -603,7 +621,7 @@ func (z *erasureServerPools) LocalStorageInfo(ctx context.Context) StorageInfo {
 	for index := range z.serverPools {
 		index := index
 		g.Go(func() error {
-			storageInfos[index] = z.serverPools[index].LocalStorageInfo(ctx)
+			storageInfos[index] = z.serverPools[index].LocalStorageInfo(ctx, metrics)
 			return nil
 		}, index)
 	}
@@ -619,8 +637,8 @@ func (z *erasureServerPools) LocalStorageInfo(ctx context.Context) StorageInfo {
 	return storageInfo
 }
 
-func (z *erasureServerPools) StorageInfo(ctx context.Context) StorageInfo {
-	return globalNotificationSys.StorageInfo(z)
+func (z *erasureServerPools) StorageInfo(ctx context.Context, metrics bool) StorageInfo {
+	return globalNotificationSys.StorageInfo(z, metrics)
 }
 
 func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataUsageInfo, wantCycle uint32, healScanMode madmin.HealScanMode) error {
@@ -1923,8 +1941,53 @@ func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts
 	hopts.Recreate = false
 	defer z.HealObject(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, bucket, bucketMetadataFile), "", hopts)
 
+	type DiskStat struct {
+		VolInfos []VolInfo
+		Errs     []error
+	}
+
 	for _, pool := range z.serverPools {
-		result, err := pool.HealBucket(ctx, bucket, opts)
+		// map of node wise disk stats
+		diskStats := make(map[string]DiskStat)
+		for _, set := range pool.sets {
+			for _, disk := range set.getDisks() {
+				if disk == OfflineDisk {
+					continue
+				}
+				vi, err := disk.StatVol(ctx, bucket)
+				hostName := disk.Hostname()
+				if disk.IsLocal() {
+					hostName = "local"
+				}
+				ds, ok := diskStats[hostName]
+				if !ok {
+					newds := DiskStat{
+						VolInfos: []VolInfo{vi},
+						Errs:     []error{err},
+					}
+					diskStats[hostName] = newds
+				} else {
+					ds.VolInfos = append(ds.VolInfos, vi)
+					ds.Errs = append(ds.Errs, err)
+					diskStats[hostName] = ds
+				}
+			}
+		}
+		nodeCount := len(diskStats)
+		bktNotFoundCount := 0
+		for _, ds := range diskStats {
+			if isAllBucketsNotFound(ds.Errs) {
+				bktNotFoundCount++
+			}
+		}
+		// if the bucket if not found on more than hslf the no of nodes, its dangling
+		if bktNotFoundCount > nodeCount/2 {
+			opts.Remove = true
+		} else {
+			opts.Recreate = true
+		}
+
+		result, err := z.s3Peer.HealBucket(ctx, bucket, opts)
 		if err != nil {
 			if _, ok := err.(BucketNotFound); ok {
 				continue
@@ -1967,7 +2030,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 				go func() {
 					defer wg.Done()
 
-					disks, _ := set.getOnlineDisksWithHealing(true)
+					disks, infos, _ := set.getOnlineDisksWithHealingAndInfo(true)
 					if len(disks) == 0 {
 						cancel()
 						return
@@ -1984,7 +2047,17 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 
 					askDisks := getListQuorum(opts.AskDisks, set.setDriveCount)
 					if askDisks == -1 {
-						askDisks = getListQuorum("strict", set.setDriveCount)
+						newDisks := getQuorumDisks(disks, infos, (len(disks)+1)/2)
+						if newDisks != nil {
+							// If we found disks signature in quorum, we proceed to list
+							// from a single drive, shuffling of the drives is subsequently.
+							disks = newDisks
+							askDisks = 1
+						} else {
+							// If we did not find suitable disks, perform strict quorum listing
+							// as no disk agrees on quorum anymore.
+							askDisks = getListQuorum("strict", set.setDriveCount)
+						}
 					}
 
 					// Special case: ask all disks if the drive count is 4
@@ -2108,10 +2181,10 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 }
 
 // HealObjectFn closure function heals the object.
-type HealObjectFn func(bucket, object, versionID string) error
+type HealObjectFn func(bucket, object, versionID string, scanMode madmin.HealScanMode) error
 
 func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObjectFn HealObjectFn) error {
-	healEntry := func(bucket string, entry metaCacheEntry) error {
+	healEntry := func(bucket string, entry metaCacheEntry, scanMode madmin.HealScanMode) error {
 		if entry.isDir() {
 			return nil
 		}
@@ -2134,7 +2207,7 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 		}
 		fivs, err := entry.fileInfoVersions(bucket)
 		if err != nil {
-			return healObjectFn(bucket, entry.name, "")
+			return healObjectFn(bucket, entry.name, "", scanMode)
 		}
 		if opts.Remove && !opts.DryRun {
 			err := z.CheckAbandonedParts(ctx, bucket, entry.name, opts)
@@ -2143,7 +2216,7 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 			}
 		}
 		for _, version := range fivs.Versions {
-			err := healObjectFn(bucket, version.Name, version.VersionID)
+			err := healObjectFn(bucket, version.Name, version.VersionID, scanMode)
 			if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 				return err
 			}
@@ -2167,7 +2240,7 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 			go func(idx int, set *erasureObjects) {
 				defer wg.Done()
 
-				errs[idx] = set.listAndHeal(bucket, prefix, healEntry)
+				errs[idx] = set.listAndHeal(bucket, prefix, opts.ScanMode, healEntry)
 			}(idx, set)
 		}
 		wg.Wait()
@@ -2267,6 +2340,8 @@ type HealthResult struct {
 		Maintenance   bool
 		PoolID, SetID int
 		HealthyDrives int
+		HealingDrives int
+		ReadQuorum    int
 		WriteQuorum   int
 	}
 	WriteQuorum   int
@@ -2313,55 +2388,51 @@ func (z *erasureServerPools) ReadHealth(ctx context.Context) bool {
 // can be used to query scenarios if health may be lost
 // if this node is taken down by an external orchestrator.
 func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) HealthResult {
-	erasureSetUpCount := make([][]int, len(z.serverPools))
+	reqInfo := (&logger.ReqInfo{}).AppendTags("maintenance", strconv.FormatBool(opts.Maintenance))
+
+	type setInfo struct {
+		online  int
+		healing int
+	}
+
+	var drivesHealing int
+
+	erasureSetUpCount := make([][]setInfo, len(z.serverPools))
 	for i := range z.serverPools {
-		erasureSetUpCount[i] = make([]int, len(z.serverPools[i].sets))
+		erasureSetUpCount[i] = make([]setInfo, len(z.serverPools[i].sets))
 	}
 
-	diskIDs := globalNotificationSys.GetLocalDiskIDs(ctx)
-	if !opts.Maintenance {
-		diskIDs = append(diskIDs, getLocalDiskIDs(z))
-	}
+	storageInfo := z.StorageInfo(ctx, false)
 
-	for _, localDiskIDs := range diskIDs {
-		for _, id := range localDiskIDs {
-			poolIdx, setIdx, _, err := z.getPoolAndSet(id)
-			if err != nil {
-				logger.LogIf(ctx, err)
-				continue
+	for _, disk := range storageInfo.Disks {
+		if disk.PoolIndex > -1 && disk.SetIndex > -1 {
+			if disk.State == madmin.DriveStateOk {
+				si := erasureSetUpCount[disk.PoolIndex][disk.SetIndex]
+				si.online++
+				if disk.Healing {
+					si.healing++
+					drivesHealing++
+				}
+				erasureSetUpCount[disk.PoolIndex][disk.SetIndex] = si
 			}
-			erasureSetUpCount[poolIdx][setIdx]++
 		}
 	}
 
-	reqInfo := (&logger.ReqInfo{}).AppendTags("maintenance", strconv.FormatBool(opts.Maintenance))
-
 	b := z.BackendInfo()
+	poolReadQuorums := make([]int, len(b.StandardSCData))
 	poolWriteQuorums := make([]int, len(b.StandardSCData))
 	for i, data := range b.StandardSCData {
+		poolReadQuorums[i] = data
 		poolWriteQuorums[i] = data
 		if data == b.StandardSCParity {
 			poolWriteQuorums[i] = data + 1
 		}
 	}
 
-	var aggHealStateResult madmin.BgHealState
 	// Check if disks are healing on in-case of VMware vsphere deployments.
 	if opts.Maintenance && opts.DeploymentType == vmware {
-		// check if local disks are being healed, if they are being healed
-		// we need to tell healthy status as 'false' so that this server
-		// is not taken down for maintenance
-		var err error
-		aggHealStateResult, err = getAggregatedBackgroundHealState(ctx, nil)
-		if err != nil {
-			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Unable to verify global heal status: %w", err))
-			return HealthResult{
-				Healthy: false,
-			}
-		}
-
-		if len(aggHealStateResult.HealDisks) > 0 {
-			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Total drives to be healed %d", len(aggHealStateResult.HealDisks)))
+		if drivesHealing > 0 {
+			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Total drives to be healed %d", drivesHealing))
 		}
 	}
 
@@ -2389,18 +2460,21 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 	for poolIdx := range erasureSetUpCount {
 		for setIdx := range erasureSetUpCount[poolIdx] {
 			result.ESHealth = append(result.ESHealth, struct {
-				Maintenance                bool
-				PoolID, SetID              int
-				HealthyDrives, WriteQuorum int
+				Maintenance                  bool
+				PoolID, SetID                int
+				HealthyDrives, HealingDrives int
+				ReadQuorum, WriteQuorum      int
 			}{
 				Maintenance:   opts.Maintenance,
 				SetID:         setIdx,
 				PoolID:        poolIdx,
-				HealthyDrives: erasureSetUpCount[poolIdx][setIdx],
+				HealthyDrives: erasureSetUpCount[poolIdx][setIdx].online,
+				HealingDrives: erasureSetUpCount[poolIdx][setIdx].healing,
+				ReadQuorum:    poolReadQuorums[poolIdx],
 				WriteQuorum:   poolWriteQuorums[poolIdx],
 			})
 
-			if erasureSetUpCount[poolIdx][setIdx] < poolWriteQuorums[poolIdx] {
+			if erasureSetUpCount[poolIdx][setIdx].online < poolWriteQuorums[poolIdx] {
 				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
 					fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
 						poolIdx, setIdx, poolWriteQuorums[poolIdx]))
@@ -2410,8 +2484,8 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 	}
 
 	if opts.Maintenance {
-		result.Healthy = result.Healthy && len(aggHealStateResult.HealDisks) == 0
-		result.HealingDrives = len(aggHealStateResult.HealDisks)
+		result.Healthy = result.Healthy && drivesHealing == 0
+		result.HealingDrives = drivesHealing
 	}
 
 	return result

cmd/erasure-sets.go

@@ -36,7 +36,6 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/internal/bpool"
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
@@ -129,9 +128,11 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 		if errors.Is(err, errUnformattedDisk) {
 			info, derr := disk.DiskInfo(context.TODO(), false)
 			if derr != nil && info.RootDisk {
+				disk.Close()
 				return nil, nil, fmt.Errorf("Drive: %s is a root drive", disk)
 			}
 		}
+		disk.Close()
 		return nil, nil, fmt.Errorf("Drive: %s returned %w", disk, err) // make sure to '%w' to wrap the error
 	}
 
@@ -262,6 +263,12 @@ func (s *erasureSets) connectDisks() {
 			disk.SetDiskLoc(s.poolIndex, setIndex, diskIndex)
 			s.erasureDisks[setIndex][diskIndex] = disk
 			s.erasureDisksMu.Unlock()
+
+			if disk.IsLocal() && globalIsDistErasure {
+				globalLocalDrivesMu.Lock()
+				globalLocalSetDrives[s.poolIndex][setIndex][diskIndex] = disk
+				globalLocalDrivesMu.Unlock()
+			}
 		}(endpoint)
 	}
 
@@ -365,21 +372,6 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 
 	mutex := newNSLock(globalIsDistErasure)
 
-	// Number of buffers, max 2GB
-	n := (2 * humanize.GiByte) / (blockSizeV2 * 2)
-
-	// Initialize byte pool once for all sets, bpool size is set to
-	// setCount * setDriveCount with each memory upto blockSizeV2.
-	bp := bpool.NewBytePoolCap(n, blockSizeV2, blockSizeV2*2)
-
-	// Initialize byte pool for all sets, bpool size is set to
-	// setCount * setDriveCount with each memory upto blockSizeV1
-	//
-	// Number of buffers, max 10GiB
-	m := (10 * humanize.GiByte) / (blockSizeV1 * 2)
-
-	bpOld := bpool.NewBytePoolCap(m, blockSizeV1, blockSizeV1*2)
-
 	for i := 0; i < setCount; i++ {
 		s.erasureDisks[i] = make([]StorageAPI, setDriveCount)
 	}
@@ -415,6 +407,19 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 				if disk == nil {
 					continue
 				}
+
+				if disk.IsLocal() && globalIsDistErasure {
+					globalLocalDrivesMu.RLock()
+					ldisk := globalLocalSetDrives[poolIdx][i][j]
+					if ldisk == nil {
+						globalLocalDrivesMu.RUnlock()
+						continue
+					}
+					disk.Close()
+					disk = ldisk
+					globalLocalDrivesMu.RUnlock()
+				}
+
 				innerWg.Add(1)
 				go func(disk StorageAPI, i, j int) {
 					defer innerWg.Done()
@@ -455,8 +460,6 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 				getLockers:         s.GetLockers(i),
 				getEndpoints:       s.GetEndpoints(i),
 				nsMutex:            mutex,
-				bp:                 bp,
-				bpOld:              bpOld,
 			}
 		}(i)
 	}
@@ -611,7 +614,7 @@ func (s *erasureSets) StorageInfo(ctx context.Context) StorageInfo {
 }
 
 // StorageInfo - combines output of StorageInfo across all erasure coded object sets.
-func (s *erasureSets) LocalStorageInfo(ctx context.Context) StorageInfo {
+func (s *erasureSets) LocalStorageInfo(ctx context.Context, metrics bool) StorageInfo {
 	var storageInfo StorageInfo
 
 	storageInfos := make([]StorageInfo, len(s.sets))
@@ -620,7 +623,7 @@ func (s *erasureSets) LocalStorageInfo(ctx context.Context) StorageInfo {
 	for index := range s.sets {
 		index := index
 		g.Go(func() error {
-			storageInfos[index] = s.sets[index].LocalStorageInfo(ctx)
+			storageInfos[index] = s.sets[index].LocalStorageInfo(ctx, metrics)
 			return nil
 		}, index)
 	}
@@ -1053,6 +1056,7 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 			// We should not heal on root disk. i.e in a situation where the minio-administrator has unmounted a
 			// defective drive we should not heal a path on the root disk.
 			logger.LogIf(GlobalContext, fmt.Errorf("Drive `%s` is part of root drive, will not be used", storageDisks[i]))
+			storageDisks[i].Close()
 			storageDisks[i] = nil
 		}
 	}
@@ -1062,7 +1066,7 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealResultItem, err error) {
 	storageDisks, _ := initStorageDisksWithErrors(s.endpoints.Endpoints, storageOpts{
 		cleanUp:     true,
-		healthCheck: true,
+		healthCheck: false,
 	})
 
 	defer func(storageDisks []StorageAPI) {
@@ -1110,7 +1114,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 	formatOpID := mustGetUUID()
 
 	// Initialize a new set of set formats which will be written to disk.
-	newFormatSets := newHealFormatSets(refFormat, s.setCount, s.setDriveCount, formats, sErrs)
+	newFormatSets, currentDisksInfo := newHealFormatSets(refFormat, s.setCount, s.setDriveCount, formats, sErrs)
 
 	if !dryRun {
 		tmpNewFormats := make([]*formatErasureV3, s.setCount*s.setDriveCount)
@@ -1132,6 +1136,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 			}
 			if err := saveFormatErasure(storageDisks[index], format, formatOpID); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Drive %s failed to write updated 'format.json': %v", storageDisks[index], err))
+				storageDisks[index].Close()
 				tmpNewFormats[index] = nil // this disk failed to write new format
 			}
 		}
@@ -1153,9 +1158,37 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 				s.erasureDisks[m][n].Close()
 			}
 
-			if storageDisks[index] != nil {
-				storageDisks[index].SetDiskLoc(s.poolIndex, m, n)
-				s.erasureDisks[m][n] = storageDisks[index]
+			if disk := storageDisks[index]; disk != nil {
+				if disk.IsLocal() {
+					disk.SetDiskLoc(s.poolIndex, m, n)
+
+					xldisk, ok := disk.(*xlStorageDiskIDCheck)
+					if ok {
+						if driveQuorum {
+							commonWrites, commonDeletes := calcCommonWritesDeletes(currentDisksInfo[m], (s.setDriveCount+1)/2)
+							xldisk.totalWrites.Store(commonWrites)
+							xldisk.totalDeletes.Store(commonDeletes)
+							xldisk.storage.setWriteAttribute(commonWrites)
+							xldisk.storage.setDeleteAttribute(commonDeletes)
+						}
+						go xldisk.monitorDiskWritable(xldisk.diskCtx)
+					}
+				} else {
+					disk.Close() // Close the remote storage client, re-initialize with healthchecks.
+					disk, err = newStorageRESTClient(disk.Endpoint(), true, globalGrid.Load())
+					if err != nil {
+						continue
+					}
+					disk.SetDiskLoc(s.poolIndex, m, n)
+				}
+
+				s.erasureDisks[m][n] = disk
+
+				if disk.IsLocal() && globalIsDistErasure {
+					globalLocalDrivesMu.Lock()
+					globalLocalSetDrives[s.poolIndex][m][n] = disk
+					globalLocalDrivesMu.Unlock()
+				}
 			}
 		}
 
@@ -1168,28 +1201,6 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 	return res, nil
 }
 
-// HealBucket - heals inconsistent buckets and bucket metadata on all sets.
-func (s *erasureSets) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) {
-	// Initialize heal result info
-	result = madmin.HealResultItem{
-		Type:      madmin.HealItemBucket,
-		Bucket:    bucket,
-		DiskCount: s.setCount * s.setDriveCount,
-		SetCount:  s.setCount,
-	}
-
-	for _, set := range s.sets {
-		healResult, err := set.HealBucket(ctx, bucket, opts)
-		if err != nil {
-			return result, toObjectErr(err, bucket)
-		}
-		result.Before.Drives = append(result.Before.Drives, healResult.Before.Drives...)
-		result.After.Drives = append(result.After.Drives, healResult.After.Drives...)
-	}
-
-	return result, nil
-}
-
 // HealObject - heals inconsistent object on a hashedSet based on object name.
 func (s *erasureSets) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
 	return s.getHashedSet(object).HealObject(ctx, bucket, object, versionID, opts)

cmd/erasure-utils.go

@@ -72,15 +72,15 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 			// Decrement offset.
 			offset -= int64(len(block))
 			continue
-		} else {
-			// Skip until offset.
-			block = block[offset:]
-
-			// Reset the offset for next iteration to read everything
-			// from subsequent blocks.
-			offset = 0
 		}
 
+		// Skip until offset.
+		block = block[offset:]
+
+		// Reset the offset for next iteration to read everything
+		// from subsequent blocks.
+		offset = 0
+
 		// We have written all the blocks, write the last remaining block.
 		if write < int64(len(block)) {
 			n, err := dst.Write(block[:write])

cmd/erasure.go

@@ -29,7 +29,6 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go/v3"
-	"github.com/minio/minio/internal/bpool"
 	"github.com/minio/minio/internal/dsync"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
@@ -65,13 +64,6 @@ type erasureObjects struct {
 
 	// Locker mutex map.
 	nsMutex *nsLockMap
-
-	// Byte pools used for temporary i/o buffers.
-	bp *bpool.BytePoolCap
-
-	// Byte pools used for temporary i/o buffers,
-	// legacy objects.
-	bpOld *bpool.BytePoolCap
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
@@ -99,7 +91,7 @@ func diskErrToDriveState(err error) (state string) {
 	switch {
 	case errors.Is(err, errDiskNotFound) || errors.Is(err, context.DeadlineExceeded):
 		state = madmin.DriveStateOffline
-	case errors.Is(err, errCorruptedFormat):
+	case errors.Is(err, errCorruptedFormat) || errors.Is(err, errCorruptedBackend):
 		state = madmin.DriveStateCorrupt
 	case errors.Is(err, errUnformattedDisk):
 		state = madmin.DriveStateUnformatted
@@ -168,7 +160,7 @@ func getOnlineOfflineDisksStats(disksInfo []madmin.Disk) (onlineDisks, offlineDi
 }
 
 // getDisksInfo - fetch disks info across all other storage API.
-func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.Disk) {
+func getDisksInfo(disks []StorageAPI, endpoints []Endpoint, metrics bool) (disksInfo []madmin.Disk) {
 	disksInfo = make([]madmin.Disk, len(disks))
 
 	g := errgroup.WithNErrs(len(disks))
@@ -186,7 +178,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 				disksInfo[index] = di
 				return nil
 			}
-			info, err := disks[index].DiskInfo(context.TODO(), true)
+			info, err := disks[index].DiskInfo(context.TODO(), metrics)
 			di.DrivePath = info.MountPath
 			di.TotalSpace = info.Total
 			di.UsedSpace = info.Used
@@ -233,8 +225,8 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 }
 
 // Get an aggregated storage info across all disks.
-func getStorageInfo(disks []StorageAPI, endpoints []Endpoint) StorageInfo {
-	disksInfo := getDisksInfo(disks, endpoints)
+func getStorageInfo(disks []StorageAPI, endpoints []Endpoint, metrics bool) StorageInfo {
+	disksInfo := getDisksInfo(disks, endpoints, metrics)
 
 	// Sort so that the first element is the smallest.
 	sort.Slice(disksInfo, func(i, j int) bool {
@@ -253,11 +245,11 @@ func getStorageInfo(disks []StorageAPI, endpoints []Endpoint) StorageInfo {
 func (er erasureObjects) StorageInfo(ctx context.Context) StorageInfo {
 	disks := er.getDisks()
 	endpoints := er.getEndpoints()
-	return getStorageInfo(disks, endpoints)
+	return getStorageInfo(disks, endpoints, true)
 }
 
 // LocalStorageInfo - returns underlying local storage statistics.
-func (er erasureObjects) LocalStorageInfo(ctx context.Context) StorageInfo {
+func (er erasureObjects) LocalStorageInfo(ctx context.Context, metrics bool) StorageInfo {
 	disks := er.getDisks()
 	endpoints := er.getEndpoints()
 
@@ -271,15 +263,15 @@ func (er erasureObjects) LocalStorageInfo(ctx context.Context) StorageInfo {
 		}
 	}
 
-	return getStorageInfo(localDisks, localEndpoints)
+	return getStorageInfo(localDisks, localEndpoints, metrics)
 }
 
-// getOnlineDisksWithHealing - returns online disks and overall healing status.
+// getOnlineDisksWithHealingAndInfo - returns online disks and overall healing status.
 // Disks are randomly ordered, but in the following groups:
 // - Non-scanning disks
 // - Non-healing disks
 // - Healing disks (if inclHealing is true)
-func (er erasureObjects) getOnlineDisksWithHealing(inclHealing bool) (newDisks []StorageAPI, healing bool) {
+func (er erasureObjects) getOnlineDisksWithHealingAndInfo(inclHealing bool) (newDisks []StorageAPI, newInfos []DiskInfo, healing bool) {
 	var wg sync.WaitGroup
 	disks := er.getDisks()
 	infos := make([]DiskInfo, len(disks))
@@ -292,32 +284,24 @@ func (er erasureObjects) getOnlineDisksWithHealing(inclHealing bool) (newDisks [
 
 			disk := disks[i]
 			if disk == nil {
-				infos[i].Error = "offline drive"
+				infos[i].Error = errDiskNotFound.Error()
 				return
 			}
 
 			di, err := disk.DiskInfo(context.Background(), false)
+			infos[i] = di
 			if err != nil {
 				// - Do not consume disks which are not reachable
 				//   unformatted or simply not accessible for some reason.
-				//
-				//
-				// - Future: skip busy disks
-				if err != nil {
-					infos[i].Error = err.Error()
-				}
-				return
+				infos[i].Error = err.Error()
 			}
-			if !inclHealing && di.Healing {
-				return
-			}
-
-			infos[i] = di
 		}()
 	}
 	wg.Wait()
 
 	var scanningDisks, healingDisks []StorageAPI
+	var scanningInfos, healingInfos []DiskInfo
+
 	for i, info := range infos {
 		// Check if one of the drives in the set is being healed.
 		// this information is used by scanner to skip healing
@@ -329,23 +313,34 @@ func (er erasureObjects) getOnlineDisksWithHealing(inclHealing bool) (newDisks [
 			healing = true
 			if inclHealing {
 				healingDisks = append(healingDisks, disks[i])
+				healingInfos = append(healingInfos, infos[i])
 			}
 			continue
 		}
 
 		if !info.Scanning {
 			newDisks = append(newDisks, disks[i])
+			newInfos = append(newInfos, infos[i])
 		} else {
 			scanningDisks = append(scanningDisks, disks[i])
+			scanningInfos = append(scanningInfos, infos[i])
 		}
 	}
 
 	// Prefer non-scanning disks over disks which are currently being scanned.
 	newDisks = append(newDisks, scanningDisks...)
+	newInfos = append(newInfos, scanningInfos...)
+
 	/// Then add healing disks.
 	newDisks = append(newDisks, healingDisks...)
+	newInfos = append(newInfos, healingInfos...)
 
-	return newDisks, healing
+	return newDisks, newInfos, healing
+}
+
+func (er erasureObjects) getOnlineDisksWithHealing(inclHealing bool) (newDisks []StorageAPI, healing bool) {
+	newDisks, _, healing = er.getOnlineDisksWithHealingAndInfo(inclHealing)
+	return
 }
 
 // Clean-up previously deleted objects. from .minio.sys/tmp/.trash/
@@ -527,7 +522,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				// Calc usage
 				before := cache.Info.LastUpdate
 				var err error
-				cache, err = disk.NSScanner(ctx, cache, updates, healScanMode)
+				cache, err = disk.NSScanner(ctx, cache, updates, healScanMode, nil)
 				if err != nil {
 					if !cache.Info.LastUpdate.IsZero() && cache.Info.LastUpdate.After(before) {
 						logger.LogIf(ctx, cache.save(ctx, er, cacheName))

cmd/event-notification.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"fmt"
 	"net/url"
+	"runtime"
 	"strings"
 	"sync"
 
@@ -36,16 +37,14 @@ import (
 type EventNotifier struct {
 	sync.RWMutex
 	targetList     *event.TargetList
-	targetResCh    chan event.TargetIDResult
 	bucketRulesMap map[string]event.RulesMap
 }
 
 // NewEventNotifier - creates new event notification object.
-func NewEventNotifier() *EventNotifier {
+func NewEventNotifier(ctx context.Context) *EventNotifier {
 	// targetList/bucketRulesMap/bucketRemoteTargetRulesMap are populated by NotificationSys.InitBucketTargets()
 	return &EventNotifier{
-		targetList:     event.NewTargetList(),
-		targetResCh:    make(chan event.TargetIDResult),
+		targetList:     event.NewTargetList(ctx),
 		bucketRulesMap: make(map[string]event.RulesMap),
 	}
 }
@@ -90,6 +89,11 @@ func (evnot *EventNotifier) set(bucket BucketInfo, meta BucketMetadata) {
 	evnot.AddRulesMap(bucket.Name, config.ToRulesMap())
 }
 
+// Targets returns all the registered targets
+func (evnot *EventNotifier) Targets() []event.Target {
+	return evnot.targetList.Targets()
+}
+
 // InitBucketTargets - initializes event notification system from notification.xml of all buckets.
 func (evnot *EventNotifier) InitBucketTargets(ctx context.Context, objAPI ObjectLayer) error {
 	if objAPI == nil {
@@ -99,17 +103,7 @@ func (evnot *EventNotifier) InitBucketTargets(ctx context.Context, objAPI Object
 	if err := evnot.targetList.Add(globalNotifyTargetList.Targets()...); err != nil {
 		return err
 	}
-
-	go func() {
-		for res := range evnot.targetResCh {
-			if res.Err != nil {
-				reqInfo := &logger.ReqInfo{}
-				reqInfo.AppendTags("targetID", res.ID.String())
-				logger.LogOnceIf(logger.SetReqInfo(GlobalContext, reqInfo), res.Err, res.ID.String())
-			}
-		}
-	}()
-
+	evnot.targetList = evnot.targetList.Init(runtime.GOMAXPROCS(0)) // TODO: make this configurable (y4m4)
 	return nil
 }
 
@@ -159,7 +153,7 @@ func (evnot *EventNotifier) Send(args eventArgs) {
 	}
 
 	// If MINIO_API_SYNC_EVENTS is set, send events synchronously.
-	evnot.targetList.Send(args.ToEvent(true), targetIDSet, evnot.targetResCh, globalAPIConfig.isSyncEventsEnabled())
+	evnot.targetList.Send(args.ToEvent(true), targetIDSet, globalAPIConfig.isSyncEventsEnabled())
 }
 
 type eventArgs struct {

cmd/format-erasure.go

@@ -123,6 +123,7 @@ type formatErasureV3 struct {
 		// to pick the right set index for an object.
 		DistributionAlgo string `json:"distributionAlgo"`
 	} `json:"xl"`
+	Info DiskInfo `json:"-"`
 }
 
 func (f *formatErasureV3) Drives() (drives int) {
@@ -328,6 +329,11 @@ func loadFormatErasureAll(storageDisks []StorageAPI, heal bool) ([]*formatErasur
 			if err != nil {
 				return err
 			}
+			info, err := storageDisks[index].DiskInfo(context.Background(), false)
+			if err != nil {
+				return err
+			}
+			format.Info = info
 			formats[index] = format
 			if !heal {
 				// If no healing required, make the disks valid and
@@ -824,11 +830,15 @@ func makeFormatErasureMetaVolumes(disk StorageAPI) error {
 }
 
 // Initialize a new set of set formats which will be written to all disks.
-func newHealFormatSets(refFormat *formatErasureV3, setCount, setDriveCount int, formats []*formatErasureV3, errs []error) [][]*formatErasureV3 {
+func newHealFormatSets(refFormat *formatErasureV3, setCount, setDriveCount int, formats []*formatErasureV3, errs []error) ([][]*formatErasureV3, [][]DiskInfo) {
 	newFormats := make([][]*formatErasureV3, setCount)
 	for i := range refFormat.Erasure.Sets {
 		newFormats[i] = make([]*formatErasureV3, setDriveCount)
 	}
+	currentDisksInfo := make([][]DiskInfo, setCount)
+	for i := range refFormat.Erasure.Sets {
+		currentDisksInfo[i] = make([]DiskInfo, setDriveCount)
+	}
 	for i := range refFormat.Erasure.Sets {
 		for j := range refFormat.Erasure.Sets[i] {
 			if errors.Is(errs[i*setDriveCount+j], errUnformattedDisk) {
@@ -841,7 +851,10 @@ func newHealFormatSets(refFormat *formatErasureV3, setCount, setDriveCount int,
 				newFormats[i][j].Erasure.Version = refFormat.Erasure.Version
 				newFormats[i][j].Erasure.DistributionAlgo = refFormat.Erasure.DistributionAlgo
 			}
+			if format := formats[i*setDriveCount+j]; format != nil && (errs[i*setDriveCount+j] == nil) {
+				currentDisksInfo[i][j] = format.Info
+			}
 		}
 	}
-	return newFormats
+	return newFormats, currentDisksInfo
 }

cmd/format-erasure_test.go

@@ -513,7 +513,7 @@ func TestNewFormatSets(t *testing.T) {
 	// 16th disk is unformatted.
 	errs[15] = errUnformattedDisk
 
-	newFormats := newHealFormatSets(quorumFormat, setCount, setDriveCount, formats, errs)
+	newFormats, _ := newHealFormatSets(quorumFormat, setCount, setDriveCount, formats, errs)
 	if newFormats == nil {
 		t.Fatal("Unexpected failure")
 	}

cmd/ftp-server-driver.go

@@ -248,12 +248,19 @@ func (driver *ftpDriver) CheckPasswd(c *ftp.Context, username, password string)
 	defer stopFn(err)
 
 	if globalIAMSys.LDAPConfig.Enabled() {
-		ldapUserDN, groupDistNames, err := globalIAMSys.LDAPConfig.Bind(username, password)
-		if err != nil {
+		sa, _, err := globalIAMSys.getServiceAccount(context.Background(), username)
+		if err != nil && !errors.Is(err, errNoSuchServiceAccount) {
 			return false, err
 		}
-		ldapPolicies, _ := globalIAMSys.PolicyDBGet(ldapUserDN, groupDistNames...)
-		return len(ldapPolicies) > 0, nil
+		if errors.Is(err, errNoSuchServiceAccount) {
+			ldapUserDN, groupDistNames, err := globalIAMSys.LDAPConfig.Bind(username, password)
+			if err != nil {
+				return false, err
+			}
+			ldapPolicies, _ := globalIAMSys.PolicyDBGet(ldapUserDN, groupDistNames...)
+			return len(ldapPolicies) > 0, nil
+		}
+		return subtle.ConstantTimeCompare([]byte(sa.Credentials.SecretKey), []byte(password)) == 1, nil
 	}
 
 	ui, ok := globalIAMSys.GetUser(context.Background(), username)
@@ -269,58 +276,70 @@ func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error)
 		return nil, errNoSuchUser
 	}
 	if !ok && globalIAMSys.LDAPConfig.Enabled() {
-		targetUser, targetGroups, err := globalIAMSys.LDAPConfig.LookupUserDN(ctx.Sess.LoginUser())
-		if err != nil {
-			return nil, err
-		}
-		ldapPolicies, _ := globalIAMSys.PolicyDBGet(targetUser, targetGroups...)
-		if len(ldapPolicies) == 0 {
-			return nil, errAuthentication
-		}
-		expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration("")
-		if err != nil {
-			return nil, err
-		}
-		claims := make(map[string]interface{})
-		claims[expClaim] = UTCNow().Add(expiryDur).Unix()
-		claims[ldapUser] = targetUser
-		claims[ldapUserN] = ctx.Sess.LoginUser()
-
-		cred, err := auth.GetNewCredentialsWithMetadata(claims, globalActiveCred.SecretKey)
-		if err != nil {
+		sa, _, err := globalIAMSys.getServiceAccount(context.Background(), ctx.Sess.LoginUser())
+		if err != nil && !errors.Is(err, errNoSuchServiceAccount) {
 			return nil, err
 		}
 
-		// Set the parent of the temporary access key, this is useful
-		// in obtaining service accounts by this cred.
-		cred.ParentUser = targetUser
+		var mcreds *credentials.Credentials
+		if errors.Is(err, errNoSuchServiceAccount) {
+			targetUser, targetGroups, err := globalIAMSys.LDAPConfig.LookupUserDN(ctx.Sess.LoginUser())
+			if err != nil {
+				return nil, err
+			}
+			ldapPolicies, _ := globalIAMSys.PolicyDBGet(targetUser, targetGroups...)
+			if len(ldapPolicies) == 0 {
+				return nil, errAuthentication
+			}
+			expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration("")
+			if err != nil {
+				return nil, err
+			}
+			claims := make(map[string]interface{})
+			claims[expClaim] = UTCNow().Add(expiryDur).Unix()
+			claims[ldapUser] = targetUser
+			claims[ldapUserN] = ctx.Sess.LoginUser()
 
-		// Set this value to LDAP groups, LDAP user can be part
-		// of large number of groups
-		cred.Groups = targetGroups
+			cred, err := auth.GetNewCredentialsWithMetadata(claims, globalActiveCred.SecretKey)
+			if err != nil {
+				return nil, err
+			}
 
-		// Set the newly generated credentials, policyName is empty on purpose
-		// LDAP policies are applied automatically using their ldapUser, ldapGroups
-		// mapping.
-		updatedAt, err := globalIAMSys.SetTempUser(context.Background(), cred.AccessKey, cred, "")
-		if err != nil {
-			return nil, err
+			// Set the parent of the temporary access key, this is useful
+			// in obtaining service accounts by this cred.
+			cred.ParentUser = targetUser
+
+			// Set this value to LDAP groups, LDAP user can be part
+			// of large number of groups
+			cred.Groups = targetGroups
+
+			// Set the newly generated credentials, policyName is empty on purpose
+			// LDAP policies are applied automatically using their ldapUser, ldapGroups
+			// mapping.
+			updatedAt, err := globalIAMSys.SetTempUser(context.Background(), cred.AccessKey, cred, "")
+			if err != nil {
+				return nil, err
+			}
+
+			// Call hook for site replication.
+			logger.LogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{
+				Type: madmin.SRIAMItemSTSAcc,
+				STSCredential: &madmin.SRSTSCredential{
+					AccessKey:    cred.AccessKey,
+					SecretKey:    cred.SecretKey,
+					SessionToken: cred.SessionToken,
+					ParentUser:   cred.ParentUser,
+				},
+				UpdatedAt: updatedAt,
+			}))
+
+			mcreds = credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken)
+		} else {
+			mcreds = credentials.NewStaticV4(sa.Credentials.AccessKey, sa.Credentials.SecretKey, "")
 		}
 
-		// Call hook for site replication.
-		logger.LogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{
-			Type: madmin.SRIAMItemSTSAcc,
-			STSCredential: &madmin.SRSTSCredential{
-				AccessKey:    cred.AccessKey,
-				SecretKey:    cred.SecretKey,
-				SessionToken: cred.SessionToken,
-				ParentUser:   cred.ParentUser,
-			},
-			UpdatedAt: updatedAt,
-		}))
-
 		return minio.New(driver.endpoint, &minio.Options{
-			Creds:     credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
+			Creds:     mcreds,
 			Secure:    globalIsTLS,
 			Transport: globalRemoteFTPClientTransport,
 		})

cmd/ftp-server.go

@@ -23,7 +23,6 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/cli"
 	"github.com/minio/minio/internal/logger"
 	ftp "goftp.io/server/v2"
 )
@@ -69,9 +68,7 @@ func (log *minioLogger) PrintResponse(sessionID string, code int, message string
 	}
 }
 
-func startFTPServer(c *cli.Context) {
-	args := c.StringSlice("ftp")
-
+func startFTPServer(args []string) {
 	var (
 		port          int
 		publicIP      string

cmd/generic-handlers.go

@@ -153,7 +153,7 @@ func setBrowserRedirectMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		read := r.Method == http.MethodGet || r.Method == http.MethodHead
 		// Re-direction is handled specifically for browser requests.
-		if guessIsBrowserReq(r) && read && globalBrowserRedirect {
+		if !guessIsHealthCheckReq(r) && guessIsBrowserReq(r) && read && globalBrowserRedirect {
 			// Fetch the redirect location if any.
 			if u := getRedirectLocation(r); u != nil {
 				// Employ a temporary re-direct.

cmd/global-heal.go

@@ -97,7 +97,7 @@ func getLocalBackgroundHealStatus(ctx context.Context, o ObjectLayer) (madmin.Bg
 		return status, true
 	}
 
-	si := o.LocalStorageInfo(ctx)
+	si := o.LocalStorageInfo(ctx, true)
 
 	indexed := make(map[string][]madmin.Disk)
 	for _, disk := range si.Disks {
@@ -143,10 +143,13 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 	healBuckets := make([]string, len(buckets))
 	copy(healBuckets, buckets)
 
-	// Heal all buckets first in this erasure set - this is useful
-	// for new objects upload in different buckets to be successful
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
 	for _, bucket := range healBuckets {
-		_, err := er.HealBucket(ctx, bucket, madmin.HealOpts{ScanMode: scanMode})
+		_, err := objAPI.HealBucket(ctx, bucket, madmin.HealOpts{ScanMode: scanMode})
 		if err != nil {
 			// Log bucket healing error if any, we shall retry again.
 			logger.LogIf(ctx, err)
@@ -195,10 +198,9 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		}
 		tracker.setObject("")
 		tracker.setBucket(bucket)
-
 		// Heal current bucket again in case if it is failed
 		// in the beginning of erasure set healing
-		if _, err := er.HealBucket(ctx, bucket, madmin.HealOpts{
+		if _, err := objAPI.HealBucket(ctx, bucket, madmin.HealOpts{
 			ScanMode: scanMode,
 		}); err != nil {
 			logger.LogIf(ctx, err)

cmd/globals.go

@@ -25,13 +25,15 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/console/restapi"
+	consoleapi "github.com/minio/console/api"
 	"github.com/minio/dnscache"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio/internal/bpool"
 	"github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/config/browser"
 	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/kms"
 	"go.uber.org/atomic"
@@ -127,13 +129,47 @@ const (
 	tlsClientSessionCacheSize = 100
 )
 
-var globalCLIContext = struct {
-	JSON, Quiet    bool
-	Anonymous      bool
-	StrictS3Compat bool
-}{}
+type poolDisksLayout struct {
+	cmdline string
+	layout  [][]string
+}
+
+type disksLayout struct {
+	legacy bool
+	pools  []poolDisksLayout
+}
+
+type serverCtxt struct {
+	JSON, Quiet               bool
+	Anonymous                 bool
+	StrictS3Compat            bool
+	PreAllocate               bool
+	Addr, ConsoleAddr         string
+	ConfigDir, CertsDir       string
+	configDirSet, certsDirSet bool
+	Interface                 string
+
+	RootUser, RootPwd string
+
+	FTP  []string
+	SFTP []string
+
+	UserTimeout       time.Duration
+	ConnReadDeadline  time.Duration
+	ConnWriteDeadline time.Duration
+
+	ShutdownTimeout   time.Duration
+	IdleTimeout       time.Duration
+	ReadHeaderTimeout time.Duration
+
+	// The layout of disks as interpreted
+	Layout disksLayout
+}
 
 var (
+	// Global user opts context
+	globalServerCtxt serverCtxt
+
 	// Indicates if the running minio server is distributed setup.
 	globalIsDistErasure = false
 
@@ -156,6 +192,9 @@ var (
 	// Disable redirect, default is enabled.
 	globalBrowserRedirect bool
 
+	// globalBrowserConfig Browser user configurable settings
+	globalBrowserConfig browser.Config
+
 	// This flag is set to 'true' when MINIO_UPDATE env is set to 'off'. Default is false.
 	globalInplaceUpdateDisabled = false
 
@@ -192,6 +231,7 @@ var (
 	globalBucketMonitor     *bandwidth.Monitor
 	globalPolicySys         *PolicySys
 	globalIAMSys            *IAMSys
+	globalBytePoolCap       *bpool.BytePoolCap
 
 	globalLifecycleSys       *LifecycleSys
 	globalBucketSSEConfigSys *BucketSSEConfigSys
@@ -359,7 +399,7 @@ var (
 
 	globalTierJournal *TierJournal
 
-	globalConsoleSrv *restapi.Server
+	globalConsoleSrv *consoleapi.Server
 
 	// handles service freeze or un-freeze S3 API calls.
 	globalServiceFreeze atomic.Value

cmd/handler-api.go

@@ -42,9 +42,10 @@ type apiConfig struct {
 	listQuorum       string
 	corsAllowOrigins []string
 	// total drives per erasure set across pools.
-	totalDriveCount     int
-	replicationPriority string
-	transitionWorkers   int
+	totalDriveCount       int
+	replicationPriority   string
+	replicationMaxWorkers int
+	transitionWorkers     int
 
 	staleUploadsExpiry          time.Duration
 	staleUploadsCleanupInterval time.Duration
@@ -152,10 +153,11 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	}
 	t.listQuorum = listQuorum
 	if globalReplicationPool != nil &&
-		cfg.ReplicationPriority != t.replicationPriority {
-		globalReplicationPool.ResizeWorkerPriority(cfg.ReplicationPriority)
+		(cfg.ReplicationPriority != t.replicationPriority || cfg.ReplicationMaxWorkers != t.replicationMaxWorkers) {
+		globalReplicationPool.ResizeWorkerPriority(cfg.ReplicationPriority, cfg.ReplicationMaxWorkers)
 	}
 	t.replicationPriority = cfg.ReplicationPriority
+	t.replicationMaxWorkers = cfg.ReplicationMaxWorkers
 	if globalTransitionState != nil && cfg.TransitionWorkers != t.transitionWorkers {
 		globalTransitionState.UpdateWorkers(cfg.TransitionWorkers)
 	}
@@ -334,15 +336,21 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 	}
 }
 
-func (t *apiConfig) getReplicationPriority() string {
+func (t *apiConfig) getReplicationOpts() replicationPoolOpts {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 
 	if t.replicationPriority == "" {
-		return "auto"
+		return replicationPoolOpts{
+			Priority:   "auto",
+			MaxWorkers: WorkerMaxLimit,
+		}
 	}
 
-	return t.replicationPriority
+	return replicationPoolOpts{
+		Priority:   t.replicationPriority,
+		MaxWorkers: t.replicationMaxWorkers,
+	}
 }
 
 func (t *apiConfig) getTransitionWorkers() int {

cmd/iam-store.go

@@ -2319,6 +2319,27 @@ func (store *IAMStoreSys) ListServiceAccounts(ctx context.Context, accessKey str
 	return serviceAccounts, nil
 }
 
+// ListSTSAccounts - lists only STS accounts from the cache.
+func (store *IAMStoreSys) ListSTSAccounts(ctx context.Context, accessKey string) ([]auth.Credentials, error) {
+	cache := store.rlock()
+	defer store.runlock()
+
+	var stsAccounts []auth.Credentials
+	for _, u := range cache.iamSTSAccountsMap {
+		v := u.Credentials
+		if accessKey != "" && v.ParentUser == accessKey {
+			if v.IsTemp() {
+				// Hide secret key & session key here
+				v.SecretKey = ""
+				v.SessionToken = ""
+				stsAccounts = append(stsAccounts, v)
+			}
+		}
+	}
+
+	return stsAccounts, nil
+}
+
 // AddUser - adds/updates long term user account to storage.
 func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq madmin.AddOrUpdateUserReq) (updatedAt time.Time, err error) {
 	cache := store.lock()

cmd/iam.go

@@ -1041,7 +1041,7 @@ func (sys *IAMSys) UpdateServiceAccount(ctx context.Context, accessKey string, o
 	return updatedAt, nil
 }
 
-// ListServiceAccounts - lists all services accounts associated to a specific user
+// ListServiceAccounts - lists all service accounts associated to a specific user
 func (sys *IAMSys) ListServiceAccounts(ctx context.Context, accessKey string) ([]auth.Credentials, error) {
 	if !sys.Initialized() {
 		return nil, errServerNotInitialized
@@ -1055,7 +1055,7 @@ func (sys *IAMSys) ListServiceAccounts(ctx context.Context, accessKey string) ([
 	}
 }
 
-// ListTempAccounts - lists all services accounts associated to a specific user
+// ListTempAccounts - lists all temporary service accounts associated to a specific user
 func (sys *IAMSys) ListTempAccounts(ctx context.Context, accessKey string) ([]UserIdentity, error) {
 	if !sys.Initialized() {
 		return nil, errServerNotInitialized
@@ -1069,6 +1069,20 @@ func (sys *IAMSys) ListTempAccounts(ctx context.Context, accessKey string) ([]Us
 	}
 }
 
+// ListSTSAccounts - lists all STS accounts associated to a specific user
+func (sys *IAMSys) ListSTSAccounts(ctx context.Context, accessKey string) ([]auth.Credentials, error) {
+	if !sys.Initialized() {
+		return nil, errServerNotInitialized
+	}
+
+	select {
+	case <-sys.configLoaded:
+		return sys.store.ListSTSAccounts(ctx, accessKey)
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
 // GetServiceAccount - wrapper method to get information about a service account
 func (sys *IAMSys) GetServiceAccount(ctx context.Context, accessKey string) (auth.Credentials, *policy.Policy, error) {
 	sa, embeddedPolicy, err := sys.getServiceAccount(ctx, accessKey)
@@ -1344,9 +1358,15 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 	// DN to ldap username mapping for each LDAP user
 	parentUserToLDAPUsernameMap := make(map[string]string)
 	for _, cred := range allCreds {
+		// Expired credentials don't need parent user updates.
+		if cred.IsExpired() {
+			continue
+		}
+
 		if !sys.LDAPConfig.IsLDAPUserDN(cred.ParentUser) {
 			continue
 		}
+
 		// Check if this is the first time we are
 		// encountering this LDAP user.
 		if _, ok := parentUserToCredsMap[cred.ParentUser]; !ok {
@@ -1411,6 +1431,11 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 				continue
 			}
 
+			// Expired credentials don't need group membership updates.
+			if cred.IsExpired() {
+				continue
+			}
+
 			cred.Groups = currGroups
 			if err := sys.store.UpdateUserIdentity(ctx, cred); err != nil {
 				// Log and continue error - perhaps it'll work the next time.

cmd/license-update.go

@@ -89,7 +89,7 @@ func performLicenseUpdate(ctx context.Context, objectAPI ObjectLayer) {
 		return
 	}
 
-	r := gjson.Parse(resp).Get("license")
+	r := gjson.Parse(resp).Get("license_v2")
 	if r.Index == 0 {
 		logger.LogIf(ctx, fmt.Errorf("license not found in response from %s", url))
 		return

cmd/metacache-set.go

@@ -590,25 +590,115 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 func getListQuorum(quorum string, driveCount int) int {
 	switch quorum {
 	case "disk":
-		// smallest possible value, generally meant for testing.
 		return 1
 	case "reduced":
 		return 2
 	case "optimal":
 		return (driveCount + 1) / 2
+	case "auto":
+		return -1
 	}
 	// defaults to 'strict'
 	return driveCount
 }
 
+func calcCommonWritesDeletes(infos []DiskInfo, readQuorum int) (commonWrite, commonDelete uint64) {
+	deletes := make([]uint64, len(infos))
+	writes := make([]uint64, len(infos))
+	for index, di := range infos {
+		deletes[index] = di.Metrics.TotalDeletes
+		writes[index] = di.Metrics.TotalWrites
+	}
+
+	filter := func(list []uint64) (commonCount uint64) {
+		max := 0
+		signatureMap := map[uint64]int{}
+		for _, v := range list {
+			signatureMap[v]++
+		}
+		for ops, count := range signatureMap {
+			if max < count && commonCount < ops {
+				max = count
+				commonCount = ops
+			}
+		}
+		if max < readQuorum {
+			return 0
+		}
+		return commonCount
+	}
+
+	commonWrite = filter(writes)
+	commonDelete = filter(deletes)
+	return
+}
+
+func calcCommonCounter(infos []DiskInfo, readQuorum int) (commonCount uint64) {
+	filter := func() (commonCount uint64) {
+		max := 0
+		signatureMap := map[uint64]int{}
+		for _, info := range infos {
+			if info.Error != "" {
+				continue
+			}
+			mutations := info.Metrics.TotalDeletes + info.Metrics.TotalWrites
+			signatureMap[mutations]++
+		}
+		for ops, count := range signatureMap {
+			if max < count && commonCount < ops {
+				max = count
+				commonCount = ops
+			}
+		}
+		if max < readQuorum {
+			return 0
+		}
+		return commonCount
+	}
+
+	return filter()
+}
+
+func getQuorumDiskInfos(disks []StorageAPI, infos []DiskInfo, readQuorum int) (newDisks []StorageAPI, newInfos []DiskInfo) {
+	commonMutations := calcCommonCounter(infos, readQuorum)
+	for i, info := range infos {
+		mutations := info.Metrics.TotalDeletes + info.Metrics.TotalWrites
+		if mutations >= commonMutations {
+			newDisks = append(newDisks, disks[i])
+			newInfos = append(newInfos, infos[i])
+		}
+	}
+
+	return newDisks, newInfos
+}
+
+func getQuorumDisks(disks []StorageAPI, infos []DiskInfo, readQuorum int) (newDisks []StorageAPI) {
+	newDisks, _ = getQuorumDiskInfos(disks, infos, readQuorum)
+	return newDisks
+}
+
 // Will return io.EOF if continuing would not yield more results.
 func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, results chan<- metaCacheEntry) (err error) {
 	defer close(results)
 	o.debugf(color.Green("listPath:")+" with options: %#v", o)
 
 	// get prioritized non-healing disks for listing
-	disks, _ := er.getOnlineDisksWithHealing(true)
+	disks, infos, _ := er.getOnlineDisksWithHealingAndInfo(true)
 	askDisks := getListQuorum(o.AskDisks, er.setDriveCount)
+	if askDisks == -1 {
+		newDisks := getQuorumDisks(disks, infos, (len(disks)+1)/2)
+		if newDisks != nil {
+			// If we found disks signature in quorum, we proceed to list
+			// from a single drive, shuffling of the drives is subsequently.
+			disks = newDisks
+			askDisks = 1
+		} else {
+			// If we did not find suitable disks, perform strict quorum listing
+			// as no disk agrees on quorum anymore.
+			askDisks = getListQuorum("strict", er.setDriveCount)
+		}
+	}
+
 	var fallbackDisks []StorageAPI
 
 	// Special case: ask all disks if the drive count is 4

cmd/metrics-realtime.go

@@ -91,7 +91,7 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 		}
 		cm, err := c.Times(false)
 		if err != nil {
-			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v  (cputimes)", globalMinioAddr, err.Error()))
+			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (cpuTimes)", globalMinioAddr, err.Error()))
 		} else {
 			// not collecting per-cpu stats, so there will be only one element
 			if len(cm) == 1 {
@@ -100,6 +100,13 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 				m.Errors = append(m.Errors, fmt.Sprintf("%s: Expected one CPU stat, got %d", globalMinioAddr, len(cm)))
 			}
 		}
+		cpuCount, err := c.Counts(true)
+		if err != nil {
+			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (cpuCount)", globalMinioAddr, err.Error()))
+		} else {
+			m.Aggregated.CPU.CPUCount = cpuCount
+		}
+
 		loadStat, err := load.Avg()
 		if err != nil {
 			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (loadStat)", globalMinioAddr, err.Error()))
@@ -129,7 +136,7 @@ func collectLocalDisksMetrics(disks map[string]struct{}) map[string]madmin.DiskM
 		return metrics
 	}
 
-	storageInfo := objLayer.LocalStorageInfo(GlobalContext)
+	storageInfo := objLayer.LocalStorageInfo(GlobalContext, true)
 	for _, d := range storageInfo.Disks {
 		if len(disks) != 0 {
 			_, ok := disks[d.Endpoint]

cmd/metrics-resource.go

@@ -53,6 +53,7 @@ const (
 
 	// memory stats
 	memUsed      MetricName = "used"
+	memUsedPerc  MetricName = "used_perc"
 	memFree      MetricName = "free"
 	memShared    MetricName = "shared"
 	memBuffers   MetricName = "buffers"
@@ -60,15 +61,18 @@ const (
 	memAvailable MetricName = "available"
 
 	// cpu stats
-	cpuUser   MetricName = "user"
-	cpuSystem MetricName = "system"
-	cpuIOWait MetricName = "iowait"
-	cpuIdle   MetricName = "idle"
-	cpuNice   MetricName = "nice"
-	cpuSteal  MetricName = "steal"
-	cpuLoad1  MetricName = "load1"
-	cpuLoad5  MetricName = "load5"
-	cpuLoad15 MetricName = "load15"
+	cpuUser       MetricName = "user"
+	cpuSystem     MetricName = "system"
+	cpuIOWait     MetricName = "iowait"
+	cpuIdle       MetricName = "idle"
+	cpuNice       MetricName = "nice"
+	cpuSteal      MetricName = "steal"
+	cpuLoad1      MetricName = "load1"
+	cpuLoad5      MetricName = "load5"
+	cpuLoad15     MetricName = "load15"
+	cpuLoad1Perc  MetricName = "load1_perc"
+	cpuLoad5Perc  MetricName = "load5_perc"
+	cpuLoad15Perc MetricName = "load15_perc"
 )
 
 var (
@@ -126,6 +130,7 @@ func init() {
 		interfaceTxErrors: "Transmit errors in " + interval,
 		total:             "Total memory on the node",
 		memUsed:           "Used memory on the node",
+		memUsedPerc:       "Used memory percentage on the node",
 		memFree:           "Free memory on the node",
 		memShared:         "Shared memory on the node",
 		memBuffers:        "Buffers memory on the node",
@@ -151,6 +156,9 @@ func init() {
 		cpuLoad1:          "CPU load average 1min",
 		cpuLoad5:          "CPU load average 5min",
 		cpuLoad15:         "CPU load average 15min",
+		cpuLoad1Perc:      "CPU load average 1min (perentage)",
+		cpuLoad5Perc:      "CPU load average 5min (percentage)",
+		cpuLoad15Perc:     "CPU load average 15min (percentage)",
 	}
 	resourceMetricsGroups = []*MetricsGroup{
 		getResourceMetrics(),
@@ -244,10 +252,10 @@ func collectDriveMetrics(m madmin.RealtimeMetrics) {
 	}
 
 	globalLocalDrivesMu.RLock()
-	gld := globalLocalDrives
+	localDrives := globalLocalDrives
 	globalLocalDrivesMu.RUnlock()
 
-	for _, d := range gld {
+	for _, d := range localDrives {
 		labels := map[string]string{"drive": d.Endpoint().RawPath}
 		di, err := d.DiskInfo(GlobalContext, false)
 		if err == nil {
@@ -283,6 +291,8 @@ func collectLocalResourceMetrics() {
 				stats := hm.Mem.Info
 				updateResourceMetrics(memSubsystem, total, float64(stats.Total), labels, false)
 				updateResourceMetrics(memSubsystem, memUsed, float64(stats.Used), labels, false)
+				perc := math.Round(float64(stats.Used*100*100)/float64(stats.Total)) / 100
+				updateResourceMetrics(memSubsystem, memUsedPerc, perc, labels, false)
 				updateResourceMetrics(memSubsystem, memFree, float64(stats.Free), labels, false)
 				updateResourceMetrics(memSubsystem, memShared, float64(stats.Shared), labels, false)
 				updateResourceMetrics(memSubsystem, memBuffers, float64(stats.Buffers), labels, false)
@@ -312,6 +322,14 @@ func collectLocalResourceMetrics() {
 					updateResourceMetrics(cpuSubsystem, cpuLoad1, ls.Load1, labels, false)
 					updateResourceMetrics(cpuSubsystem, cpuLoad5, ls.Load5, labels, false)
 					updateResourceMetrics(cpuSubsystem, cpuLoad15, ls.Load15, labels, false)
+					if hm.CPU.CPUCount > 0 {
+						perc := math.Round(ls.Load1*100*100/float64(hm.CPU.CPUCount)) / 100
+						updateResourceMetrics(cpuSubsystem, cpuLoad1Perc, perc, labels, false)
+						perc = math.Round(ls.Load5*100*100/float64(hm.CPU.CPUCount)) / 100
+						updateResourceMetrics(cpuSubsystem, cpuLoad5Perc, perc, labels, false)
+						perc = math.Round(ls.Load15*100*100/float64(hm.CPU.CPUCount)) / 100
+						updateResourceMetrics(cpuSubsystem, cpuLoad15Perc, perc, labels, false)
+					}
 				}
 			}
 			break // only one host expected

cmd/metrics-v2.go

@@ -50,32 +50,32 @@ var (
 
 func init() {
 	clusterMetricsGroups := []*MetricsGroup{
-		getNodeHealthMetrics(),
-		getClusterStorageMetrics(),
-		getClusterTierMetrics(),
-		getClusterUsageMetrics(),
-		getKMSMetrics(),
-		getClusterHealthMetrics(),
-		getIAMNodeMetrics(),
-		getReplicationSiteMetrics(),
+		getNodeHealthMetrics(MetricsGroupOpts{dependGlobalNotificationSys: true}),
+		getClusterStorageMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}),
+		getClusterTierMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}),
+		getClusterUsageMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}),
+		getKMSMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true, dependGlobalKMS: true}),
+		getClusterHealthMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}),
+		getIAMNodeMetrics(MetricsGroupOpts{dependGlobalAuthNPlugin: true, dependGlobalIAMSys: true}),
+		getReplicationSiteMetrics(MetricsGroupOpts{dependGlobalSiteReplicationSys: true}),
 	}
 
 	peerMetricsGroups = []*MetricsGroup{
 		getGoMetrics(),
-		getHTTPMetrics(false),
-		getNotificationMetrics(),
-		getLocalStorageMetrics(),
+		getHTTPMetrics(MetricsGroupOpts{}),
+		getNotificationMetrics(MetricsGroupOpts{dependGlobalLambdaTargetList: true}),
+		getLocalStorageMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}),
 		getMinioProcMetrics(),
 		getMinioVersionMetrics(),
 		getNetworkMetrics(),
 		getS3TTFBMetric(),
 		getILMNodeMetrics(),
 		getScannerNodeMetrics(),
-		getIAMNodeMetrics(),
-		getKMSNodeMetrics(),
-		getMinioHealingMetrics(),
+		getIAMNodeMetrics(MetricsGroupOpts{dependGlobalAuthNPlugin: true, dependGlobalIAMSys: true}),
+		getKMSNodeMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true, dependGlobalKMS: true}),
+		getMinioHealingMetrics(MetricsGroupOpts{dependGlobalBackgroundHealState: true}),
 		getWebhookMetrics(),
-		getReplicationClusterMetrics(),
+		getReplicationClusterMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true, dependBucketTargetSys: true}),
 	}
 
 	allMetricsGroups := func() (allMetrics []*MetricsGroup) {
@@ -85,25 +85,27 @@ func init() {
 	}()
 
 	nodeGroups := []*MetricsGroup{
-		getNodeHealthMetrics(),
-		getHTTPMetrics(false),
+		getNodeHealthMetrics(MetricsGroupOpts{dependGlobalNotificationSys: true}),
+		getHTTPMetrics(MetricsGroupOpts{}),
 		getNetworkMetrics(),
 		getMinioVersionMetrics(),
 		getS3TTFBMetric(),
-		getNotificationMetrics(),
-		getDistLockMetrics(),
-		getIAMNodeMetrics(),
+		getTierMetrics(),
+		getNotificationMetrics(MetricsGroupOpts{dependGlobalLambdaTargetList: true}),
+		getDistLockMetrics(MetricsGroupOpts{dependGlobalIsDistErasure: true, dependGlobalLockServer: true}),
+		getIAMNodeMetrics(MetricsGroupOpts{dependGlobalAuthNPlugin: true, dependGlobalIAMSys: true}),
+		getLocalStorageMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}),
 	}
 
 	bucketMetricsGroups := []*MetricsGroup{
-		getBucketUsageMetrics(),
-		getHTTPMetrics(true),
+		getBucketUsageMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}),
+		getHTTPMetrics(MetricsGroupOpts{bucketOnly: true}),
 		getBucketTTFBMetric(),
-		getBatchJobsMetrics(),
+		getBatchJobsMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}),
 	}
 
 	bucketPeerMetricsGroups = []*MetricsGroup{
-		getHTTPMetrics(true),
+		getHTTPMetrics(MetricsGroupOpts{bucketOnly: true}),
 		getBucketTTFBMetric(),
 	}
 
@@ -155,6 +157,7 @@ const (
 	usageSubsystem            MetricSubsystem = "usage"
 	quotaSubsystem            MetricSubsystem = "quota"
 	ilmSubsystem              MetricSubsystem = "ilm"
+	tierSubsystem             MetricSubsystem = "tier"
 	scannerSubsystem          MetricSubsystem = "scanner"
 	iamSubsystem              MetricSubsystem = "iam"
 	kmsSubsystem              MetricSubsystem = "kms"
@@ -246,6 +249,7 @@ const (
 	sizeDistribution    = "size_distribution"
 	versionDistribution = "version_distribution"
 	ttfbDistribution    = "seconds_distribution"
+	ttlbDistribution    = "ttlb_seconds_distribution"
 
 	lastActivityTime = "last_activity_nano_seconds"
 	startTime        = "starttime_seconds"
@@ -256,11 +260,15 @@ const (
 	expiryPendingTasks     MetricName = "expiry_pending_tasks"
 	transitionPendingTasks MetricName = "transition_pending_tasks"
 	transitionActiveTasks  MetricName = "transition_active_tasks"
+	transitionMissedTasks  MetricName = "transition_missed_immediate_tasks"
 
 	transitionedBytes    MetricName = "transitioned_bytes"
 	transitionedObjects  MetricName = "transitioned_objects"
 	transitionedVersions MetricName = "transitioned_versions"
 
+	tierRequestsSuccess MetricName = "requests_success"
+	tierRequestsFailure MetricName = "requests_failure"
+
 	kmsOnline          = "online"
 	kmsRequestsSuccess = "request_success"
 	kmsRequestsError   = "request_error"
@@ -307,8 +315,25 @@ type Metric struct {
 
 // MetricsGroup are a group of metrics that are initialized together.
 type MetricsGroup struct {
-	metricsCache  timedValue
-	cacheInterval time.Duration
+	metricsCache     timedValue
+	cacheInterval    time.Duration
+	metricsGroupOpts MetricsGroupOpts
+}
+
+// MetricsGroupOpts are a group of metrics opts to be used to initialize the metrics group.
+type MetricsGroupOpts struct {
+	dependGlobalObjectAPI           bool
+	dependGlobalAuthNPlugin         bool
+	dependGlobalSiteReplicationSys  bool
+	dependGlobalNotificationSys     bool
+	dependGlobalKMS                 bool
+	bucketOnly                      bool
+	dependGlobalLambdaTargetList    bool
+	dependGlobalIAMSys              bool
+	dependGlobalLockServer          bool
+	dependGlobalIsDistErasure       bool
+	dependGlobalBackgroundHealState bool
+	dependBucketTargetSys           bool
 }
 
 // RegisterRead register the metrics populator function to be used
@@ -318,6 +343,63 @@ func (g *MetricsGroup) RegisterRead(read func(ctx context.Context) []Metric) {
 		g.metricsCache.Relax = true
 		g.metricsCache.TTL = g.cacheInterval
 		g.metricsCache.Update = func() (interface{}, error) {
+			if g.metricsGroupOpts.dependGlobalObjectAPI {
+				objLayer := newObjectLayerFn()
+				// Service not initialized yet
+				if objLayer == nil {
+					return []Metric{}, nil
+				}
+			}
+			if g.metricsGroupOpts.dependGlobalAuthNPlugin {
+				if globalAuthNPlugin == nil {
+					return []Metric{}, nil
+				}
+			}
+			if g.metricsGroupOpts.dependGlobalSiteReplicationSys {
+				if !globalSiteReplicationSys.isEnabled() {
+					return []Metric{}, nil
+				}
+			}
+			if g.metricsGroupOpts.dependGlobalNotificationSys {
+				if globalNotificationSys == nil {
+					return []Metric{}, nil
+				}
+			}
+			if g.metricsGroupOpts.dependGlobalKMS {
+				if GlobalKMS == nil {
+					return []Metric{}, nil
+				}
+			}
+			if g.metricsGroupOpts.dependGlobalLambdaTargetList {
+				if globalLambdaTargetList == nil {
+					return []Metric{}, nil
+				}
+			}
+			if g.metricsGroupOpts.dependGlobalIAMSys {
+				if globalIAMSys == nil {
+					return []Metric{}, nil
+				}
+			}
+			if g.metricsGroupOpts.dependGlobalLockServer {
+				if globalLockServer == nil {
+					return []Metric{}, nil
+				}
+			}
+			if g.metricsGroupOpts.dependGlobalIsDistErasure {
+				if !globalIsDistErasure {
+					return []Metric{}, nil
+				}
+			}
+			if g.metricsGroupOpts.dependGlobalBackgroundHealState {
+				if globalBackgroundHealState == nil {
+					return []Metric{}, nil
+				}
+			}
+			if g.metricsGroupOpts.dependBucketTargetSys {
+				if globalBucketTargetSys == nil {
+					return []Metric{}, nil
+				}
+			}
 			return read(GlobalContext), nil
 		}
 	})
@@ -1597,47 +1679,52 @@ func getGoMetrics() *MetricsGroup {
 	return mg
 }
 
+// getHistogramMetrics fetches histogram metrics and returns it in a []Metric
+// Note: Typically used in MetricGroup.RegisterRead
+func getHistogramMetrics(hist *prometheus.HistogramVec, desc MetricDescription) []Metric {
+	ch := make(chan prometheus.Metric)
+	go func() {
+		defer close(ch)
+		// Collects prometheus metrics from hist and sends it over ch
+		hist.Collect(ch)
+	}()
+
+	// Converts metrics received into internal []Metric type
+	var metrics []Metric
+	for promMetric := range ch {
+		dtoMetric := &dto.Metric{}
+		err := promMetric.Write(dtoMetric)
+		if err != nil {
+			// Log error and continue to receive other metric
+			// values
+			logger.LogIf(GlobalContext, err)
+			continue
+		}
+
+		h := dtoMetric.GetHistogram()
+		for _, b := range h.Bucket {
+			labels := make(map[string]string)
+			for _, lp := range dtoMetric.GetLabel() {
+				labels[*lp.Name] = *lp.Value
+			}
+			labels["le"] = fmt.Sprintf("%.3f", *b.UpperBound)
+			metric := Metric{
+				Description:    desc,
+				VariableLabels: labels,
+				Value:          float64(b.GetCumulativeCount()),
+			}
+			metrics = append(metrics, metric)
+		}
+	}
+	return metrics
+}
+
 func getBucketTTFBMetric() *MetricsGroup {
 	mg := &MetricsGroup{
 		cacheInterval: 10 * time.Second,
 	}
-	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
-		// Read prometheus metric on this channel
-		ch := make(chan prometheus.Metric)
-		var wg sync.WaitGroup
-		wg.Add(1)
-
-		// Read prometheus histogram data and convert it to internal metric data
-		go func() {
-			defer wg.Done()
-			for promMetric := range ch {
-				dtoMetric := &dto.Metric{}
-				err := promMetric.Write(dtoMetric)
-				if err != nil {
-					logger.LogIf(GlobalContext, err)
-					return
-				}
-				h := dtoMetric.GetHistogram()
-				for _, b := range h.Bucket {
-					labels := make(map[string]string)
-					for _, lp := range dtoMetric.GetLabel() {
-						labels[*lp.Name] = *lp.Value
-					}
-					labels["le"] = fmt.Sprintf("%.3f", *b.UpperBound)
-					metric := Metric{
-						Description:    getBucketTTFBDistributionMD(),
-						VariableLabels: labels,
-						Value:          float64(b.GetCumulativeCount()),
-					}
-					metrics = append(metrics, metric)
-				}
-			}
-		}()
-
-		bucketHTTPRequestsDuration.Collect(ch)
-		close(ch)
-		wg.Wait()
-		return
+	mg.RegisterRead(func(ctx context.Context) []Metric {
+		return getHistogramMetrics(bucketHTTPRequestsDuration, getBucketTTFBDistributionMD())
 	})
 	return mg
 }
@@ -1646,43 +1733,18 @@ func getS3TTFBMetric() *MetricsGroup {
 	mg := &MetricsGroup{
 		cacheInterval: 10 * time.Second,
 	}
-	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
-		// Read prometheus metric on this channel
-		ch := make(chan prometheus.Metric)
-		var wg sync.WaitGroup
-		wg.Add(1)
+	mg.RegisterRead(func(ctx context.Context) []Metric {
+		return getHistogramMetrics(httpRequestsDuration, getS3TTFBDistributionMD())
+	})
+	return mg
+}
 
-		// Read prometheus histogram data and convert it to internal metric data
-		go func() {
-			defer wg.Done()
-			for promMetric := range ch {
-				dtoMetric := &dto.Metric{}
-				err := promMetric.Write(dtoMetric)
-				if err != nil {
-					logger.LogIf(GlobalContext, err)
-					return
-				}
-				h := dtoMetric.GetHistogram()
-				for _, b := range h.Bucket {
-					labels := make(map[string]string)
-					for _, lp := range dtoMetric.GetLabel() {
-						labels[*lp.Name] = *lp.Value
-					}
-					labels["le"] = fmt.Sprintf("%.3f", *b.UpperBound)
-					metric := Metric{
-						Description:    getS3TTFBDistributionMD(),
-						VariableLabels: labels,
-						Value:          float64(b.GetCumulativeCount()),
-					}
-					metrics = append(metrics, metric)
-				}
-			}
-		}()
-
-		httpRequestsDuration.Collect(ch)
-		close(ch)
-		wg.Wait()
-		return
+func getTierMetrics() *MetricsGroup {
+	mg := &MetricsGroup{
+		cacheInterval: 10 * time.Second,
+	}
+	mg.RegisterRead(func(ctx context.Context) []Metric {
+		return globalTierMetrics.Report()
 	})
 	return mg
 }
@@ -1707,6 +1769,16 @@ func getTransitionActiveTasksMD() MetricDescription {
 	}
 }
 
+func getTransitionMissedTasksMD() MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: ilmSubsystem,
+		Name:      transitionMissedTasks,
+		Help:      "Number of missed immediate ILM transition tasks",
+		Type:      gaugeMetric,
+	}
+}
+
 func getExpiryPendingTasksMD() MetricDescription {
 	return MetricDescription{
 		Namespace: nodeMetricNamespace,
@@ -1781,17 +1853,22 @@ func getILMNodeMetrics() *MetricsGroup {
 		trActiveTasks := Metric{
 			Description: getTransitionActiveTasksMD(),
 		}
+		trMissedTasks := Metric{
+			Description: getTransitionMissedTasksMD(),
+		}
 		if globalExpiryState != nil {
 			expPendingTasks.Value = float64(globalExpiryState.PendingTasks())
 		}
 		if globalTransitionState != nil {
 			trPendingTasks.Value = float64(globalTransitionState.PendingTasks())
 			trActiveTasks.Value = float64(globalTransitionState.ActiveTasks())
+			trMissedTasks.Value = float64(globalTransitionState.MissedImmediateTasks())
 		}
 		return []Metric{
 			expPendingTasks,
 			trPendingTasks,
 			trActiveTasks,
+			trMissedTasks,
 		}
 	})
 	return mg
@@ -1886,9 +1963,10 @@ func getScannerNodeMetrics() *MetricsGroup {
 	return mg
 }
 
-func getIAMNodeMetrics() *MetricsGroup {
+func getIAMNodeMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 10 * time.Second,
+		cacheInterval:    10 * time.Second,
+		metricsGroupOpts: opts,
 	}
 	mg.RegisterRead(func(_ context.Context) (metrics []Metric) {
 		lastSyncTime := atomic.LoadUint64(&globalIAMSys.LastRefreshTimeUnixNano)
@@ -2007,9 +2085,10 @@ func getIAMNodeMetrics() *MetricsGroup {
 }
 
 // replication metrics for each node - published to the cluster endpoint with nodename as label
-func getReplicationClusterMetrics() *MetricsGroup {
+func getReplicationClusterMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 1 * time.Minute,
+		cacheInterval:    1 * time.Minute,
+		metricsGroupOpts: opts,
 	}
 	const (
 		Online  = 1
@@ -2194,9 +2273,10 @@ func getReplicationClusterMetrics() *MetricsGroup {
 }
 
 // replication metrics for site replication
-func getReplicationSiteMetrics() *MetricsGroup {
+func getReplicationSiteMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 1 * time.Minute,
+		cacheInterval:    1 * time.Minute,
+		metricsGroupOpts: opts,
 	}
 	mg.RegisterRead(func(_ context.Context) []Metric {
 		ml := []Metric{}
@@ -2293,9 +2373,10 @@ func getMinioVersionMetrics() *MetricsGroup {
 	return mg
 }
 
-func getNodeHealthMetrics() *MetricsGroup {
+func getNodeHealthMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 1 * time.Minute,
+		cacheInterval:    1 * time.Minute,
+		metricsGroupOpts: opts,
 	}
 	mg.RegisterRead(func(_ context.Context) (metrics []Metric) {
 		metrics = make([]Metric, 0, 16)
@@ -2313,9 +2394,10 @@ func getNodeHealthMetrics() *MetricsGroup {
 	return mg
 }
 
-func getMinioHealingMetrics() *MetricsGroup {
+func getMinioHealingMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 10 * time.Second,
+		cacheInterval:    10 * time.Second,
+		metricsGroupOpts: opts,
 	}
 	mg.RegisterRead(func(_ context.Context) (metrics []Metric) {
 		bgSeq, exists := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
@@ -2383,9 +2465,10 @@ func getObjectsScanned(seq *healSequence) (m []Metric) {
 	return
 }
 
-func getDistLockMetrics() *MetricsGroup {
+func getDistLockMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 1 * time.Second,
+		cacheInterval:    1 * time.Second,
+		metricsGroupOpts: opts,
 	}
 	mg.RegisterRead(func(ctx context.Context) []Metric {
 		if !globalIsDistErasure {
@@ -2430,9 +2513,10 @@ func getDistLockMetrics() *MetricsGroup {
 	return mg
 }
 
-func getNotificationMetrics() *MetricsGroup {
+func getNotificationMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 10 * time.Second,
+		cacheInterval:    10 * time.Second,
+		metricsGroupOpts: opts,
 	}
 	mg.RegisterRead(func(ctx context.Context) []Metric {
 		metrics := make([]Metric, 0, 3)
@@ -2444,7 +2528,7 @@ func getNotificationMetrics() *MetricsGroup {
 					Namespace: minioNamespace,
 					Subsystem: notifySubsystem,
 					Name:      "current_send_in_progress",
-					Help:      "Number of concurrent async Send calls active to all targets",
+					Help:      "Number of concurrent async Send calls active to all targets (deprecated, please use 'minio_notify_target_current_send_in_progress' instead)",
 					Type:      gaugeMetric,
 				},
 				Value: float64(nstats.CurrentSendCalls),
@@ -2454,31 +2538,74 @@ func getNotificationMetrics() *MetricsGroup {
 					Namespace: minioNamespace,
 					Subsystem: notifySubsystem,
 					Name:      "events_skipped_total",
-					Help:      "Events that were skipped due to full queue",
+					Help:      "Events that were skipped to be sent to the targets due to the in-memory queue being full",
 					Type:      counterMetric,
 				},
 				Value: float64(nstats.EventsSkipped),
 			})
+			metrics = append(metrics, Metric{
+				Description: MetricDescription{
+					Namespace: minioNamespace,
+					Subsystem: notifySubsystem,
+					Name:      "events_errors_total",
+					Help:      "Events that were failed to be sent to the targets (deprecated, please use 'minio_notify_target_failed_events' instead)",
+					Type:      counterMetric,
+				},
+				Value: float64(nstats.EventsErrorsTotal),
+			})
 			metrics = append(metrics, Metric{
 				Description: MetricDescription{
 					Namespace: minioNamespace,
 					Subsystem: notifySubsystem,
 					Name:      "events_sent_total",
-					Help:      "Total number of events sent since start",
+					Help:      "Total number of events sent to the targets (deprecated, please use 'minio_notify_target_total_events' instead)",
 					Type:      counterMetric,
 				},
 				Value: float64(nstats.TotalEvents),
 			})
-			for _, st := range nstats.TargetStats {
+			for id, st := range nstats.TargetStats {
+				metrics = append(metrics, Metric{
+					Description: MetricDescription{
+						Namespace: minioNamespace,
+						Subsystem: notifySubsystem,
+						Name:      "target_total_events",
+						Help:      "Total number of events sent (or) queued to the target",
+						Type:      counterMetric,
+					},
+					VariableLabels: map[string]string{"target_id": id.ID, "target_name": id.Name},
+					Value:          float64(st.TotalEvents),
+				})
+				metrics = append(metrics, Metric{
+					Description: MetricDescription{
+						Namespace: minioNamespace,
+						Subsystem: notifySubsystem,
+						Name:      "target_failed_events",
+						Help:      "Number of events failed to be sent (or) queued to the target",
+						Type:      counterMetric,
+					},
+					VariableLabels: map[string]string{"target_id": id.ID, "target_name": id.Name},
+					Value:          float64(st.FailedEvents),
+				})
+				metrics = append(metrics, Metric{
+					Description: MetricDescription{
+						Namespace: minioNamespace,
+						Subsystem: notifySubsystem,
+						Name:      "target_current_send_in_progress",
+						Help:      "Number of concurrent async Send calls active to the target",
+						Type:      gaugeMetric,
+					},
+					VariableLabels: map[string]string{"target_id": id.ID, "target_name": id.Name},
+					Value:          float64(st.CurrentSendCalls),
+				})
 				metrics = append(metrics, Metric{
 					Description: MetricDescription{
 						Namespace: minioNamespace,
 						Subsystem: notifySubsystem,
 						Name:      "target_queue_length",
-						Help:      "Number of unsent notifications in queue for target",
+						Help:      "Number of events currently staged in the queue_dir configured for the target",
 						Type:      gaugeMetric,
 					},
-					VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name},
+					VariableLabels: map[string]string{"target_id": id.ID, "target_name": id.Name},
 					Value:          float64(st.CurrentQueue),
 				})
 			}
@@ -2562,12 +2689,13 @@ func getNotificationMetrics() *MetricsGroup {
 	return mg
 }
 
-func getHTTPMetrics(bucketOnly bool) *MetricsGroup {
+func getHTTPMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 10 * time.Second,
+		cacheInterval:    10 * time.Second,
+		metricsGroupOpts: opts,
 	}
 	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
-		if !bucketOnly {
+		if !mg.metricsGroupOpts.bucketOnly {
 			httpStats := globalHTTPStats.toServerHTTPStats()
 			metrics = make([]Metric, 0, 3+
 				len(httpStats.CurrentS3Requests.APIStats)+
@@ -2753,16 +2881,13 @@ func getNetworkMetrics() *MetricsGroup {
 	return mg
 }
 
-func getClusterUsageMetrics() *MetricsGroup {
+func getClusterUsageMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 1 * time.Minute,
+		cacheInterval:    1 * time.Minute,
+		metricsGroupOpts: opts,
 	}
 	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
 		objLayer := newObjectLayerFn()
-		// Service not initialized yet
-		if objLayer == nil {
-			return
-		}
 
 		metrics = make([]Metric, 0, 50)
 		dataUsageInfo, err := loadDataUsageFromBackend(ctx, objLayer)
@@ -2859,16 +2984,13 @@ func getClusterUsageMetrics() *MetricsGroup {
 	return mg
 }
 
-func getBucketUsageMetrics() *MetricsGroup {
+func getBucketUsageMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 1 * time.Minute,
+		cacheInterval:    1 * time.Minute,
+		metricsGroupOpts: opts,
 	}
 	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
 		objLayer := newObjectLayerFn()
-		// Service not initialized yet
-		if objLayer == nil {
-			return
-		}
 
 		metrics = make([]Metric, 0, 50)
 		dataUsageInfo, err := loadDataUsageFromBackend(ctx, objLayer)
@@ -3048,15 +3170,14 @@ func getClusterTransitionedVersionsMD() MetricDescription {
 	}
 }
 
-func getClusterTierMetrics() *MetricsGroup {
+func getClusterTierMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 1 * time.Minute,
+		cacheInterval:    1 * time.Minute,
+		metricsGroupOpts: opts,
 	}
 	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
 		objLayer := newObjectLayerFn()
-		if objLayer == nil {
-			return
-		}
+
 		if globalTierConfigMgr.Empty() {
 			return
 		}
@@ -3076,19 +3197,16 @@ func getClusterTierMetrics() *MetricsGroup {
 	return mg
 }
 
-func getLocalStorageMetrics() *MetricsGroup {
+func getLocalStorageMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 1 * time.Minute,
+		cacheInterval:    1 * time.Minute,
+		metricsGroupOpts: opts,
 	}
 	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
 		objLayer := newObjectLayerFn()
-		// Service not initialized yet
-		if objLayer == nil {
-			return
-		}
 
 		metrics = make([]Metric, 0, 50)
-		storageInfo := objLayer.LocalStorageInfo(ctx)
+		storageInfo := objLayer.LocalStorageInfo(ctx, true)
 		onlineDrives, offlineDrives := getOnlineOfflineDisksStats(storageInfo.Disks)
 		totalDrives := onlineDrives.Merge(offlineDrives)
 
@@ -3190,32 +3308,59 @@ func getClusterHealthStatusMD() MetricDescription {
 	}
 }
 
-func getClusterErasureSetToleranceMD() MetricDescription {
+func getClusterErasureSetReadQuorumMD() MetricDescription {
 	return MetricDescription{
 		Namespace: clusterMetricNamespace,
 		Subsystem: "health",
-		Name:      "erasure_set_tolerance",
-		Help:      "Get erasure set tolerance status",
+		Name:      "erasure_set_read_quorum",
+		Help:      "Get the read quorum for this erasure set",
 		Type:      gaugeMetric,
 	}
 }
 
-func getClusterHealthMetrics() *MetricsGroup {
+func getClusterErasureSetWriteQuorumMD() MetricDescription {
+	return MetricDescription{
+		Namespace: clusterMetricNamespace,
+		Subsystem: "health",
+		Name:      "erasure_set_write_quorum",
+		Help:      "Get the write quorum for this erasure set",
+		Type:      gaugeMetric,
+	}
+}
+
+func getClusterErasureSetOnlineDrivesMD() MetricDescription {
+	return MetricDescription{
+		Namespace: clusterMetricNamespace,
+		Subsystem: "health",
+		Name:      "erasure_set_online_drives",
+		Help:      "Get the count of the online drives in this erasure set",
+		Type:      gaugeMetric,
+	}
+}
+
+func getClusterErasureSetHealingDrivesMD() MetricDescription {
+	return MetricDescription{
+		Namespace: clusterMetricNamespace,
+		Subsystem: "health",
+		Name:      "erasure_set_healing_drives",
+		Help:      "Get the count of healing drives of this erasure set",
+		Type:      gaugeMetric,
+	}
+}
+
+func getClusterHealthMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 10 * time.Second,
+		cacheInterval:    10 * time.Second,
+		metricsGroupOpts: opts,
 	}
 	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
 		objLayer := newObjectLayerFn()
-		// Service not initialized yet
-		if objLayer == nil {
-			return
-		}
-
-		metrics = make([]Metric, 0, 2)
 
 		opts := HealthOptions{}
 		result := objLayer.Health(ctx, opts)
 
+		metrics = make([]Metric, 0, 2+4*len(result.ESHealth))
+
 		metrics = append(metrics, Metric{
 			Description: getClusterWriteQuorumMD(),
 			Value:       float64(result.WriteQuorum),
@@ -3237,9 +3382,24 @@ func getClusterHealthMetrics() *MetricsGroup {
 				"set":  strconv.Itoa(h.SetID),
 			}
 			metrics = append(metrics, Metric{
-				Description:    getClusterErasureSetToleranceMD(),
+				Description:    getClusterErasureSetReadQuorumMD(),
 				VariableLabels: labels,
-				Value:          float64(h.HealthyDrives - h.WriteQuorum),
+				Value:          float64(h.ReadQuorum),
+			})
+			metrics = append(metrics, Metric{
+				Description:    getClusterErasureSetWriteQuorumMD(),
+				VariableLabels: labels,
+				Value:          float64(h.WriteQuorum),
+			})
+			metrics = append(metrics, Metric{
+				Description:    getClusterErasureSetOnlineDrivesMD(),
+				VariableLabels: labels,
+				Value:          float64(h.HealthyDrives),
+			})
+			metrics = append(metrics, Metric{
+				Description:    getClusterErasureSetHealingDrivesMD(),
+				VariableLabels: labels,
+				Value:          float64(h.HealingDrives),
 			})
 		}
 
@@ -3249,18 +3409,13 @@ func getClusterHealthMetrics() *MetricsGroup {
 	return mg
 }
 
-func getBatchJobsMetrics() *MetricsGroup {
+func getBatchJobsMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 10 * time.Second,
+		cacheInterval:    10 * time.Second,
+		metricsGroupOpts: opts,
 	}
 
 	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
-		objLayer := newObjectLayerFn()
-		// Service not initialized yet
-		if objLayer == nil {
-			return
-		}
-
 		var m madmin.RealtimeMetrics
 		mLocal := collectLocalMetrics(madmin.MetricsBatchJobs, collectMetricsOpts{})
 		m.Merge(&mLocal)
@@ -3320,20 +3475,17 @@ func getBatchJobsMetrics() *MetricsGroup {
 	return mg
 }
 
-func getClusterStorageMetrics() *MetricsGroup {
+func getClusterStorageMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 1 * time.Minute,
+		cacheInterval:    1 * time.Minute,
+		metricsGroupOpts: opts,
 	}
 	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
 		objLayer := newObjectLayerFn()
-		// Service not initialized yet
-		if objLayer == nil {
-			return
-		}
 
 		// Fetch disk space info, ignore errors
 		metrics = make([]Metric, 0, 10)
-		storageInfo := objLayer.StorageInfo(ctx)
+		storageInfo := objLayer.StorageInfo(ctx, true)
 		onlineDrives, offlineDrives := getOnlineOfflineDisksStats(storageInfo.Disks)
 		totalDrives := onlineDrives.Merge(offlineDrives)
 
@@ -3376,18 +3528,13 @@ func getClusterStorageMetrics() *MetricsGroup {
 	return mg
 }
 
-func getKMSNodeMetrics() *MetricsGroup {
+func getKMSNodeMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 10 * time.Second,
+		cacheInterval:    10 * time.Second,
+		metricsGroupOpts: opts,
 	}
 
 	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
-		objLayer := newObjectLayerFn()
-		// Service not initialized yet
-		if objLayer == nil || GlobalKMS == nil {
-			return
-		}
-
 		const (
 			Online  = 1
 			Offline = 0
@@ -3481,18 +3628,13 @@ func getWebhookMetrics() *MetricsGroup {
 	return mg
 }
 
-func getKMSMetrics() *MetricsGroup {
+func getKMSMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
-		cacheInterval: 10 * time.Second,
+		cacheInterval:    10 * time.Second,
+		metricsGroupOpts: opts,
 	}
 
 	mg.RegisterRead(func(ctx context.Context) []Metric {
-		objLayer := newObjectLayerFn()
-		// Service not initialized yet
-		if objLayer == nil || GlobalKMS == nil {
-			return []Metric{}
-		}
-
 		metrics := make([]Metric, 0, 4)
 		metric, err := GlobalKMS.Metrics(ctx)
 		if err != nil {

cmd/metrics-v2_test.go

@@ -0,0 +1,87 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"testing"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+func TestGetHistogramMetrics(t *testing.T) {
+	histBuckets := []float64{0.05, 0.1, 0.25, 0.5, 0.75}
+	labels := []string{"GetObject", "PutObject", "CopyObject", "CompleteMultipartUpload"}
+	ttfbHist := prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "s3_ttfb_seconds",
+			Help:    "Time taken by requests served by current MinIO server instance",
+			Buckets: histBuckets,
+		},
+		[]string{"api"},
+	)
+	observations := []struct {
+		val   float64
+		label string
+	}{
+		{
+			val:   0.02,
+			label: labels[0],
+		},
+		{
+			val:   0.07,
+			label: labels[1],
+		},
+		{
+			val:   0.11,
+			label: labels[1],
+		},
+		{
+			val:   0.19,
+			label: labels[1],
+		},
+		{
+			val:   0.31,
+			label: labels[1],
+		},
+		{
+			val:   0.61,
+			label: labels[3],
+		},
+		{
+			val:   0.79,
+			label: labels[2],
+		},
+	}
+	ticker := time.NewTicker(1 * time.Millisecond)
+	defer ticker.Stop()
+	for _, obs := range observations {
+		// Send observations once every 1ms, to simulate delay between
+		// observations. This is to test the channel based
+		// synchronization used internally.
+		select {
+		case <-ticker.C:
+			ttfbHist.With(prometheus.Labels{"api": obs.label}).Observe(obs.val)
+		}
+	}
+
+	metrics := getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD())
+	if expPoints := len(labels) * len(histBuckets); expPoints != len(metrics) {
+		t.Fatalf("Expected %v data points but got %v", expPoints, len(metrics))
+	}
+}

cmd/metrics.go

@@ -406,7 +406,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 		float64(GetTotalCapacityFree(server.Disks)),
 	)
 
-	sinfo := objLayer.StorageInfo(GlobalContext)
+	sinfo := objLayer.StorageInfo(GlobalContext, true)
 
 	// Report total usable capacity
 	ch <- prometheus.MustNewConstMetric(

cmd/mrf.go

@@ -38,6 +38,7 @@ type partialOperation struct {
 	allVersions         bool
 	setIndex, poolIndex int
 	queued              time.Time
+	scanMode            madmin.HealScanMode
 }
 
 // mrfState sncapsulates all the information
@@ -97,19 +98,23 @@ func (m *mrfState) healRoutine() {
 				// let recently failed networks to reconnect
 				// making MRF wait for 1s before retrying,
 				// i.e 4 reconnect attempts.
-				time.Sleep(1 * time.Second)
+				time.Sleep(time.Second)
 			}
 
 			// wait on timer per heal
 			wait := healSleeper.Timer(context.Background())
 
+			scan := madmin.HealNormalScan
+			if u.scanMode != 0 {
+				scan = u.scanMode
+			}
 			if u.object == "" {
-				healBucket(u.bucket, madmin.HealNormalScan)
+				healBucket(u.bucket, scan)
 			} else {
 				if u.allVersions {
-					m.pools.serverPools[u.poolIndex].sets[u.setIndex].listAndHeal(u.bucket, u.object, healObjectVersionsDisparity)
+					m.pools.serverPools[u.poolIndex].sets[u.setIndex].listAndHeal(u.bucket, u.object, u.scanMode, healObjectVersionsDisparity)
 				} else {
-					healObject(u.bucket, u.object, u.versionID, madmin.HealNormalScan)
+					healObject(u.bucket, u.object, u.versionID, scan)
 				}
 			}
 

cmd/naughty-disk_test.go

@@ -112,8 +112,11 @@ func (d *naughtyDisk) SetDiskID(id string) {
 	d.disk.SetDiskID(id)
 }
 
-func (d *naughtyDisk) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode) (info dataUsageCache, err error) {
-	return d.disk.NSScanner(ctx, cache, updates, scanMode)
+func (d *naughtyDisk) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode, weSleep func() bool) (info dataUsageCache, err error) {
+	if err := d.calcError(); err != nil {
+		return info, err
+	}
+	return d.disk.NSScanner(ctx, cache, updates, scanMode, weSleep)
 }
 
 func (d *naughtyDisk) DiskInfo(ctx context.Context, metrics bool) (info DiskInfo, err error) {
@@ -200,11 +203,11 @@ func (d *naughtyDisk) AppendFile(ctx context.Context, volume string, path string
 	return d.disk.AppendFile(ctx, volume, path, buf)
 }
 
-func (d *naughtyDisk) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string) (uint64, error) {
+func (d *naughtyDisk) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (uint64, error) {
 	if err := d.calcError(); err != nil {
 		return 0, err
 	}
-	return d.disk.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath)
+	return d.disk.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts)
 }
 
 func (d *naughtyDisk) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error {
@@ -228,7 +231,7 @@ func (d *naughtyDisk) Delete(ctx context.Context, volume string, path string, de
 	return d.disk.Delete(ctx, volume, path, deleteOpts)
 }
 
-func (d *naughtyDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions) []error {
+func (d *naughtyDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions, opts DeleteOptions) []error {
 	if err := d.calcError(); err != nil {
 		errs := make([]error, len(versions))
 		for i := range errs {
@@ -236,7 +239,7 @@ func (d *naughtyDisk) DeleteVersions(ctx context.Context, volume string, version
 		}
 		return errs
 	}
-	return d.disk.DeleteVersions(ctx, volume, versions)
+	return d.disk.DeleteVersions(ctx, volume, versions, opts)
 }
 
 func (d *naughtyDisk) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) {
@@ -253,11 +256,11 @@ func (d *naughtyDisk) UpdateMetadata(ctx context.Context, volume, path string, f
 	return d.disk.UpdateMetadata(ctx, volume, path, fi, opts)
 }
 
-func (d *naughtyDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool) (err error) {
+func (d *naughtyDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) (err error) {
 	if err := d.calcError(); err != nil {
 		return err
 	}
-	return d.disk.DeleteVersion(ctx, volume, path, fi, forceDelMarker)
+	return d.disk.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts)
 }
 
 func (d *naughtyDisk) ReadVersion(ctx context.Context, volume, path, versionID string, opts ReadOptions) (fi FileInfo, err error) {

cmd/notification.go

@@ -975,8 +975,11 @@ func getOfflineDisks(offlineHost string, endpoints EndpointServerPools) []madmin
 		for _, ep := range pool.Endpoints {
 			if offlineHost == "" && ep.IsLocal || offlineHost == ep.Host {
 				offlineDisks = append(offlineDisks, madmin.Disk{
-					Endpoint: ep.String(),
-					State:    string(madmin.ItemOffline),
+					Endpoint:  ep.String(),
+					State:     string(madmin.ItemOffline),
+					PoolIndex: ep.PoolIdx,
+					SetIndex:  ep.SetIdx,
+					DiskIndex: ep.DiskIdx,
 				})
 			}
 		}
@@ -985,7 +988,7 @@ func getOfflineDisks(offlineHost string, endpoints EndpointServerPools) []madmin
 }
 
 // StorageInfo returns disk information across all peers
-func (sys *NotificationSys) StorageInfo(objLayer ObjectLayer) StorageInfo {
+func (sys *NotificationSys) StorageInfo(objLayer ObjectLayer, metrics bool) StorageInfo {
 	var storageInfo StorageInfo
 	replies := make([]StorageInfo, len(sys.peerClients))
 
@@ -997,7 +1000,7 @@ func (sys *NotificationSys) StorageInfo(objLayer ObjectLayer) StorageInfo {
 		wg.Add(1)
 		go func(client *peerRESTClient, idx int) {
 			defer wg.Done()
-			info, err := client.LocalStorageInfo()
+			info, err := client.LocalStorageInfo(metrics)
 			if err != nil {
 				info.Disks = getOfflineDisks(client.host.String(), globalEndpoints)
 			}
@@ -1007,7 +1010,7 @@ func (sys *NotificationSys) StorageInfo(objLayer ObjectLayer) StorageInfo {
 	wg.Wait()
 
 	// Add local to this server.
-	replies = append(replies, objLayer.LocalStorageInfo(GlobalContext))
+	replies = append(replies, objLayer.LocalStorageInfo(GlobalContext, metrics))
 
 	storageInfo.Backend = objLayer.BackendInfo()
 	for _, sinfo := range replies {

cmd/object-api-datatypes.go

@@ -54,16 +54,35 @@ type objectHistogramInterval struct {
 }
 
 const (
+	// dataUsageBucketLenV1 must be length of ObjectsHistogramIntervalsV1
+	dataUsageBucketLenV1 = 7
 	// dataUsageBucketLen must be length of ObjectsHistogramIntervals
-	dataUsageBucketLen  = 7
+	dataUsageBucketLen  = 11
 	dataUsageVersionLen = 7
 )
 
+// ObjectsHistogramIntervalsV1 is the list of all intervals
+// of object sizes to be included in objects histogram(V1).
+var ObjectsHistogramIntervalsV1 = [dataUsageBucketLenV1]objectHistogramInterval{
+	{"LESS_THAN_1024_B", 0, humanize.KiByte - 1},
+	{"BETWEEN_1024B_AND_1_MB", humanize.KiByte, humanize.MiByte - 1},
+	{"BETWEEN_1_MB_AND_10_MB", humanize.MiByte, humanize.MiByte*10 - 1},
+	{"BETWEEN_10_MB_AND_64_MB", humanize.MiByte * 10, humanize.MiByte*64 - 1},
+	{"BETWEEN_64_MB_AND_128_MB", humanize.MiByte * 64, humanize.MiByte*128 - 1},
+	{"BETWEEN_128_MB_AND_512_MB", humanize.MiByte * 128, humanize.MiByte*512 - 1},
+	{"GREATER_THAN_512_MB", humanize.MiByte * 512, math.MaxInt64},
+}
+
 // ObjectsHistogramIntervals is the list of all intervals
 // of object sizes to be included in objects histogram.
+// Note: this histogram expands 1024B-1MB to incl. 1024B-64KB, 64KB-256KB, 256KB-512KB and 512KB-1MiB
 var ObjectsHistogramIntervals = [dataUsageBucketLen]objectHistogramInterval{
 	{"LESS_THAN_1024_B", 0, humanize.KiByte - 1},
-	{"BETWEEN_1024_B_AND_1_MB", humanize.KiByte, humanize.MiByte - 1},
+	{"BETWEEN_1024_B_AND_64_KB", humanize.KiByte, 64*humanize.KiByte - 1},         // not exported, for support use only
+	{"BETWEEN_64_KB_AND_256_KB", 64 * humanize.KiByte, 256*humanize.KiByte - 1},   // not exported, for support use only
+	{"BETWEEN_256_KB_AND_512_KB", 256 * humanize.KiByte, 512*humanize.KiByte - 1}, // not exported, for support use only
+	{"BETWEEN_512_KB_AND_1_MB", 512 * humanize.KiByte, humanize.MiByte - 1},       // not exported, for support use only
+	{"BETWEEN_1024B_AND_1_MB", humanize.KiByte, humanize.MiByte - 1},
 	{"BETWEEN_1_MB_AND_10_MB", humanize.MiByte, humanize.MiByte*10 - 1},
 	{"BETWEEN_10_MB_AND_64_MB", humanize.MiByte * 10, humanize.MiByte*64 - 1},
 	{"BETWEEN_64_MB_AND_128_MB", humanize.MiByte * 64, humanize.MiByte*128 - 1},
@@ -181,8 +200,6 @@ type ObjectInfo struct {
 
 	Legacy bool // indicates object on disk is in legacy data format
 
-	// backendType indicates which backend filled this structure
-	backendType BackendType
 	// internal representation of version purge status
 	VersionPurgeStatusInternal string
 	VersionPurgeStatus         VersionPurgeStatusType
@@ -263,7 +280,6 @@ func (o *ObjectInfo) Clone() (cinfo ObjectInfo) {
 		metadataOnly:               o.metadataOnly,
 		versionOnly:                o.versionOnly,
 		keyRotation:                o.keyRotation,
-		backendType:                o.backendType,
 		AccTime:                    o.AccTime,
 		Legacy:                     o.Legacy,
 		VersionPurgeStatus:         o.VersionPurgeStatus,
@@ -612,3 +628,42 @@ type NewMultipartUploadResult struct {
 	UploadID     string
 	ChecksumAlgo string
 }
+
+type getObjectAttributesResponse struct {
+	ETag         string                    `xml:",omitempty"`
+	Checksum     *objectAttributesChecksum `xml:",omitempty"`
+	ObjectParts  *objectAttributesParts    `xml:",omitempty"`
+	StorageClass string                    `xml:",omitempty"`
+	ObjectSize   int64                     `xml:",omitempty"`
+}
+
+type objectAttributesChecksum struct {
+	ChecksumCRC32  string `xml:",omitempty"`
+	ChecksumCRC32C string `xml:",omitempty"`
+	ChecksumSHA1   string `xml:",omitempty"`
+	ChecksumSHA256 string `xml:",omitempty"`
+}
+
+type objectAttributesParts struct {
+	IsTruncated          bool
+	MaxParts             int
+	NextPartNumberMarker int
+	PartNumberMarker     int
+	PartsCount           int
+	Parts                []*objectAttributesPart `xml:"Part"`
+}
+
+type objectAttributesPart struct {
+	PartNumber     int
+	Size           int64
+	ChecksumCRC32  string `xml:",omitempty"`
+	ChecksumCRC32C string `xml:",omitempty"`
+	ChecksumSHA1   string `xml:",omitempty"`
+	ChecksumSHA256 string `xml:",omitempty"`
+}
+
+type objectAttributesErrorResponse struct {
+	ArgumentValue *string `xml:"ArgumentValue,omitempty"`
+	ArgumentName  *string `xml:"ArgumentName"`
+	APIErrorResponse
+}

cmd/object-api-interface.go

@@ -69,6 +69,9 @@ type ObjectOptions struct {
 	Tagging                 bool // Is only in GET/HEAD operations to return tagging metadata along with regular metadata and body.
 
 	UserDefined         map[string]string   // only set in case of POST/PUT operations
+	ObjectAttributes    map[string]struct{} // Attribute tags defined by the users for the GetObjectAttributes request
+	MaxParts            int                 // used in GetObjectAttributes. Signals how many parts we should return
+	PartNumberMarker    int                 // used in GetObjectAttributes. Signals the part number after which results should be returned
 	PartNumber          int                 // only useful in case of GetObject/HeadObject
 	CheckPrecondFn      CheckPreconditionFn // only set during GetObject/HeadObject/CopyObjectPart preconditional valuation
 	EvalMetadataFn      EvalMetadataFn      // only set for retention settings, meant to be used only when updating metadata in-place.
@@ -228,8 +231,8 @@ type ObjectLayer interface {
 	Shutdown(context.Context) error
 	NSScanner(ctx context.Context, updates chan<- DataUsageInfo, wantCycle uint32, scanMode madmin.HealScanMode) error
 	BackendInfo() madmin.BackendInfo
-	StorageInfo(ctx context.Context) StorageInfo
-	LocalStorageInfo(ctx context.Context) StorageInfo
+	StorageInfo(ctx context.Context, metrics bool) StorageInfo
+	LocalStorageInfo(ctx context.Context, metrics bool) StorageInfo
 
 	// Bucket operations.
 	MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error

cmd/object-api-options.go

@@ -133,6 +133,121 @@ func getOpts(ctx context.Context, r *http.Request, bucket, object string) (Objec
 	return opts, nil
 }
 
+func getAndValidateAttributesOpts(ctx context.Context, w http.ResponseWriter, r *http.Request, bucket, object string) (opts ObjectOptions, valid bool) {
+	var argumentName string
+	var argumentValue string
+	var apiErr APIError
+	var err error
+	valid = true
+
+	defer func() {
+		if valid {
+			return
+		}
+
+		errResp := objectAttributesErrorResponse{
+			ArgumentName:  &argumentName,
+			ArgumentValue: &argumentValue,
+			APIErrorResponse: getAPIErrorResponse(
+				ctx,
+				apiErr,
+				r.URL.Path,
+				w.Header().Get(xhttp.AmzRequestID),
+				w.Header().Get(xhttp.AmzRequestHostID),
+			),
+		}
+
+		writeResponse(w, apiErr.HTTPStatusCode, encodeResponse(errResp), mimeXML)
+	}()
+
+	opts, err = getOpts(ctx, r, bucket, object)
+	if err != nil {
+		switch vErr := err.(type) {
+		case InvalidVersionID:
+			apiErr = toAPIError(ctx, vErr)
+			argumentName = strings.ToLower("versionId")
+			argumentValue = vErr.VersionID
+		default:
+			apiErr = toAPIError(ctx, vErr)
+		}
+		valid = false
+		return
+	}
+
+	opts.MaxParts, err = parseIntHeader(bucket, object, r.Header, xhttp.AmzMaxParts)
+	if err != nil {
+		apiErr = toAPIError(ctx, err)
+		argumentName = strings.ToLower(xhttp.AmzMaxParts)
+		valid = false
+		return
+	}
+
+	if opts.MaxParts == 0 {
+		opts.MaxParts = maxPartsList
+	}
+
+	opts.PartNumberMarker, err = parseIntHeader(bucket, object, r.Header, xhttp.AmzPartNumberMarker)
+	if err != nil {
+		apiErr = toAPIError(ctx, err)
+		argumentName = strings.ToLower(xhttp.AmzPartNumberMarker)
+		valid = false
+		return
+	}
+
+	opts.ObjectAttributes = parseObjectAttributes(r.Header)
+	if len(opts.ObjectAttributes) < 1 {
+		apiErr = errorCodes.ToAPIErr(ErrInvalidAttributeName)
+		argumentName = strings.ToLower(xhttp.AmzObjectAttributes)
+		valid = false
+		return
+	}
+
+	for tag := range opts.ObjectAttributes {
+		switch tag {
+		case xhttp.ETag:
+		case xhttp.Checksum:
+		case xhttp.StorageClass:
+		case xhttp.ObjectSize:
+		case xhttp.ObjectParts:
+		default:
+			apiErr = errorCodes.ToAPIErr(ErrInvalidAttributeName)
+			argumentName = strings.ToLower(xhttp.AmzObjectAttributes)
+			argumentValue = tag
+			valid = false
+			return
+		}
+	}
+
+	return
+}
+
+func parseObjectAttributes(h http.Header) (attributes map[string]struct{}) {
+	attributes = make(map[string]struct{})
+	for _, v := range strings.Split(strings.TrimSpace(h.Get(xhttp.AmzObjectAttributes)), ",") {
+		if v != "" {
+			attributes[v] = struct{}{}
+		}
+	}
+
+	return
+}
+
+func parseIntHeader(bucket, object string, h http.Header, headerName string) (value int, err error) {
+	stringInt := strings.TrimSpace(h.Get(headerName))
+	if stringInt == "" {
+		return
+	}
+	value, err = strconv.Atoi(stringInt)
+	if err != nil {
+		return 0, InvalidArgument{
+			Bucket: bucket,
+			Object: object,
+			Err:    fmt.Errorf("Unable to parse %s, value should be an integer", headerName),
+		}
+	}
+	return
+}
+
 func parseBoolHeader(bucket, object string, h http.Header, headerName string) (bool, error) {
 	value := strings.TrimSpace(h.Get(headerName))
 	if value != "" {

cmd/object-api-utils.go

@@ -533,6 +533,14 @@ func (o ObjectInfo) GetActualSize() (int64, error) {
 		return size, nil
 	}
 	if _, ok := crypto.IsEncrypted(o.UserDefined); ok {
+		sizeStr, ok := o.UserDefined[ReservedMetadataPrefix+"actual-size"]
+		if ok {
+			size, err := strconv.ParseInt(sizeStr, 10, 64)
+			if err != nil {
+				return -1, errObjectTampered
+			}
+			return size, nil
+		}
 		return o.DecryptedSize()
 	}
 
@@ -632,16 +640,14 @@ func getCompressedOffsets(oi ObjectInfo, offset int64, decrypt func([]byte) ([]b
 	var skipLength int64
 	var cumulativeActualSize int64
 	var firstPartIdx int
-	if len(oi.Parts) > 0 {
-		for i, part := range oi.Parts {
-			cumulativeActualSize += part.ActualSize
-			if cumulativeActualSize <= offset {
-				compressedOffset += part.Size
-			} else {
-				firstPartIdx = i
-				skipLength = cumulativeActualSize - part.ActualSize
-				break
-			}
+	for i, part := range oi.Parts {
+		cumulativeActualSize += part.ActualSize
+		if cumulativeActualSize <= offset {
+			compressedOffset += part.Size
+		} else {
+			firstPartIdx = i
+			skipLength = cumulativeActualSize - part.ActualSize
+			break
 		}
 	}
 	partSkip = offset - skipLength

cmd/object-handlers.go

@@ -689,6 +689,145 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 	})
 }
 
+// GetObjectAttributes ...
+func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, objectAPI ObjectLayer, bucket, object string, w http.ResponseWriter, r *http.Request) {
+	opts, valid := getAndValidateAttributesOpts(ctx, w, r, bucket, object)
+	if !valid {
+		return
+	}
+
+	var s3Error APIErrorCode
+	if opts.VersionID != "" {
+		s3Error = checkRequestAuthType(ctx, r, policy.GetObjectVersionAttributesAction, bucket, object)
+		if s3Error == ErrNone {
+			s3Error = checkRequestAuthType(ctx, r, policy.GetObjectVersionAction, bucket, object)
+		}
+	} else {
+		s3Error = checkRequestAuthType(ctx, r, policy.GetObjectAttributesAction, bucket, object)
+		if s3Error == ErrNone {
+			s3Error = checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object)
+		}
+	}
+
+	if s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+
+	objInfo, err := objectAPI.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		s3Error = checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, object)
+		if s3Error == ErrNone {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
+	if _, err = DecryptObjectInfo(&objInfo, r); err != nil {
+		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
+		return
+	}
+
+	if checkPreconditions(ctx, w, r, objInfo, opts) {
+		return
+	}
+
+	OA := new(getObjectAttributesResponse)
+
+	if opts.Versioned {
+		w.Header().Set(xhttp.AmzVersionID, objInfo.VersionID)
+	}
+
+	lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat)
+	w.Header().Set(xhttp.LastModified, lastModified)
+	w.Header().Del(xhttp.ContentType)
+
+	if _, ok := opts.ObjectAttributes[xhttp.Checksum]; ok {
+		chkSums := objInfo.decryptChecksums(0)
+		// AWS does not appear to append part number on this API call.
+		switch {
+		case chkSums["CRC32"] != "":
+			OA.Checksum = new(objectAttributesChecksum)
+			OA.Checksum.ChecksumCRC32 = strings.Split(chkSums["CRC32"], "-")[0]
+		case chkSums["CRC32C"] != "":
+			OA.Checksum = new(objectAttributesChecksum)
+			OA.Checksum.ChecksumCRC32C = strings.Split(chkSums["CRC32C"], "-")[0]
+		case chkSums["SHA256"] != "":
+			OA.Checksum = new(objectAttributesChecksum)
+			OA.Checksum.ChecksumSHA1 = strings.Split(chkSums["SHA1"], "-")[0]
+		case chkSums["SHA1"] != "":
+			OA.Checksum = new(objectAttributesChecksum)
+			OA.Checksum.ChecksumSHA256 = strings.Split(chkSums["SHA256"], "-")[0]
+		}
+
+	}
+
+	if _, ok := opts.ObjectAttributes[xhttp.ETag]; ok {
+		OA.ETag = objInfo.ETag
+	}
+
+	if _, ok := opts.ObjectAttributes[xhttp.ObjectSize]; ok {
+		OA.ObjectSize, _ = objInfo.GetActualSize()
+	}
+
+	if _, ok := opts.ObjectAttributes[xhttp.StorageClass]; ok {
+		OA.StorageClass = objInfo.StorageClass
+	}
+
+	objInfo.decryptPartsChecksums()
+
+	if _, ok := opts.ObjectAttributes[xhttp.ObjectParts]; ok {
+		OA.ObjectParts = new(objectAttributesParts)
+		OA.ObjectParts.PartNumberMarker = opts.PartNumberMarker
+
+		OA.ObjectParts.MaxParts = opts.MaxParts
+		partsLength := len(objInfo.Parts)
+		OA.ObjectParts.PartsCount = partsLength
+
+		if opts.MaxParts > -1 {
+			for i, v := range objInfo.Parts {
+				if v.Number <= opts.PartNumberMarker {
+					continue
+				}
+
+				if len(OA.ObjectParts.Parts) == opts.MaxParts {
+					break
+				}
+
+				OA.ObjectParts.NextPartNumberMarker = v.Number
+				OA.ObjectParts.Parts = append(OA.ObjectParts.Parts, &objectAttributesPart{
+					ChecksumSHA1:   objInfo.Parts[i].Checksums["SHA1"],
+					ChecksumSHA256: objInfo.Parts[i].Checksums["SHA256"],
+					ChecksumCRC32:  objInfo.Parts[i].Checksums["CRC32"],
+					ChecksumCRC32C: objInfo.Parts[i].Checksums["CRC32C"],
+					PartNumber:     objInfo.Parts[i].Number,
+					Size:           objInfo.Parts[i].Size,
+				})
+			}
+		}
+
+		if OA.ObjectParts.NextPartNumberMarker != partsLength {
+			OA.ObjectParts.IsTruncated = true
+		}
+	}
+
+	writeSuccessResponseXML(w, encodeResponse(OA))
+
+	sendEvent(eventArgs{
+		EventName:    event.ObjectAccessedAttributes,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         handlers.GetSourceIP(r),
+	})
+
+	return
+}
+
 // GetObjectHandler - GET Object
 // ----------
 // This implementation of the GET operation retrieves object. To use GET,
@@ -1043,6 +1182,30 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 	})
 }
 
+// GetObjectAttributesHandles - GET Object
+// -----------
+// This operation retrieves metadata and part metadata from an object without returning the object itself.
+func (api objectAPIHandlers) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetObjectAttributes")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrServerNotInitialized))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object, err := unescapePath(vars["object"])
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	api.getObjectAttributesHandler(ctx, objectAPI, bucket, object, w, r)
+}
+
 // HeadObjectHandler - HEAD Object
 // -----------
 // The HEAD operation retrieves metadata from an object without returning the object itself.
@@ -1982,20 +2145,21 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		md5hex = "" // Do not try to verify the content.
 		sha256hex = ""
 	}
-	var hashReader *hash.Reader
-	// Optimization: If SSE-KMS and SSE-C did not request Content-Md5. Use uuid as etag
-	if !etag.ContentMD5Requested(r.Header) && (crypto.S3KMS.IsRequested(r.Header) || crypto.SSEC.IsRequested(r.Header)) {
-		hashReader, err = hash.NewReaderWithOpts(ctx, reader, hash.Options{
-			Size:       size,
-			MD5Hex:     md5hex,
-			SHA256Hex:  sha256hex,
-			ActualSize: actualSize,
-			DisableMD5: false,
-			ForceMD5:   mustGetUUIDBytes(),
-		})
-	} else {
-		hashReader, err = hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
+
+	var forceMD5 []byte
+	// Optimization: If SSE-KMS and SSE-C did not request Content-Md5. Use uuid as etag. Optionally enable this also
+	// for server that is started with `--no-compat`.
+	if !etag.ContentMD5Requested(r.Header) && (crypto.S3KMS.IsRequested(r.Header) || crypto.SSEC.IsRequested(r.Header) || !globalServerCtxt.StrictS3Compat) {
+		forceMD5 = mustGetUUIDBytes()
 	}
+	hashReader, err := hash.NewReaderWithOpts(ctx, reader, hash.Options{
+		Size:       size,
+		MD5Hex:     md5hex,
+		SHA256Hex:  sha256hex,
+		ActualSize: actualSize,
+		DisableMD5: false,
+		ForceMD5:   forceMD5,
+	})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -3140,8 +3304,28 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
 
 	ot, err := objAPI.GetObjectTags(ctx, bucket, object, opts)
 	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
+		// if object/version is not found locally, but exists on peer site - proxy
+		// the tagging request to peer site. The response to client will
+		// return tags from peer site.
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			proxytgts := getProxyTargets(ctx, bucket, object, opts)
+			if !proxytgts.Empty() {
+				// proxy to replication target if site replication is in place.
+				tags, gerr := proxyGetTaggingToRepTarget(ctx, bucket, object, opts, proxytgts)
+				if gerr.Err != nil {
+					writeErrorResponse(ctx, w, toAPIError(ctx, gerr.Err), r.URL)
+					return
+				} // overlay tags from peer site.
+				ot = tags
+				w.Header()[xhttp.MinIOTaggingProxied] = []string{"true"} // indicate that the request was proxied.
+			} else {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+				return
+			}
+		} else {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
 	}
 
 	// Set this such that authorization policies can be applied on the object tags.
@@ -3221,6 +3405,33 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
 
 	objInfo, err := objAPI.GetObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
+		// if object is not found locally, but exists on peer site - proxy
+		// the tagging request to peer site. The response to client will
+		// be 200 with extra header indicating that the request was proxied.
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			proxytgts := getProxyTargets(ctx, bucket, object, opts)
+			if !proxytgts.Empty() {
+				// proxy to replication target if site replication is in place.
+				perr := proxyTaggingToRepTarget(ctx, bucket, object, tags, opts, proxytgts)
+				if perr.Err != nil {
+					writeErrorResponse(ctx, w, toAPIError(ctx, perr.Err), r.URL)
+					return
+				}
+				w.Header()[xhttp.MinIOTaggingProxied] = []string{"true"}
+				writeSuccessResponseHeadersOnly(w)
+				// when tagging is proxied, the object version is not available to return
+				// as header in the response, or ObjectInfo in the notification event.
+				sendEvent(eventArgs{
+					EventName:    event.ObjectCreatedPutTagging,
+					BucketName:   bucket,
+					ReqParams:    extractReqParams(r),
+					RespElements: extractRespElements(w),
+					UserAgent:    r.UserAgent(),
+					Host:         handlers.GetSourceIP(r),
+				})
+				return
+			}
+		}
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -3289,6 +3500,34 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
 
 	oi, err := objAPI.GetObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
+		// if object is not found locally, but exists on peer site - proxy
+		// the tagging request to peer site. The response to client will
+		// be 200 OK with extra header indicating that the request was proxied.
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			proxytgts := getProxyTargets(ctx, bucket, object, opts)
+			if !proxytgts.Empty() {
+				// proxy to replication target if active-active replication is in place.
+				perr := proxyTaggingToRepTarget(ctx, bucket, object, nil, opts, proxytgts)
+				if perr.Err != nil {
+					writeErrorResponse(ctx, w, toAPIError(ctx, perr.Err), r.URL)
+					return
+				}
+				// when delete tagging is proxied, the object version/tags are not available to return
+				// as header in the response, nor ObjectInfo in the notification event.
+				w.Header()[xhttp.MinIOTaggingProxied] = []string{"true"}
+				writeSuccessNoContent(w)
+				sendEvent(eventArgs{
+					EventName:    event.ObjectCreatedDeleteTagging,
+					BucketName:   bucket,
+					Object:       oi,
+					ReqParams:    extractReqParams(r),
+					RespElements: extractRespElements(w),
+					UserAgent:    r.UserAgent(),
+					Host:         handlers.GetSourceIP(r),
+				})
+				return
+			}
+		}
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/object-multipart-handlers.go

@@ -728,7 +728,14 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		sha256hex = ""
 	}
 
-	hashReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
+	hashReader, err := hash.NewReaderWithOpts(ctx, reader, hash.Options{
+		Size:       size,
+		MD5Hex:     md5hex,
+		SHA256Hex:  sha256hex,
+		ActualSize: actualSize,
+		DisableMD5: false,
+		ForceMD5:   nil,
+	})
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/os-instrumented.go

@@ -39,6 +39,8 @@ const (
 	osMetricRename
 	osMetricOpenFileW
 	osMetricOpenFileR
+	osMetricOpenFileWFd
+	osMetricOpenFileRFd
 	osMetricOpen
 	osMetricOpenFileDirectIO
 	osMetricLstat

cmd/os_unix.go

@@ -39,6 +39,22 @@ func access(name string) error {
 	return nil
 }
 
+// openFileWithFD return 'fd' based file descriptor
+func openFileWithFD(name string, flag int, perm os.FileMode) (fd int, err error) {
+	switch flag & writeMode {
+	case writeMode:
+		defer updateOSMetrics(osMetricOpenFileWFd, name)(err)
+	default:
+		defer updateOSMetrics(osMetricOpenFileRFd, name)(err)
+	}
+	var e error
+	fd, e = syscall.Open(name, flag|syscall.O_CLOEXEC, uint32(perm))
+	if e != nil {
+		return -1, &os.PathError{Op: "open", Path: name, Err: e}
+	}
+	return fd, nil
+}
+
 // Forked from Golang but chooses to avoid performing lookup
 //
 // osMkdirAll creates a directory named path,
@@ -149,7 +165,7 @@ func parseDirEnt(buf []byte) (consumed int, name []byte, typ os.FileMode, err er
 // the directory itself, if the dirPath doesn't exist this function doesn't return
 // an error.
 func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) error {
-	f, err := OpenFile(dirPath, readMode, 0o666)
+	fd, err := openFileWithFD(dirPath, readMode, 0o666)
 	if err != nil {
 		if osErrToFileErr(err) == errFileNotFound {
 			return nil
@@ -160,15 +176,16 @@ func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) erro
 		// There may be permission error when dirPath
 		// is at the root of the disk mount that may
 		// not have the permissions to avoid 'noatime'
-		f, err = Open(dirPath)
+		fd, err = openFileWithFD(dirPath, os.O_RDONLY, 0o666)
 		if err != nil {
 			if osErrToFileErr(err) == errFileNotFound {
 				return nil
 			}
 			return osErrToFileErr(err)
 		}
+
 	}
-	defer f.Close()
+	defer syscall.Close(fd)
 
 	bufp := direntPool.Get().(*[]byte)
 	defer direntPool.Put(bufp)
@@ -181,7 +198,7 @@ func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) erro
 		if boff >= nbuf {
 			boff = 0
 			stop := globalOSMetrics.time(osMetricReadDirent)
-			nbuf, err = syscall.ReadDirent(int(f.Fd()), buf)
+			nbuf, err = syscall.ReadDirent(fd, buf)
 			stop()
 			if err != nil {
 				if isSysErrNotDir(err) {
@@ -241,7 +258,7 @@ func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) erro
 // Return count entries at the directory dirPath and all entries
 // if count is set to -1
 func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) {
-	f, err := OpenFile(dirPath, readMode, 0o666)
+	fd, err := openFileWithFD(dirPath, readMode, 0o666)
 	if err != nil {
 		if !osIsPermission(err) {
 			return nil, osErrToFileErr(err)
@@ -249,12 +266,12 @@ func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err er
 		// There may be permission error when dirPath
 		// is at the root of the disk mount that may
 		// not have the permissions to avoid 'noatime'
-		f, err = Open(dirPath)
+		fd, err = openFileWithFD(dirPath, os.O_RDONLY, 0o666)
 		if err != nil {
 			return nil, osErrToFileErr(err)
 		}
 	}
-	defer f.Close()
+	defer syscall.Close(fd)
 
 	bufp := direntPool.Get().(*[]byte)
 	defer direntPool.Put(bufp)
@@ -273,7 +290,7 @@ func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err er
 		if boff >= nbuf {
 			boff = 0
 			stop := globalOSMetrics.time(osMetricReadDirent)
-			nbuf, err = syscall.ReadDirent(int(f.Fd()), buf)
+			nbuf, err = syscall.ReadDirent(fd, buf)
 			stop()
 			if err != nil {
 				if isSysErrNotDir(err) {

cmd/osmetric_string.go

@@ -14,23 +14,25 @@ func _() {
 	_ = x[osMetricRename-3]
 	_ = x[osMetricOpenFileW-4]
 	_ = x[osMetricOpenFileR-5]
-	_ = x[osMetricOpen-6]
-	_ = x[osMetricOpenFileDirectIO-7]
-	_ = x[osMetricLstat-8]
-	_ = x[osMetricRemove-9]
-	_ = x[osMetricStat-10]
-	_ = x[osMetricAccess-11]
-	_ = x[osMetricCreate-12]
-	_ = x[osMetricReadDirent-13]
-	_ = x[osMetricFdatasync-14]
-	_ = x[osMetricSync-15]
-	_ = x[osMetricRename2-16]
-	_ = x[osMetricLast-17]
+	_ = x[osMetricOpenFileWFd-6]
+	_ = x[osMetricOpenFileRFd-7]
+	_ = x[osMetricOpen-8]
+	_ = x[osMetricOpenFileDirectIO-9]
+	_ = x[osMetricLstat-10]
+	_ = x[osMetricRemove-11]
+	_ = x[osMetricStat-12]
+	_ = x[osMetricAccess-13]
+	_ = x[osMetricCreate-14]
+	_ = x[osMetricReadDirent-15]
+	_ = x[osMetricFdatasync-16]
+	_ = x[osMetricSync-17]
+	_ = x[osMetricRename2-18]
+	_ = x[osMetricLast-19]
 }
 
-const _osMetric_name = "RemoveAllMkdirAllMkdirRenameOpenFileWOpenFileROpenOpenFileDirectIOLstatRemoveStatAccessCreateReadDirentFdatasyncSyncRename2Last"
+const _osMetric_name = "RemoveAllMkdirAllMkdirRenameOpenFileWOpenFileROpenFileWFdOpenFileRFdOpenOpenFileDirectIOLstatRemoveStatAccessCreateReadDirentFdatasyncSyncRename2Last"
 
-var _osMetric_index = [...]uint8{0, 9, 17, 22, 28, 37, 46, 50, 66, 71, 77, 81, 87, 93, 103, 112, 116, 123, 127}
+var _osMetric_index = [...]uint8{0, 9, 17, 22, 28, 37, 46, 57, 68, 72, 88, 93, 99, 103, 109, 115, 125, 134, 138, 145, 149}
 
 func (i osMetric) String() string {
 	if i >= osMetric(len(_osMetric_index)-1) {

cmd/peer-rest-client.go

@@ -102,8 +102,10 @@ func (client *peerRESTClient) GetLocks() (lockMap map[string][]lockRequesterInfo
 }
 
 // LocalStorageInfo - fetch server information for a remote node.
-func (client *peerRESTClient) LocalStorageInfo() (info StorageInfo, err error) {
-	respBody, err := client.call(peerRESTMethodLocalStorageInfo, nil, nil, -1)
+func (client *peerRESTClient) LocalStorageInfo(metrics bool) (info StorageInfo, err error) {
+	values := make(url.Values)
+	values.Set(peerRESTMetrics, strconv.FormatBool(metrics))
+	respBody, err := client.call(peerRESTMethodLocalStorageInfo, values, nil, -1)
 	if err != nil {
 		return
 	}
@@ -919,6 +921,7 @@ func (client *peerRESTClient) SpeedTest(ctx context.Context, opts speedTestOpts)
 	values.Set(peerRESTDuration, opts.duration.String())
 	values.Set(peerRESTStorageClass, opts.storageClass)
 	values.Set(peerRESTBucket, opts.bucketName)
+	values.Set(peerRESTEnableSha256, strconv.FormatBool(opts.enableSha256))
 
 	respBody, err := client.callWithContext(context.Background(), peerRESTMethodSpeedTest, values, nil, -1)
 	if err != nil {

cmd/peer-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	peerRESTVersion = "v33" // Add SRMetrics
+	peerRESTVersion = "v34" // Add metrics flag to LocalStorageInfo call
 
 	peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
 	peerRESTPrefix        = minioReservedBucketPath + "/peer"
@@ -100,12 +100,14 @@ const (
 	peerRESTConcurrent     = "concurrent"
 	peerRESTDuration       = "duration"
 	peerRESTStorageClass   = "storage-class"
+	peerRESTEnableSha256   = "enableSha256"
 	peerRESTMetricsTypes   = "types"
 	peerRESTDisk           = "disk"
 	peerRESTHost           = "host"
 	peerRESTJobID          = "job-id"
 	peerRESTDepID          = "depID"
 	peerRESTStartRebalance = "start-rebalance"
+	peerRESTMetrics        = "metrics"
 
 	peerRESTListenBucket = "bucket"
 	peerRESTListenPrefix = "prefix"

cmd/peer-rest-server.go

@@ -345,7 +345,14 @@ func (s *peerRESTServer) LocalStorageInfoHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
-	logger.LogIf(ctx, gob.NewEncoder(w).Encode(objLayer.LocalStorageInfo(r.Context())))
+	metrics, err := strconv.ParseBool(r.Form.Get(peerRESTMetrics))
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+
+	}
+
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(objLayer.LocalStorageInfo(r.Context(), metrics)))
 }
 
 // ServerInfoHandler - returns Server Info
@@ -1297,6 +1304,7 @@ func (s *peerRESTServer) SpeedTestHandler(w http.ResponseWriter, r *http.Request
 	concurrentStr := r.Form.Get(peerRESTConcurrent)
 	storageClass := r.Form.Get(peerRESTStorageClass)
 	bucketName := r.Form.Get(peerRESTBucket)
+	enableSha256 := r.Form.Get(peerRESTEnableSha256) == "true"
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -1321,6 +1329,7 @@ func (s *peerRESTServer) SpeedTestHandler(w http.ResponseWriter, r *http.Request
 		duration:     duration,
 		storageClass: storageClass,
 		bucketName:   bucketName,
+		enableSha256: enableSha256,
 	})
 	if err != nil {
 		result.Error = err.Error()

cmd/peer-s3-client.go

@@ -25,7 +25,9 @@ import (
 	"net/url"
 	"sort"
 	"strconv"
+	"time"
 
+	"github.com/minio/madmin-go/v3"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/rest"
 	"github.com/minio/pkg/v2/sync/errgroup"
@@ -36,6 +38,7 @@ var errPeerOffline = errors.New("peer is offline")
 
 type peerS3Client interface {
 	ListBuckets(ctx context.Context, opts BucketOptions) ([]BucketInfo, error)
+	HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error)
 	GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error)
 	MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error
 	DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error
@@ -67,6 +70,10 @@ func (l localPeerS3Client) ListBuckets(ctx context.Context, opts BucketOptions)
 	return listBucketsLocal(ctx, opts)
 }
 
+func (l localPeerS3Client) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
+	return healBucketLocal(ctx, bucket, opts)
+}
+
 func (l localPeerS3Client) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) {
 	return getBucketInfoLocal(ctx, bucket, opts)
 }
@@ -124,6 +131,51 @@ func NewS3PeerSys(endpoints EndpointServerPools) *S3PeerSys {
 	}
 }
 
+// HealBucket - heals buckets at node level
+func (sys *S3PeerSys) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
+	g := errgroup.WithNErrs(len(sys.peerClients))
+
+	healBucketResults := make([]madmin.HealResultItem, len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		idx := idx
+		client := client
+		g.Go(func() error {
+			if client == nil {
+				return errPeerOffline
+			}
+			res, err := client.HealBucket(ctx, bucket, opts)
+			if err != nil {
+				return err
+			}
+			healBucketResults[idx] = res
+			return nil
+		}, idx)
+	}
+
+	errs := g.Wait()
+
+	for poolIdx := 0; poolIdx < sys.poolsCount; poolIdx++ {
+		perPoolErrs := make([]error, 0, len(sys.peerClients))
+		for i, client := range sys.peerClients {
+			if slices.Contains(client.GetPools(), poolIdx) {
+				perPoolErrs = append(perPoolErrs, errs[i])
+			}
+		}
+		quorum := len(perPoolErrs) / 2
+		if poolErr := reduceWriteQuorumErrs(ctx, perPoolErrs, bucketOpIgnoredErrs, quorum); poolErr != nil {
+			return madmin.HealResultItem{}, poolErr
+		}
+	}
+
+	for i, err := range errs {
+		if err == nil {
+			return healBucketResults[i], nil
+		}
+	}
+
+	return madmin.HealResultItem{}, toObjectErr(errVolumeNotFound, bucket)
+}
+
 // ListBuckets lists buckets across all nodes and returns a consistent view:
 //   - Return an error when a pool cannot return N/2+1 valid bucket information
 //   - For each pool, check if the bucket exists in N/2+1 nodes before including it in the final result
@@ -185,6 +237,22 @@ func (sys *S3PeerSys) ListBuckets(ctx context.Context, opts BucketOptions) ([]Bu
 				}
 			}
 		}
+		// loop through buckets and see if some with lost quorum
+		// these could be stale buckets lying around, queue a heal
+		// of such a bucket. This is needed here as we identify such
+		// buckets here while listing buckets. As part of regular
+		// globalBucketMetadataSys.Init() call would get a valid
+		// buckets only and not the quourum lost ones like this, so
+		// explicit call
+		for bktName, count := range bucketsMap {
+			if count < quorum {
+				// Queue a bucket heal task
+				globalMRFState.addPartialOp(partialOperation{
+					bucket: bktName,
+					queued: time.Now(),
+				})
+			}
+		}
 	}
 
 	result := make([]BucketInfo, 0, len(resultMap))
@@ -259,6 +327,23 @@ func (client *remotePeerS3Client) ListBuckets(ctx context.Context, opts BucketOp
 	return buckets, err
 }
 
+func (client *remotePeerS3Client) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
+	v := url.Values{}
+	v.Set(peerS3Bucket, bucket)
+	v.Set(peerS3BucketDeleted, strconv.FormatBool(opts.Remove))
+
+	respBody, err := client.call(peerS3MethodHealBucket, v, nil, -1)
+	if err != nil {
+		return madmin.HealResultItem{}, err
+	}
+	defer xhttp.DrainBody(respBody)
+
+	var res madmin.HealResultItem
+	err = gob.NewDecoder(respBody).Decode(&res)
+
+	return res, err
+}
+
 // GetBucketInfo returns bucket stat info from a peer
 func (client *remotePeerS3Client) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) {
 	v := url.Values{}

cmd/peer-s3-server.go

@@ -21,8 +21,10 @@ import (
 	"context"
 	"encoding/gob"
 	"errors"
+	"fmt"
 	"net/http"
 
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/sync/errgroup"
@@ -42,6 +44,7 @@ const (
 	peerS3MethodGetBucketInfo = "/get-bucket-info"
 	peerS3MethodDeleteBucket  = "/delete-bucket"
 	peerS3MethodListBuckets   = "/list-buckets"
+	peerS3MethodHealBucket    = "/heal-bucket"
 )
 
 const (
@@ -78,18 +81,146 @@ func (s *peerS3Server) HealthHandler(w http.ResponseWriter, r *http.Request) {
 	s.IsValid(w, r)
 }
 
-func listBucketsLocal(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
+func healBucketLocal(ctx context.Context, bucket string, opts madmin.HealOpts) (res madmin.HealResultItem, err error) {
 	globalLocalDrivesMu.RLock()
 	globalLocalDrives := globalLocalDrives
 	globalLocalDrivesMu.RUnlock()
 
-	quorum := (len(globalLocalDrives) / 2)
+	// Initialize sync waitgroup.
+	g := errgroup.WithNErrs(len(globalLocalDrives))
+
+	// Disk states slices
+	beforeState := make([]string, len(globalLocalDrives))
+	afterState := make([]string, len(globalLocalDrives))
+
+	// Make a volume entry on all underlying storage disks.
+	for index := range globalLocalDrives {
+		index := index
+		g.Go(func() (serr error) {
+			if globalLocalDrives[index] == nil {
+				beforeState[index] = madmin.DriveStateOffline
+				afterState[index] = madmin.DriveStateOffline
+				return errDiskNotFound
+			}
+
+			beforeState[index] = madmin.DriveStateOk
+			afterState[index] = madmin.DriveStateOk
+
+			if bucket == minioReservedBucket {
+				return nil
+			}
+
+			_, serr = globalLocalDrives[index].StatVol(ctx, bucket)
+			if serr != nil {
+				if serr == errDiskNotFound {
+					beforeState[index] = madmin.DriveStateOffline
+					afterState[index] = madmin.DriveStateOffline
+					return serr
+				}
+				if serr != errVolumeNotFound {
+					beforeState[index] = madmin.DriveStateCorrupt
+					afterState[index] = madmin.DriveStateCorrupt
+					return serr
+				}
+
+				beforeState[index] = madmin.DriveStateMissing
+				afterState[index] = madmin.DriveStateMissing
+
+				return serr
+			}
+			return nil
+		}, index)
+	}
+
+	errs := g.Wait()
+
+	// Initialize heal result info
+	res = madmin.HealResultItem{
+		Type:      madmin.HealItemBucket,
+		Bucket:    bucket,
+		DiskCount: len(globalLocalDrives),
+		SetCount:  -1, // explicitly set an invalid value -1, for bucket heal scenario
+	}
+
+	// mutate only if not a dry-run
+	if opts.DryRun {
+		return res, nil
+	}
+
+	for i := range beforeState {
+		res.Before.Drives = append(res.Before.Drives, madmin.HealDriveInfo{
+			UUID:     "",
+			Endpoint: globalLocalDrives[i].String(),
+			State:    beforeState[i],
+		})
+	}
+
+	// check dangling and delete bucket only if its not a meta bucket
+	if !isMinioMetaBucketName(bucket) && !isAllBucketsNotFound(errs) && opts.Remove {
+		g := errgroup.WithNErrs(len(globalLocalDrives))
+		for index := range globalLocalDrives {
+			index := index
+			g.Go(func() error {
+				if globalLocalDrives[index] == nil {
+					return errDiskNotFound
+				}
+				err := globalLocalDrives[index].DeleteVol(ctx, bucket, false)
+				if errors.Is(err, errVolumeNotEmpty) {
+					logger.LogOnceIf(ctx, fmt.Errorf("While deleting dangling Bucket (%s), Drive %s:%s returned an error (%w)",
+						bucket, globalLocalDrives[index].Hostname(), globalLocalDrives[index], err), "delete-dangling-bucket-"+bucket)
+				}
+				return nil
+			}, index)
+		}
+
+		g.Wait()
+	}
+
+	// Create the quorum lost volume only if its nor makred for delete
+	if !opts.Remove {
+		// Initialize sync waitgroup.
+		g = errgroup.WithNErrs(len(globalLocalDrives))
+
+		// Make a volume entry on all underlying storage disks.
+		for index := range globalLocalDrives {
+			index := index
+			g.Go(func() error {
+				if beforeState[index] == madmin.DriveStateMissing {
+					makeErr := globalLocalDrives[index].MakeVol(ctx, bucket)
+					if makeErr == nil {
+						afterState[index] = madmin.DriveStateOk
+					}
+					return makeErr
+				}
+				return errs[index]
+			}, index)
+		}
+
+		errs = g.Wait()
+	}
+
+	for i := range afterState {
+		res.After.Drives = append(res.After.Drives, madmin.HealDriveInfo{
+			UUID:     "",
+			Endpoint: globalLocalDrives[i].String(),
+			State:    afterState[i],
+		})
+	}
+	return res, nil
+}
+
+func listBucketsLocal(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
+	globalLocalDrivesMu.RLock()
+	localDrives := globalLocalDrives
+	globalLocalDrivesMu.RUnlock()
+
+	quorum := (len(localDrives) / 2)
 
 	buckets = make([]BucketInfo, 0, 32)
 	healBuckets := map[string]VolInfo{}
 
 	// lists all unique buckets across drives.
-	if err := listAllBuckets(ctx, globalLocalDrives, healBuckets, quorum); err != nil {
+	if err := listAllBuckets(ctx, localDrives, healBuckets, quorum); err != nil {
 		return nil, err
 	}
 
@@ -98,7 +229,7 @@ func listBucketsLocal(ctx context.Context, opts BucketOptions) (buckets []Bucket
 
 	if opts.Deleted {
 		// lists all deleted buckets across drives.
-		if err := listDeletedBuckets(ctx, globalLocalDrives, deletedBuckets, quorum); err != nil {
+		if err := listDeletedBuckets(ctx, localDrives, deletedBuckets, quorum); err != nil {
 			return nil, err
 		}
 	}
@@ -128,23 +259,23 @@ func listBucketsLocal(ctx context.Context, opts BucketOptions) (buckets []Bucket
 
 func getBucketInfoLocal(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) {
 	globalLocalDrivesMu.RLock()
-	globalLocalDrives := globalLocalDrives
+	localDrives := globalLocalDrives
 	globalLocalDrivesMu.RUnlock()
 
-	g := errgroup.WithNErrs(len(globalLocalDrives)).WithConcurrency(32)
-	bucketsInfo := make([]BucketInfo, len(globalLocalDrives))
+	g := errgroup.WithNErrs(len(localDrives)).WithConcurrency(32)
+	bucketsInfo := make([]BucketInfo, len(localDrives))
 
 	// Make a volume entry on all underlying storage disks.
-	for index := range globalLocalDrives {
+	for index := range localDrives {
 		index := index
 		g.Go(func() error {
-			if globalLocalDrives[index] == nil {
+			if localDrives[index] == nil {
 				return errDiskNotFound
 			}
-			volInfo, err := globalLocalDrives[index].StatVol(ctx, bucket)
+			volInfo, err := localDrives[index].StatVol(ctx, bucket)
 			if err != nil {
 				if opts.Deleted {
-					dvi, derr := globalLocalDrives[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucket))
+					dvi, derr := localDrives[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucket))
 					if derr != nil {
 						return err
 					}
@@ -160,7 +291,7 @@ func getBucketInfoLocal(ctx context.Context, bucket string, opts BucketOptions)
 	}
 
 	errs := g.Wait()
-	if err := reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, (len(globalLocalDrives) / 2)); err != nil {
+	if err := reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, (len(localDrives) / 2)); err != nil {
 		return BucketInfo{}, err
 	}
 
@@ -177,19 +308,19 @@ func getBucketInfoLocal(ctx context.Context, bucket string, opts BucketOptions)
 
 func deleteBucketLocal(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
 	globalLocalDrivesMu.RLock()
-	globalLocalDrives := globalLocalDrives
+	localDrives := globalLocalDrives
 	globalLocalDrivesMu.RUnlock()
 
-	g := errgroup.WithNErrs(len(globalLocalDrives)).WithConcurrency(32)
+	g := errgroup.WithNErrs(len(localDrives)).WithConcurrency(32)
 
 	// Make a volume entry on all underlying storage disks.
-	for index := range globalLocalDrives {
+	for index := range localDrives {
 		index := index
 		g.Go(func() error {
-			if globalLocalDrives[index] == nil {
+			if localDrives[index] == nil {
 				return errDiskNotFound
 			}
-			return globalLocalDrives[index].DeleteVol(ctx, bucket, opts.Force)
+			return localDrives[index].DeleteVol(ctx, bucket, opts.Force)
 		}, index)
 	}
 
@@ -201,7 +332,7 @@ func deleteBucketLocal(ctx context.Context, bucket string, opts DeleteBucketOpti
 		}
 		if err == nil && recreate {
 			// ignore any errors
-			globalLocalDrives[index].MakeVol(ctx, bucket)
+			localDrives[index].MakeVol(ctx, bucket)
 		}
 	}
 
@@ -210,24 +341,24 @@ func deleteBucketLocal(ctx context.Context, bucket string, opts DeleteBucketOpti
 		return errVolumeNotEmpty
 	} // for all other errors reduce by write quorum.
 
-	return reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, (len(globalLocalDrives)/2)+1)
+	return reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, (len(localDrives)/2)+1)
 }
 
 func makeBucketLocal(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	globalLocalDrivesMu.RLock()
-	globalLocalDrives := globalLocalDrives
+	localDrives := globalLocalDrives
 	globalLocalDrivesMu.RUnlock()
 
-	g := errgroup.WithNErrs(len(globalLocalDrives)).WithConcurrency(32)
+	g := errgroup.WithNErrs(len(localDrives)).WithConcurrency(32)
 
 	// Make a volume entry on all underlying storage disks.
-	for index := range globalLocalDrives {
+	for index := range localDrives {
 		index := index
 		g.Go(func() error {
-			if globalLocalDrives[index] == nil {
+			if localDrives[index] == nil {
 				return errDiskNotFound
 			}
-			err := globalLocalDrives[index].MakeVol(ctx, bucket)
+			err := localDrives[index].MakeVol(ctx, bucket)
 			if opts.ForceCreate && errors.Is(err, errVolumeExists) {
 				// No need to return error when force create was
 				// requested.
@@ -238,7 +369,7 @@ func makeBucketLocal(ctx context.Context, bucket string, opts MakeBucketOptions)
 	}
 
 	errs := g.Wait()
-	return reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, (len(globalLocalDrives)/2)+1)
+	return reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, (len(localDrives)/2)+1)
 }
 
 func (s *peerS3Server) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
@@ -259,6 +390,30 @@ func (s *peerS3Server) ListBucketsHandler(w http.ResponseWriter, r *http.Request
 	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(buckets))
 }
 
+func (s *peerS3Server) HealBucketHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+
+	bucketDeleted := r.Form.Get(peerS3BucketDeleted) == "true"
+
+	bucket := r.Form.Get(peerS3Bucket)
+	if isMinioMetaBucket(bucket) {
+		s.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	res, err := healBucketLocal(r.Context(), bucket, madmin.HealOpts{
+		Remove: bucketDeleted,
+	})
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(res))
+}
+
 // GetBucketInfoHandler implements peer BuckeInfo call, returns bucket create date.
 func (s *peerS3Server) GetBucketInfoHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -333,4 +488,5 @@ func registerPeerS3Handlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodDeleteBucket).HandlerFunc(h(server.DeleteBucketHandler))
 	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodGetBucketInfo).HandlerFunc(h(server.GetBucketInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodListBuckets).HandlerFunc(h(server.ListBucketsHandler))
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodHealBucket).HandlerFunc(h(server.HealBucketHandler))
 }

cmd/perf-tests.go

@@ -95,7 +95,7 @@ func selfSpeedTest(ctx context.Context, opts speedTestOpts) (SpeedTestResult, er
 	userMetadata[globalObjectPerfUserMetadata] = "true" // Bypass S3 API freeze
 	popts := minio.PutObjectOptions{
 		UserMetadata:         userMetadata,
-		DisableContentSha256: true,
+		DisableContentSha256: !opts.enableSha256,
 		DisableMultipart:     true,
 	}
 

cmd/rebalance-admin.go

@@ -55,7 +55,7 @@ func rebalanceStatus(ctx context.Context, z *erasureServerPools) (r rebalanceAdm
 	}
 
 	// Compute disk usage percentage
-	si := z.StorageInfo(ctx)
+	si := z.StorageInfo(ctx, true)
 	diskStats := make([]struct {
 		AvailableSpace uint64
 		TotalSpace     uint64

cmd/server-main.go

@@ -51,10 +51,16 @@ import (
 	"github.com/minio/pkg/v2/certs"
 	"github.com/minio/pkg/v2/env"
 	"golang.org/x/exp/slices"
+	"gopkg.in/yaml.v2"
 )
 
 // ServerFlags - server command specific flags
 var ServerFlags = []cli.Flag{
+	cli.StringFlag{
+		Name:   "config",
+		Usage:  "specify server configuration via YAML configuration",
+		EnvVar: "MINIO_CONFIG",
+	},
 	cli.StringFlag{
 		Name:   "address",
 		Value:  ":" + GlobalMinioDefaultPort,
@@ -68,6 +74,12 @@ var ServerFlags = []cli.Flag{
 		EnvVar: "MINIO_LISTENERS",
 		Hidden: true,
 	},
+	cli.BoolFlag{
+		Name:   "pre-allocate",
+		Usage:  "Number of 1MiB sized buffers to pre-allocate. Default 2048",
+		EnvVar: "MINIO_PRE_ALLOCATE",
+		Hidden: true,
+	},
 	cli.StringFlag{
 		Name:   "console-address",
 		Usage:  "bind to a specific ADDRESS:PORT for embedded Console UI, ADDRESS can be an IP or hostname",
@@ -226,9 +238,57 @@ func serverCmdArgs(ctx *cli.Context) []string {
 	return strings.Fields(v)
 }
 
-func serverHandleCmdArgs(ctx *cli.Context) {
-	// Handle common command args.
-	handleCommonCmdArgs(ctx)
+func mergeServerCtxtFromConfigFile(configFile string, ctxt *serverCtxt) error {
+	rd, err := Open(configFile)
+	if err != nil {
+		return err
+	}
+	defer rd.Close()
+
+	cf := &config.ServerConfig{}
+	dec := yaml.NewDecoder(rd)
+	dec.SetStrict(true)
+	if err = dec.Decode(cf); err != nil {
+		return err
+	}
+	if cf.Version != "v1" {
+		return fmt.Errorf("unexpected version: %s", cf.Version)
+	}
+
+	ctxt.RootUser = cf.RootUser
+	ctxt.RootPwd = cf.RootPwd
+
+	if cf.Addr != "" {
+		ctxt.Addr = cf.Addr
+	}
+	if cf.ConsoleAddr != "" {
+		ctxt.ConsoleAddr = cf.ConsoleAddr
+	}
+	if cf.CertsDir != "" {
+		ctxt.CertsDir = cf.CertsDir
+		ctxt.certsDirSet = true
+	}
+
+	if cf.Options.FTP.Address != "" {
+		ctxt.FTP = append(ctxt.FTP, fmt.Sprintf("address=%s", cf.Options.FTP.Address))
+	}
+	if cf.Options.FTP.PassivePortRange != "" {
+		ctxt.FTP = append(ctxt.FTP, fmt.Sprintf("passive-port-range=%s", cf.Options.FTP.PassivePortRange))
+	}
+
+	if cf.Options.SFTP.Address != "" {
+		ctxt.SFTP = append(ctxt.SFTP, fmt.Sprintf("address=%s", cf.Options.SFTP.Address))
+	}
+	if cf.Options.SFTP.SSHPrivateKey != "" {
+		ctxt.SFTP = append(ctxt.SFTP, fmt.Sprintf("ssh-private-key=%s", cf.Options.SFTP.SSHPrivateKey))
+	}
+
+	ctxt.Layout, err = buildDisksLayoutFromConfFile(cf.Pools)
+	return err
+}
+
+func serverHandleCmdArgs(ctxt serverCtxt) {
+	handleCommonArgs(ctxt)
 
 	logger.FatalIf(CheckLocalServerAddr(globalMinioAddr), "Unable to validate passed arguments")
 
@@ -251,7 +311,7 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	// Register root CAs for remote ENVs
 	env.RegisterGlobalCAs(globalRootCAs)
 
-	globalEndpoints, setupType, err = createServerEndpoints(globalMinioAddr, serverCmdArgs(ctx)...)
+	globalEndpoints, setupType, err = createServerEndpoints(globalMinioAddr, ctxt.Layout.pools, ctxt.Layout.legacy)
 	logger.FatalIf(err, "Invalid command line arguments")
 	globalNodes = globalEndpoints.GetNodes()
 
@@ -262,7 +322,7 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	}
 	globalIsErasureSD = (setupType == ErasureSDSetupType)
 	if globalDynamicAPIPort && globalIsDistErasure {
-		logger.FatalIf(errInvalidArgument, "Invalid --address=\"%s\", port '0' is not allowed in a distributed erasure coded setup", ctx.String("address"))
+		logger.FatalIf(errInvalidArgument, "Invalid --address=\"%s\", port '0' is not allowed in a distributed erasure coded setup", ctxt.Addr)
 	}
 
 	globalLocalNodeName = GetLocalPeer(globalEndpoints, globalMinioHost, globalMinioPort)
@@ -270,7 +330,7 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	globalLocalNodeNameHex = hex.EncodeToString(nodeNameSum[:])
 
 	// Initialize, see which NIC the service is running on, and save it as global value
-	setGlobalInternodeInterface(ctx.String("interface"))
+	setGlobalInternodeInterface(ctxt.Interface)
 
 	// allow transport to be HTTP/1.1 for proxying.
 	globalProxyTransport = NewCustomHTTPProxyTransport()()
@@ -289,8 +349,8 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	})
 
 	globalTCPOptions = xhttp.TCPOptions{
-		UserTimeout: int(ctx.Duration("conn-user-timeout").Milliseconds()),
-		Interface:   ctx.String("interface"),
+		UserTimeout: int(ctxt.UserTimeout.Milliseconds()),
+		Interface:   ctxt.Interface,
 	}
 
 	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
@@ -299,13 +359,8 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	// To avoid this error situation we check for port availability.
 	logger.FatalIf(xhttp.CheckPortAvailability(globalMinioHost, globalMinioPort, globalTCPOptions), "Unable to start the server")
 
-	globalConnReadDeadline = ctx.Duration("conn-read-deadline")
-	globalConnWriteDeadline = ctx.Duration("conn-write-deadline")
-}
-
-func serverHandleEnvVars() {
-	// Handle common environment variables.
-	handleCommonEnvVars()
+	globalConnReadDeadline = ctxt.ConnReadDeadline
+	globalConnWriteDeadline = ctxt.ConnWriteDeadline
 }
 
 var globalHealStateLK sync.RWMutex
@@ -321,7 +376,7 @@ func initAllSubsystems(ctx context.Context) {
 	globalNotificationSys = NewNotificationSys(globalEndpoints)
 
 	// Create new notification system
-	globalEventNotifier = NewEventNotifier()
+	globalEventNotifier = NewEventNotifier(GlobalContext)
 
 	// Create new bucket metadata system.
 	if globalBucketMetadataSys == nil {
@@ -590,14 +645,24 @@ func serverMain(ctx *cli.Context) {
 	// Always load ENV variables from files first.
 	loadEnvVarsFromFiles()
 
-	// Handle all server command args.
+	// Handle all server command args and build the disks layout
 	bootstrapTrace("serverHandleCmdArgs", func() {
-		serverHandleCmdArgs(ctx)
+		err := buildServerCtxt(ctx, &globalServerCtxt)
+		logger.FatalIf(err, "Unable to prepare the list of endpoints")
+
+		serverHandleCmdArgs(globalServerCtxt)
 	})
 
+	// DNS cache subsystem to reduce outgoing DNS requests
+	runDNSCache(ctx)
+
 	// Handle all server environment vars.
 	serverHandleEnvVars()
 
+	// Load the root credentials from the shell environment or from
+	// the config file if not defined, set the default one.
+	loadRootCredentials()
+
 	// Initialize globalConsoleSys system
 	bootstrapTrace("newConsoleLogger", func() {
 		globalConsoleSys = NewConsoleLogger(GlobalContext)
@@ -637,7 +702,7 @@ func serverMain(ctx *cli.Context) {
 
 	// Check for updates in non-blocking manner.
 	go func() {
-		if !globalCLIContext.Quiet && !globalInplaceUpdateDisabled {
+		if !globalServerCtxt.Quiet && !globalInplaceUpdateDisabled {
 			// Check for new updates from dl.min.io.
 			bootstrapTrace("checkUpdate", func() {
 				checkUpdate(getMinioMode())
@@ -683,9 +748,9 @@ func serverMain(ctx *cli.Context) {
 		httpServer := xhttp.NewServer(getServerListenAddrs()).
 			UseHandler(setCriticalErrorHandler(corsHandler(handler))).
 			UseTLSConfig(newTLSConfig(getCert)).
-			UseShutdownTimeout(ctx.Duration("shutdown-timeout")).
-			UseIdleTimeout(ctx.Duration("idle-timeout")).
-			UseReadHeaderTimeout(ctx.Duration("read-header-timeout")).
+			UseShutdownTimeout(globalServerCtxt.ShutdownTimeout).
+			UseIdleTimeout(globalServerCtxt.IdleTimeout).
+			UseReadHeaderTimeout(globalServerCtxt.ReadHeaderTimeout).
 			UseBaseContext(GlobalContext).
 			UseCustomLogger(log.New(io.Discard, "", 0)). // Turn-off random logging by Go stdlib
 			UseTCPOptions(globalTCPOptions)
@@ -779,7 +844,7 @@ func serverMain(ctx *cli.Context) {
 			logger.LogIf(GlobalContext, err)
 		}
 
-		if !globalCLIContext.StrictS3Compat {
+		if !globalServerCtxt.StrictS3Compat {
 			logger.Info(color.RedBold("WARNING: Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production"))
 		}
 	})
@@ -813,16 +878,16 @@ func serverMain(ctx *cli.Context) {
 		}
 
 		// if we see FTP args, start FTP if possible
-		if len(ctx.StringSlice("ftp")) > 0 {
+		if len(globalServerCtxt.FTP) > 0 {
 			bootstrapTrace("go startFTPServer", func() {
-				go startFTPServer(ctx)
+				go startFTPServer(globalServerCtxt.FTP)
 			})
 		}
 
 		// If we see SFTP args, start SFTP if possible
-		if len(ctx.StringSlice("sftp")) > 0 {
-			bootstrapTrace("go startFTPServer", func() {
-				go startSFTPServer(ctx)
+		if len(globalServerCtxt.SFTP) > 0 {
+			bootstrapTrace("go startSFTPServer", func() {
+				go startSFTPServer(globalServerCtxt.SFTP)
 			})
 		}
 	}()

cmd/server-main_test.go

@@ -23,6 +23,57 @@ import (
 	"testing"
 )
 
+func TestServerConfigFile(t *testing.T) {
+	for _, testcase := range []struct {
+		config      string
+		expectedErr bool
+		hash        string
+	}{
+		{
+			config:      "testdata/config/1.yaml",
+			expectedErr: false,
+			hash:        "hash:02bf70285dc71f76",
+		},
+		{
+			config:      "testdata/config/2.yaml",
+			expectedErr: false,
+			hash:        "hash:676d2da00f71f205",
+		},
+		{
+			config:      "testdata/config/invalid.yaml",
+			expectedErr: true,
+		},
+		{
+			config:      "testdata/config/invalid-types.yaml",
+			expectedErr: true,
+		},
+		{
+			config:      "testdata/config/invalid-disks.yaml",
+			expectedErr: true,
+		},
+	} {
+		testcase := testcase
+		t.Run(testcase.config, func(t *testing.T) {
+			sctx := &serverCtxt{}
+			err := mergeServerCtxtFromConfigFile(testcase.config, sctx)
+			if testcase.expectedErr && err == nil {
+				t.Error("expected failure, got success")
+			}
+			if !testcase.expectedErr && err != nil {
+				t.Error("expected success, got failure", err)
+			}
+			if err == nil {
+				if len(sctx.Layout.pools) != 2 {
+					t.Error("expected parsed pools to be 2, not", len(sctx.Layout.pools))
+				}
+				if sctx.Layout.pools[0].cmdline != testcase.hash {
+					t.Error("expected hash", testcase.hash, "got", sctx.Layout.pools[0].cmdline)
+				}
+			}
+		})
+	}
+}
+
 // Tests initializing new object layer.
 func TestNewObjectLayer(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())

cmd/server-startup-msg.go

@@ -55,7 +55,7 @@ func printStartupMessage(apiEndpoints []string, err error) {
 	// Object layer is initialized then print StorageInfo.
 	objAPI := newObjectLayerFn()
 	if objAPI != nil {
-		printStorageInfo(objAPI.StorageInfo(GlobalContext))
+		printStorageInfo(objAPI.StorageInfo(GlobalContext, true))
 	}
 
 	// Prints credential, region and browser access.
@@ -121,7 +121,7 @@ func printServerCommonMsg(apiEndpoints []string) {
 
 	// Colorize the message and print.
 	logger.Info(color.Blue("S3-API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
-	if color.IsTerminal() && (!globalCLIContext.Anonymous && !globalCLIContext.JSON) {
+	if color.IsTerminal() && (!globalServerCtxt.Anonymous && !globalServerCtxt.JSON) {
 		logger.Info(color.Blue("RootUser: ") + color.Bold("%s ", cred.AccessKey))
 		logger.Info(color.Blue("RootPass: ") + color.Bold("%s \n", cred.SecretKey))
 		if region != "" {
@@ -132,7 +132,7 @@ func printServerCommonMsg(apiEndpoints []string) {
 	if globalBrowserEnabled {
 		consoleEndpointStr := strings.Join(stripStandardPorts(getConsoleEndpoints(), globalMinioConsoleHost), " ")
 		logger.Info(color.Blue("Console: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
-		if color.IsTerminal() && (!globalCLIContext.Anonymous && !globalCLIContext.JSON) {
+		if color.IsTerminal() && (!globalServerCtxt.Anonymous && !globalServerCtxt.JSON) {
 			logger.Info(color.Blue("RootUser: ") + color.Bold("%s ", cred.AccessKey))
 			logger.Info(color.Blue("RootPass: ") + color.Bold("%s ", cred.SecretKey))
 		}
@@ -187,7 +187,7 @@ func printCLIAccessMsg(endPoint string, alias string) {
 	const mcQuickStartGuide = "https://min.io/docs/minio/linux/reference/minio-mc.html#quickstart"
 
 	// Configure 'mc', following block prints platform specific information for minio client.
-	if color.IsTerminal() && !globalCLIContext.Anonymous {
+	if color.IsTerminal() && !globalServerCtxt.Anonymous {
 		logger.Info(color.Blue("\nCommand-line: ") + mcQuickStartGuide)
 		mcMessage := fmt.Sprintf("$ mc alias set '%s' '%s' '%s' '%s'", alias,
 			endPoint, cred.AccessKey, cred.SecretKey)

cmd/server_test.go

@@ -137,7 +137,7 @@ func TestServerSuite(t *testing.T) {
 		// Init and run test on ErasureSet backend.
 		{serverType: "ErasureSet", signer: signerV4},
 	}
-	globalCLIContext.StrictS3Compat = true
+	globalServerCtxt.StrictS3Compat = true
 	for i, testCase := range testCases {
 		t.Run(fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.serverType), func(t *testing.T) {
 			runAllTests(testCase, &check{t, testCase.serverType})