fix: always load ENVs from files first as soon as server starts (#18247)

This is a regression from #18231, however reading from ENV files
must happen well before any parsing logic is invoked.

.github/workflows/go-cross.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/go-fips.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/go-healing.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/go-lint.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/go.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/helm-lint.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/iam-integrations.yaml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/mint.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - master
+      - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -37,7 +38,11 @@ jobs:
 
       - name: build-minio
         run: |
-          TAG="minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
+          TAG="quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
+
+      - name: multipart uploads test
+        run: |
+          ${GITHUB_WORKSPACE}/.github/workflows/multipart/migrate.sh "${{ steps.vars.outputs.sha_short }}"
 
       - name: compress and encrypt
         run: |
@@ -59,7 +64,14 @@ jobs:
              docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml down || true
              docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml rm || true
           done
-          docker rmi -f minio/minio:${{ steps.vars.outputs.sha_short }}
+
+          docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site1.yaml rm -s -f || true
+          docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site2.yaml rm -s -f || true
+          for volume in $(docker volume ls -q | grep minio); do
+            docker volume rm ${volume} || true
+          done
+
+          docker rmi -f quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}
           docker system prune -f || true
           docker volume prune -f || true
           docker volume rm $(docker volume ls -q -f dangling=true) || true

.github/workflows/mint/minio-compress-encrypt.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: minio/minio:${JOB_NAME}
+  image: quay.io/minio/minio:${JOB_NAME}
   command: server --console-address ":9001" http://minio{1...4}/cdata{1...2}
   expose:
     - "9000"

.github/workflows/mint/minio-erasure.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: minio/minio:${JOB_NAME}
+  image: quay.io/minio/minio:${JOB_NAME}
   command: server --console-address ":9001" edata{1...4}
   expose:
     - "9000"

.github/workflows/mint/minio-pools.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: minio/minio:${JOB_NAME}
+  image: quay.io/minio/minio:${JOB_NAME}
   command: server --console-address ":9001" http://minio{1...4}/pdata{1...2} http://minio{5...8}/pdata{1...2}
   expose:
     - "9000"

.github/workflows/multipart/docker-compose-site1.yaml

@@ -0,0 +1,66 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: quay.io/minio/minio:${RELEASE}
+  command: server http://site1-minio{1...4}/data{1...2}
+  environment:
+    - MINIO_PROMETHEUS_AUTH_TYPE=public
+    - CI=true
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  site1-minio1:
+    <<: *minio-common
+    hostname: site1-minio1
+    volumes:
+      - site1-data1-1:/data1
+      - site1-data1-2:/data2
+
+  site1-minio2:
+    <<: *minio-common
+    hostname: site1-minio2
+    volumes:
+      - site1-data2-1:/data1
+      - site1-data2-2:/data2
+
+  site1-minio3:
+    <<: *minio-common
+    hostname: site1-minio3
+    volumes:
+      - site1-data3-1:/data1
+      - site1-data3-2:/data2
+
+  site1-minio4:
+    <<: *minio-common
+    hostname: site1-minio4
+    volumes:
+      - site1-data4-1:/data1
+      - site1-data4-2:/data2
+
+  site1-nginx:
+    image: nginx:1.19.2-alpine
+    hostname: site1-nginx
+    volumes:
+      - ./nginx-site1.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9001:9001"
+    depends_on:
+      - site1-minio1
+      - site1-minio2
+      - site1-minio3
+      - site1-minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  site1-data1-1:
+  site1-data1-2:
+  site1-data2-1:
+  site1-data2-2:
+  site1-data3-1:
+  site1-data3-2:
+  site1-data4-1:
+  site1-data4-2:

.github/workflows/multipart/docker-compose-site2.yaml

@@ -0,0 +1,66 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: quay.io/minio/minio:${RELEASE}
+  command: server http://site2-minio{1...4}/data{1...2}
+  environment:
+    - MINIO_PROMETHEUS_AUTH_TYPE=public
+    - CI=true
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  site2-minio1:
+    <<: *minio-common
+    hostname: site2-minio1
+    volumes:
+      - site2-data1-1:/data1
+      - site2-data1-2:/data2
+
+  site2-minio2:
+    <<: *minio-common
+    hostname: site2-minio2
+    volumes:
+      - site2-data2-1:/data1
+      - site2-data2-2:/data2
+
+  site2-minio3:
+    <<: *minio-common
+    hostname: site2-minio3
+    volumes:
+      - site2-data3-1:/data1
+      - site2-data3-2:/data2
+
+  site2-minio4:
+    <<: *minio-common
+    hostname: site2-minio4
+    volumes:
+      - site2-data4-1:/data1
+      - site2-data4-2:/data2
+
+  site2-nginx:
+    image: nginx:1.19.2-alpine
+    hostname: site2-nginx
+    volumes:
+      - ./nginx-site2.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9002:9002"
+    depends_on:
+      - site2-minio1
+      - site2-minio2
+      - site2-minio3
+      - site2-minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  site2-data1-1:
+  site2-data1-2:
+  site2-data2-1:
+  site2-data2-2:
+  site2-data3-1:
+  site2-data3-2:
+  site2-data4-1:
+  site2-data4-2:

.github/workflows/multipart/migrate.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+
+set -x
+
+## change working directory
+cd .github/workflows/multipart/
+
+docker-compose -f docker-compose-site1.yaml rm -s -f
+docker-compose -f docker-compose-site2.yaml rm -s -f
+for volume in $(docker volume ls -q | grep minio); do
+	docker volume rm ${volume}
+done
+
+if [ ! -f ./mc ]; then
+	wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
+		chmod +x mc
+fi
+
+(
+	cd /tmp
+	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
+)
+
+export RELEASE=RELEASE.2023-08-29T23-07-35Z
+
+docker-compose -f docker-compose-site1.yaml up -d
+docker-compose -f docker-compose-site2.yaml up -d
+
+sleep 30s
+
+./mc alias set site1 http://site1-nginx:9001 minioadmin minioadmin --api s3v4
+./mc alias set site2 http://site2-nginx:9002 minioadmin minioadmin --api s3v4
+
+./mc ready site1/
+./mc ready site2/
+
+./mc admin replicate add site1 site2
+./mc mb site1/testbucket/
+./mc cp -r --quiet /usr/bin site1/testbucket/
+
+sleep 5
+
+s3-check-md5 -h
+
+failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+
+if [ $failed_count_site1 -ne 0 ]; then
+	echo "failed with multipart on site1 uploads"
+	exit 1
+fi
+
+if [ $failed_count_site2 -ne 0 ]; then
+	echo "failed with multipart on site2 uploads"
+	exit 1
+fi
+
+./mc cp -r --quiet /usr/bin site1/testbucket/
+
+sleep 5
+
+failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+
+## we do not need to fail here, since we are going to test
+## upgrading to master, healing and being able to recover
+## the last version.
+if [ $failed_count_site1 -ne 0 ]; then
+	echo "failed with multipart on site1 uploads ${failed_count_site1}"
+fi
+
+if [ $failed_count_site2 -ne 0 ]; then
+	echo "failed with multipart on site2 uploads ${failed_count_site2}"
+fi
+
+export RELEASE=${1}
+
+docker-compose -f docker-compose-site1.yaml up -d
+docker-compose -f docker-compose-site2.yaml up -d
+
+./mc ready site1/
+./mc ready site2/
+
+for i in $(seq 1 10); do
+	# mc admin heal -r --remove when used against a LB endpoint
+	# behaves flaky, let this run 10 times before giving up
+	./mc admin heal -r --remove --json site1/ 2>&1 >/dev/null
+	./mc admin heal -r --remove --json site2/ 2>&1 >/dev/null
+done
+
+failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+
+if [ $failed_count_site1 -ne 0 ]; then
+	echo "failed with multipart on site1 uploads"
+	exit 1
+fi
+
+if [ $failed_count_site2 -ne 0 ]; then
+	echo "failed with multipart on site2 uploads"
+	exit 1
+fi
+
+docker-compose -f docker-compose-site1.yaml rm -s -f
+docker-compose -f docker-compose-site2.yaml rm -s -f
+for volume in $(docker volume ls -q | grep minio); do
+	docker volume rm ${volume}
+done
+
+docker system prune -f || true
+docker volume prune -f || true
+docker volume rm $(docker volume ls -q -f dangling=true) || true
+
+## change working directory
+cd ../../../

.github/workflows/multipart/nginx-site1.conf

@@ -0,0 +1,61 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server site1-minio1:9000;
+        server site1-minio2:9000;
+        server site1-minio3:9000;
+        server site1-minio4:9000;
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+}

.github/workflows/multipart/nginx-site2.conf

@@ -0,0 +1,61 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server site2-minio1:9000;
+        server site2-minio2:9000;
+        server site2-minio3:9000;
+        server site2-minio4:9000;
+    }
+
+    server {
+        listen       9002;
+        listen  [::]:9002;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+}

.github/workflows/replication.yaml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/root-disable.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/run-mint.sh

@@ -16,7 +16,7 @@ docker volume rm $(docker volume ls -f dangling=true) || true
 cd .github/workflows/mint
 
 docker-compose -f minio-${MODE}.yaml up -d
-sleep 5m
+sleep 30s
 
 docker system prune -f || true
 docker volume prune -f || true

.github/workflows/shfmt.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 permissions:
   contents: read

.github/workflows/upgrade-ci-cd.yaml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
     - master
+    - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.

.github/workflows/vulncheck.yml

@@ -3,9 +3,11 @@ on:
   pull_request:
     branches:
       - master
+      - next
   push:
     branches:
       - master
+      - next
 
 permissions:
   contents: read # to fetch code (actions/checkout)
@@ -20,7 +22,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.21.1
+          go-version: 1.21.3
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

Makefile

@@ -6,7 +6,7 @@ GOARCH := $(shell go env GOARCH)
 GOOS := $(shell go env GOOS)
 
 VERSION ?= $(shell git describe --tags)
-TAG ?= "minio/minio:$(VERSION)"
+TAG ?= "quay.io/minio/minio:$(VERSION)"
 
 GOLANGCI_VERSION = v1.51.2
 GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)

buildscripts/rewrite-old-new.sh

@@ -87,8 +87,8 @@ function verify_rewrite() {
 		exit 1
 	fi
 
-	go build ./docs/debugging/s3-check-md5/
-	if ! ./s3-check-md5 \
+	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
+	if ! s3-check-md5 \
 		-debug \
 		-versions \
 		-access-key minio \
@@ -113,7 +113,7 @@ function verify_rewrite() {
 	go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
 	sleep 1
 
-	if ! ./s3-check-md5 \
+	if ! s3-check-md5 \
 		-debug \
 		-versions \
 		-access-key minio \

buildscripts/unaligned-healing.sh

@@ -82,8 +82,8 @@ function start_minio_16drive() {
 	rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
 	sleep 10
 
-	go build ./docs/debugging/s3-check-md5/
-	if ! ./s3-check-md5 \
+	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
+	if ! s3-check-md5 \
 		-debug \
 		-access-key minio \
 		-secret-key minio123 \
@@ -111,7 +111,7 @@ function start_minio_16drive() {
 		exit 1
 	fi
 
-	if ! ./s3-check-md5 \
+	if ! s3-check-md5 \
 		-debug \
 		-access-key minio \
 		-secret-key minio123 \
@@ -134,7 +134,7 @@ function start_minio_16drive() {
 
 	"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
 
-	if ! ./s3-check-md5 \
+	if ! s3-check-md5 \
 		-debug \
 		-access-key minio \
 		-secret-key minio123 \

cmd/admin-bucket-handlers.go

@@ -392,7 +392,8 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 	// of bucket metadata
 	zipWriter := zip.NewWriter(w)
 	defer zipWriter.Close()
-	rawDataFn := func(r io.Reader, filename string, sz int) error {
+
+	rawDataFn := func(r io.Reader, filename string, sz int) {
 		header, zerr := zip.FileInfoHeader(dummyFileInfo{
 			name:    filename,
 			size:    int64(sz),
@@ -401,20 +402,13 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 			isDir:   false,
 			sys:     nil,
 		})
-		if zerr != nil {
-			logger.LogIf(ctx, zerr)
-			return nil
+		if zerr == nil {
+			header.Method = zip.Deflate
+			zwriter, zerr := zipWriter.CreateHeader(header)
+			if zerr == nil {
+				io.Copy(zwriter, r)
+			}
 		}
-		header.Method = zip.Deflate
-		zwriter, zerr := zipWriter.CreateHeader(header)
-		if zerr != nil {
-			logger.LogIf(ctx, zerr)
-			return nil
-		}
-		if _, err := io.Copy(zwriter, r); err != nil {
-			logger.LogIf(ctx, err)
-		}
-		return nil
 	}
 
 	cfgFiles := []string{
@@ -446,10 +440,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketLifecycleConfig:
 				config, _, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
 				if err != nil {
@@ -465,10 +456,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketQuotaConfigFile:
 				config, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
 				if err != nil {
@@ -483,10 +471,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketSSEConfig:
 				config, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
 				if err != nil {
@@ -501,10 +486,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketTaggingConfig:
 				config, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
 				if err != nil {
@@ -519,10 +501,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case objectLockConfig:
 				config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
 				if err != nil {
@@ -538,10 +517,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketVersioningConfig:
 				config, _, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
 				if err != nil {
@@ -557,10 +533,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketReplicationConfig:
 				config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
 				if err != nil {
@@ -575,11 +548,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketTargetsFile:
 				config, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket)
 				if err != nil {
@@ -595,10 +564,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			}
 		}
 	}

cmd/admin-handlers-config-kv.go

@@ -28,7 +28,6 @@ import (
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
-	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/etcd"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
@@ -500,8 +499,6 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 			switch hkv.Key {
 			case config.EtcdSubSys:
 				off = !etcd.Enabled(item.Config)
-			case config.CacheSubSys:
-				off = !cache.Enabled(item.Config)
 			case config.StorageClassSubSys:
 				off = !storageclass.Enabled(item.Config)
 			case config.PolicyPluginSubSys:

cmd/admin-handlers-idp-config.go

@@ -121,7 +121,7 @@ func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.R
 
 	// IDP config is not dynamic. Sanity check.
 	if dynamic {
-		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), "", r.URL)
 		return
 	}
 

cmd/admin-handlers-users.go

@@ -1147,7 +1147,11 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 
 	// We do not care if service account is readable or not at this point,
 	// since this is a delete call we shall allow it to be deleted if possible.
-	svcAccount, _, _ := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
+	svcAccount, _, err := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
+	if errors.Is(err, errNoSuchServiceAccount) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
+		return
+	}
 
 	adminPrivilege := globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,

cmd/admin-handlers.go

@@ -1581,7 +1581,9 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 
 	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	traceCh := make(chan madmin.TraceInfo, 4000)
+
+	// Keep 100k buffered channel, should be sufficient to ensure we do not lose any events.
+	traceCh := make(chan madmin.TraceInfo, 100000)
 
 	peers, _ := newPeerRestClients(globalEndpoints)
 
@@ -1605,7 +1607,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		peer.Trace(traceCh, ctx.Done(), traceOpts)
 	}
 
-	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	keepAliveTicker := time.NewTicker(time.Second)
 	defer keepAliveTicker.Stop()
 
 	enc := json.NewEncoder(w)

cmd/api-errors.go

@@ -2337,6 +2337,12 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case dns.ErrBucketConflict:
 		apiErr = ErrBucketAlreadyExists
 	default:
+		if _, ok := err.(tags.Error); ok {
+			// tag errors are not exported, so we check their custom interface to avoid logging.
+			// The correct type is inserted by toAPIError.
+			apiErr = ErrInternalError
+			break
+		}
 		var ie, iw int
 		// This work-around is to handle the issue golang/go#30648
 		//nolint:gocritic

cmd/api-headers.go

@@ -133,11 +133,6 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
 	}
 
-	if globalCacheConfig.Enabled {
-		w.Header().Set(xhttp.XCache, objInfo.CacheStatus.String())
-		w.Header().Set(xhttp.XCacheLookup, objInfo.CacheLookupStatus.String())
-	}
-
 	// Set tag count if object has tags
 	if len(objInfo.UserTags) > 0 {
 		tags, _ := tags.ParseObjectTags(objInfo.UserTags)

cmd/api-router.go

@@ -61,18 +61,6 @@ func newObjectLayerFn() ObjectLayer {
 	return globalObjectAPI
 }
 
-func newCachedObjectLayerFn() CacheObjectLayer {
-	globalObjLayerMutex.RLock()
-	defer globalObjLayerMutex.RUnlock()
-	return globalCacheObjectAPI
-}
-
-func setCacheObjectLayer(c CacheObjectLayer) {
-	globalObjLayerMutex.Lock()
-	globalCacheObjectAPI = c
-	globalObjLayerMutex.Unlock()
-}
-
 func setObjectLayer(o ObjectLayer) {
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = o
@@ -82,7 +70,6 @@ func setObjectLayer(o ObjectLayer) {
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
 	ObjectAPI func() ObjectLayer
-	CacheAPI  func() CacheObjectLayer
 }
 
 // getHost tries its best to return the request host.
@@ -189,7 +176,6 @@ func registerAPIRouter(router *mux.Router) {
 	// Initialize API.
 	api := objectAPIHandlers{
 		ObjectAPI: newObjectLayerFn,
-		CacheAPI:  newCachedObjectLayerFn,
 	}
 
 	// API Router

cmd/bucket-handlers.go

@@ -474,9 +474,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	}
 
 	deleteObjectsFn := objectAPI.DeleteObjects
-	if api.CacheAPI() != nil {
-		deleteObjectsFn = api.CacheAPI().DeleteObjects
-	}
 
 	// Return Malformed XML as S3 spec if the number of objects is empty
 	if len(deleteObjectsReq.Objects) == 0 || len(deleteObjectsReq.Objects) > maxDeleteList {
@@ -486,9 +483,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	objectsToDelete := map[ObjectToDelete]int{}
 	getObjectInfoFn := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfoFn = api.CacheAPI().GetObjectInfo
-	}
 
 	var (
 		hasLockEnabled bool
@@ -716,7 +710,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		if os == nil { // skip objects that weren't deleted due to invalid versionID etc.
 			continue
 		}
-		logger.LogIf(ctx, os.Sweep())
+		os.Sweep()
 	}
 }
 
@@ -794,7 +788,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 
 	// check if client is attempting to create more buckets, complain about it.
 	if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
-		logger.LogIf(ctx, fmt.Errorf("An attempt to create %d buckets beyond recommended %d", currBuckets+1, maxBuckets))
+		logger.LogIf(ctx, fmt.Errorf("Please avoid creating more buckets %d beyond recommended %d", currBuckets+1, maxBuckets))
 	}
 
 	opts := MakeBucketOptions{
@@ -1045,11 +1039,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		break
 	}
 
-	if _, ok := formValues["Key"]; !ok {
+	if keyName, ok := formValues["Key"]; !ok {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
 		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The name of the uploaded key is missing"))
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
+	} else if fileName == "" && len(keyName) >= 1 {
+		// if we can't get fileName. We use keyName[0] to fileName
+		fileName = keyName[0]
 	}
 
 	if fileName == "" {
@@ -1141,7 +1138,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	hashReader, err := hash.NewReader(ctx, reader, fileSize, "", "", fileSize)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-replication.go

@@ -376,8 +376,10 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 			} else {
 				// can be the case that other cluster is down and duplicate `mc rm --vid`
 				// is issued - this still needs to be replicated back to the other target
-				replicate = oi.VersionPurgeStatus == Pending || oi.VersionPurgeStatus == Failed
-				dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync))
+				if !oi.VersionPurgeStatus.Empty() {
+					replicate = oi.VersionPurgeStatus == Pending || oi.VersionPurgeStatus == Failed
+					dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync))
+				}
 				continue
 			}
 		}
@@ -2485,7 +2487,7 @@ func newresyncer() *replicationResyncer {
 }
 
 // mark status of replication resync on remote target for the bucket
-func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpts) {
+func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpts, objAPI ObjectLayer) {
 	s.Lock()
 	defer s.Unlock()
 
@@ -2496,6 +2498,10 @@ func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpt
 	m.TargetsMap[opts.arn] = st
 	m.LastUpdate = UTCNow()
 	s.statusMap[opts.bucket] = m
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+	saveResyncStatus(ctx, opts.bucket, m, objAPI)
 }
 
 // update replication resync stats for bucket's remote target
@@ -2525,7 +2531,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 
 	resyncStatus := ResyncFailed
 	defer func() {
-		s.markStatus(resyncStatus, opts)
+		s.markStatus(resyncStatus, opts, objectAPI)
 		globalSiteResyncMetrics.incBucket(opts, resyncStatus)
 		s.workerCh <- struct{}{}
 	}()
@@ -2561,7 +2567,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	}
 	// mark resync status as resync started
 	if !heal {
-		s.markStatus(ResyncStarted, opts)
+		s.markStatus(ResyncStarted, opts, objectAPI)
 	}
 
 	// Walk through all object versions - Walk() is always in ascending order needed to ensure
@@ -2632,16 +2638,18 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 					roi.EventType = ReplicateExisting
 					replicateObject(ctx, roi, objectAPI)
 				}
-				_, err = tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
+
+				st := TargetReplicationResyncStatus{
+					Object: roi.Name,
+					Bucket: roi.Bucket,
+				}
+
+				_, err := tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
 					VersionID: roi.VersionID,
 					Internal: minio.AdvancedGetOptions{
 						ReplicationProxyRequest: "false",
 					},
 				})
-				st := TargetReplicationResyncStatus{
-					Object: roi.Name,
-					Bucket: roi.Bucket,
-				}
 				if err != nil {
 					if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
 						st.ReplicatedCount++

cmd/common-main.go

@@ -619,8 +619,6 @@ func loadEnvVarsFromFiles() {
 }
 
 func handleCommonEnvVars() {
-	loadEnvVarsFromFiles()
-
 	var err error
 	globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.EnableOn))
 	if err != nil {

cmd/config-current.go

@@ -27,7 +27,6 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/api"
-	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/callhome"
 	"github.com/minio/minio/internal/config/compress"
 	"github.com/minio/minio/internal/config/dns"
@@ -46,7 +45,6 @@ import (
 	"github.com/minio/minio/internal/config/subnet"
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
 )
@@ -54,7 +52,6 @@ import (
 func initHelp() {
 	kvs := map[string]config.KVS{
 		config.EtcdSubSys:           etcd.DefaultKVS,
-		config.CacheSubSys:          cache.DefaultKVS,
 		config.CompressionSubSys:    compress.DefaultKVS,
 		config.IdentityLDAPSubSys:   xldap.DefaultKVS,
 		config.IdentityOpenIDSubSys: openid.DefaultKVS,
@@ -209,10 +206,6 @@ func initHelp() {
 			Key:         config.EtcdSubSys,
 			Description: "persist IAM assets externally to etcd",
 		},
-		config.HelpKV{
-			Key:         config.CacheSubSys,
-			Description: "[DEPRECATED] add caching storage tier",
-		},
 	}
 
 	if globalIsErasure {
@@ -232,7 +225,6 @@ func initHelp() {
 		config.APISubSys:            api.Help,
 		config.StorageClassSubSys:   storageclass.Help,
 		config.EtcdSubSys:           etcd.Help,
-		config.CacheSubSys:          cache.Help,
 		config.CompressionSubSys:    compress.Help,
 		config.HealSubSys:           heal.Help,
 		config.ScannerSubSys:        scanner.Help,
@@ -302,10 +294,6 @@ func validateSubSysConfig(ctx context.Context, s config.Config, subSys string, o
 				return err
 			}
 		}
-	case config.CacheSubSys:
-		if _, err := cache.LookupConfig(s[config.CacheSubSys][config.Default]); err != nil {
-			return err
-		}
 	case config.CompressionSubSys:
 		if _, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default]); err != nil {
 			return err
@@ -493,20 +481,6 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err))
 	}
 
-	globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
-	}
-
-	if globalCacheConfig.Enabled {
-		if cacheEncKey := env.Get(cache.EnvCacheEncryptionKey, ""); cacheEncKey != "" {
-			globalCacheKMS, err = kms.Parse(cacheEncKey)
-			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to setup encryption cache: %w", err))
-			}
-		}
-	}
-
 	globalAutoEncryption = crypto.LookupAutoEncryption() // Enable auto-encryption if enabled
 	if globalAutoEncryption && GlobalKMS == nil {
 		logger.Fatal(errors.New("no KMS configured"), "MINIO_KMS_AUTO_ENCRYPTION requires a valid KMS configuration")

cmd/config-migrate.go

@@ -29,7 +29,6 @@ import (
 
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
-	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/compress"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
@@ -1997,11 +1996,6 @@ func migrateV22ToV23() error {
 	srvConfig.StorageClass.RRS = cv22.StorageClass.RRS
 	srvConfig.StorageClass.Standard = cv22.StorageClass.Standard
 
-	// Init cache config.For future migration, Cache config needs to be copied over from previous version.
-	srvConfig.Cache.Drives = []string{}
-	srvConfig.Cache.Exclude = []string{}
-	srvConfig.Cache.Expiry = 90
-
 	if err = Save(configFile, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv22.Version, srvConfig.Version, err)
 	}
@@ -2110,11 +2104,6 @@ func migrateV23ToV24() error {
 	srvConfig.StorageClass.RRS = cv23.StorageClass.RRS
 	srvConfig.StorageClass.Standard = cv23.StorageClass.Standard
 
-	// Load cache config from existing cache config in the file.
-	srvConfig.Cache.Drives = cv23.Cache.Drives
-	srvConfig.Cache.Exclude = cv23.Cache.Exclude
-	srvConfig.Cache.Expiry = cv23.Cache.Expiry
-
 	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv23.Version, srvConfig.Version, err)
 	}
@@ -2228,11 +2217,6 @@ func migrateV24ToV25() error {
 	srvConfig.StorageClass.RRS = cv24.StorageClass.RRS
 	srvConfig.StorageClass.Standard = cv24.StorageClass.Standard
 
-	// Load cache config from existing cache config in the file.
-	srvConfig.Cache.Drives = cv24.Cache.Drives
-	srvConfig.Cache.Exclude = cv24.Cache.Exclude
-	srvConfig.Cache.Expiry = cv24.Cache.Expiry
-
 	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv24.Version, srvConfig.Version, err)
 	}
@@ -2344,14 +2328,6 @@ func migrateV25ToV26() error {
 	srvConfig.StorageClass.RRS = cv25.StorageClass.RRS
 	srvConfig.StorageClass.Standard = cv25.StorageClass.Standard
 
-	// Load cache config from existing cache config in the file.
-	srvConfig.Cache.Drives = cv25.Cache.Drives
-	srvConfig.Cache.Exclude = cv25.Cache.Exclude
-	srvConfig.Cache.Expiry = cv25.Cache.Expiry
-
-	// Add predefined value to new server config.
-	srvConfig.Cache.MaxUse = 80
-
 	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %w", cv25.Version, srvConfig.Version, err)
 	}
@@ -2574,7 +2550,6 @@ func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.C
 
 	xldap.SetIdentityLDAP(newCfg, cfg.LDAPServerConfig)
 	opa.SetPolicyOPAConfig(newCfg, cfg.Policy.OPA)
-	cache.SetCacheConfig(newCfg, cfg.Cache)
 	compress.SetCompressionConfig(newCfg, cfg.Compression)
 
 	for k, args := range cfg.Notify.AMQP {

cmd/config-versions.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
-	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/compress"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
@@ -589,9 +588,6 @@ type serverConfigV23 struct {
 	// Storage class configuration
 	StorageClass storageclass.Config `json:"storageclass"`
 
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 }
@@ -610,9 +606,6 @@ type serverConfigV24 struct {
 	// Storage class configuration
 	StorageClass storageclass.Config `json:"storageclass"`
 
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 }
@@ -634,9 +627,6 @@ type serverConfigV25 struct {
 	// Storage class configuration
 	StorageClass storageclass.Config `json:"storageclass"`
 
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 }
@@ -658,9 +648,6 @@ type serverConfigV26 struct {
 	// Storage class configuration
 	StorageClass storageclass.Config `json:"storageclass"`
 
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 }
@@ -682,9 +669,6 @@ type serverConfigV27 struct {
 	// Storage class configuration
 	StorageClass storageclass.Config `json:"storageclass"`
 
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 
@@ -707,9 +691,6 @@ type serverConfigV28 struct {
 	// Storage class configuration
 	StorageClass storageclass.Config `json:"storageclass"`
 
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 
@@ -731,9 +712,6 @@ type serverConfigV33 struct {
 	// Storage class configuration
 	StorageClass storageclass.Config `json:"storageclass"`
 
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
 	// Notification queue configuration.
 	Notify notify.Config `json:"notify"`
 

cmd/data-scanner.go

@@ -929,7 +929,8 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 
 	versionID := oi.VersionID
 	rCfg, _ := globalBucketObjectLockSys.Get(i.bucket)
-	lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi)
+	replcfg, _ := getReplicationConfig(ctx, i.bucket)
+	lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, replcfg, oi)
 	if i.debug {
 		if versionID != "" {
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, lcEvt.Action)
@@ -1129,7 +1130,7 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 	return size
 }
 
-func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo) lifecycle.Event {
+func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, obj ObjectInfo) lifecycle.Event {
 	event := lc.Eval(obj.ToLifecycleOpts())
 	if serverDebugLog {
 		console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", event.Action)
@@ -1161,6 +1162,9 @@ func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr loc
 			}
 			return lifecycle.Event{Action: lifecycle.NoneAction}
 		}
+		if rcfg != nil && !obj.VersionPurgeStatus.Empty() && rcfg.HasActiveRules(obj.Name, true) {
+			return lifecycle.Event{Action: lifecycle.NoneAction}
+		}
 	}
 
 	return event

cmd/data-usage-cache.go

@@ -924,7 +924,7 @@ type objectIO interface {
 // load the cache content with name from minioMetaBackgroundOpsBucket.
 // Only backend errors are returned as errors.
 // The loader is optimistic and has no locking, but tries 5 times before giving up.
-// If the object is not found or unable to deserialize d is cleared and nil error is returned.
+// If the object is not found, a nil error with empty data usage cache is returned.
 func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error {
 	// By defaut, empty data usage cache
 	*d = dataUsageCache{}
@@ -960,13 +960,19 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string)
 		if !retry {
 			break
 		}
-		retry, _ = load(name+".bkp", 30*time.Second)
-		if !retry {
+		retry, err = load(name+".bkp", 30*time.Second)
+		if err == nil && !retry {
+			// Only return when we have valid data from the backup
 			break
 		}
 		retries++
 		time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
 	}
+
+	if retries == 5 {
+		logger.LogOnceIf(ctx, fmt.Errorf("maximum retry reached to load the data usage cache `%s`", name), "retry-loading-data-usage-cache")
+	}
+
 	return nil
 }
 

cmd/disk-cache-check-support_contrib_windows.go

@@ -1,60 +0,0 @@
-//go:build windows
-// +build windows
-
-/*
- * MinIO Object Storage (c) 2021 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"errors"
-	"os"
-
-	"github.com/djherbis/atime"
-	"golang.org/x/sys/windows/registry"
-)
-
-// Return error if Atime is disabled on the O/S
-func checkAtimeSupport(dir string) (err error) {
-	file, err := os.CreateTemp(dir, "prefix")
-	if err != nil {
-		return
-	}
-	defer os.Remove(file.Name())
-	defer file.Close()
-	finfo1, err := os.Stat(file.Name())
-	if err != nil {
-		return
-	}
-	atime.Get(finfo1)
-
-	k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SYSTEM\CurrentControlSet\Control\FileSystem`, registry.QUERY_VALUE)
-	if err != nil {
-		return
-	}
-	defer k.Close()
-
-	setting, _, err := k.GetIntegerValue("NtfsDisableLastAccessUpdate")
-	if err != nil {
-		return
-	}
-
-	lowSetting := setting & 0xFFFF
-	if lowSetting != uint64(0x0000) && lowSetting != uint64(0x0002) {
-		return errors.New("Atime not supported")
-	}
-	return
-}

cmd/disk-cache-check-support_other.go

@@ -1,57 +0,0 @@
-//go:build !windows
-// +build !windows
-
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"errors"
-	"io"
-	"os"
-	"time"
-
-	"github.com/djherbis/atime"
-)
-
-// Return error if Atime is disabled on the O/S
-func checkAtimeSupport(dir string) (err error) {
-	file, err := os.CreateTemp(dir, "prefix")
-	if err != nil {
-		return
-	}
-	defer os.Remove(file.Name())
-	defer file.Close()
-	finfo1, err := os.Stat(file.Name())
-	if err != nil {
-		return
-	}
-	// add a sleep to ensure atime change is detected
-	time.Sleep(10 * time.Millisecond)
-
-	if _, err = io.Copy(io.Discard, file); err != nil {
-		return
-	}
-
-	finfo2, err := os.Stat(file.Name())
-
-	if atime.Get(finfo2).Equal(atime.Get(finfo1)) {
-		return errors.New("Atime not supported")
-	}
-	return
-}

cmd/disk-cache-stats.go

@@ -1,88 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"sync/atomic"
-)
-
-// CacheDiskStats represents cache disk statistics
-// such as current disk usage and available.
-type CacheDiskStats struct {
-	// used cache size
-	UsageSize uint64
-	// total cache disk capacity
-	TotalCapacity uint64
-	// indicates if usage is high or low, if high value is '1', if low its '0'
-	UsageState int32
-	// indicates the current usage percentage of this cache disk
-	UsagePercent uint64
-	Dir          string
-}
-
-// GetUsageLevelString gets the string representation for the usage level.
-func (c *CacheDiskStats) GetUsageLevelString() (u string) {
-	if atomic.LoadInt32(&c.UsageState) == 0 {
-		return "low"
-	}
-	return "high"
-}
-
-// CacheStats - represents bytes served from cache,
-// cache hits and cache misses.
-type CacheStats struct {
-	BytesServed  uint64
-	Hits         uint64
-	Misses       uint64
-	GetDiskStats func() []CacheDiskStats
-}
-
-// Increase total bytes served from cache
-func (s *CacheStats) incBytesServed(n int64) {
-	atomic.AddUint64(&s.BytesServed, uint64(n))
-}
-
-// Increase cache hit by 1
-func (s *CacheStats) incHit() {
-	atomic.AddUint64(&s.Hits, 1)
-}
-
-// Increase cache miss by 1
-func (s *CacheStats) incMiss() {
-	atomic.AddUint64(&s.Misses, 1)
-}
-
-// Get total bytes served
-func (s *CacheStats) getBytesServed() uint64 {
-	return atomic.LoadUint64(&s.BytesServed)
-}
-
-// Get total cache hits
-func (s *CacheStats) getHits() uint64 {
-	return atomic.LoadUint64(&s.Hits)
-}
-
-// Get total cache misses
-func (s *CacheStats) getMisses() uint64 {
-	return atomic.LoadUint64(&s.Misses)
-}
-
-// Prepare new CacheStats structure
-func newCacheStats() *CacheStats {
-	return &CacheStats{}
-}

cmd/disk-cache-utils.go

@@ -1,587 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"container/list"
-	"errors"
-	"fmt"
-	"io"
-	"math"
-	"os"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/minio/minio/internal/crypto"
-	"github.com/minio/minio/internal/etag"
-)
-
-// CacheStatusType - whether the request was served from cache.
-type CacheStatusType string
-
-const (
-	// CacheHit - whether object was served from cache.
-	CacheHit CacheStatusType = "HIT"
-
-	// CacheMiss - object served from backend.
-	CacheMiss CacheStatusType = "MISS"
-)
-
-func (c CacheStatusType) String() string {
-	if c != "" {
-		return string(c)
-	}
-	return string(CacheMiss)
-}
-
-type cacheControl struct {
-	expiry       time.Time
-	maxAge       int
-	sMaxAge      int
-	minFresh     int
-	maxStale     int
-	noStore      bool
-	onlyIfCached bool
-	noCache      bool
-}
-
-func (c *cacheControl) isStale(modTime time.Time) bool {
-	if c == nil {
-		return false
-	}
-	// response will never be stale if only-if-cached is set
-	if c.onlyIfCached {
-		return false
-	}
-	// Cache-Control value no-store indicates never cache
-	if c.noStore {
-		return true
-	}
-	// Cache-Control value no-cache indicates cache entry needs to be revalidated before
-	// serving from cache
-	if c.noCache {
-		return true
-	}
-	now := time.Now()
-
-	if c.sMaxAge > 0 && c.sMaxAge < int(now.Sub(modTime).Seconds()) {
-		return true
-	}
-	if c.maxAge > 0 && c.maxAge < int(now.Sub(modTime).Seconds()) {
-		return true
-	}
-
-	if !c.expiry.Equal(time.Time{}) && c.expiry.Before(time.Now().Add(time.Duration(c.maxStale))) {
-		return true
-	}
-
-	if c.minFresh > 0 && c.minFresh <= int(now.Sub(modTime).Seconds()) {
-		return true
-	}
-
-	return false
-}
-
-// returns struct with cache-control settings from user metadata.
-func cacheControlOpts(o ObjectInfo) *cacheControl {
-	c := cacheControl{}
-	m := o.UserDefined
-	if !o.Expires.Equal(timeSentinel) {
-		c.expiry = o.Expires
-	}
-
-	var headerVal string
-	for k, v := range m {
-		if strings.EqualFold(k, "cache-control") {
-			headerVal = v
-		}
-	}
-	if headerVal == "" {
-		return nil
-	}
-	headerVal = strings.ToLower(headerVal)
-	headerVal = strings.TrimSpace(headerVal)
-
-	vals := strings.Split(headerVal, ",")
-	for _, val := range vals {
-		val = strings.TrimSpace(val)
-
-		if val == "no-store" {
-			c.noStore = true
-			continue
-		}
-		if val == "only-if-cached" {
-			c.onlyIfCached = true
-			continue
-		}
-		if val == "no-cache" {
-			c.noCache = true
-			continue
-		}
-		p := strings.Split(val, "=")
-
-		if len(p) != 2 {
-			continue
-		}
-		if p[0] == "max-age" ||
-			p[0] == "s-maxage" ||
-			p[0] == "min-fresh" ||
-			p[0] == "max-stale" {
-			i, err := strconv.Atoi(p[1])
-			if err != nil {
-				return nil
-			}
-			if p[0] == "max-age" {
-				c.maxAge = i
-			}
-			if p[0] == "s-maxage" {
-				c.sMaxAge = i
-			}
-			if p[0] == "min-fresh" {
-				c.minFresh = i
-			}
-			if p[0] == "max-stale" {
-				c.maxStale = i
-			}
-		}
-	}
-	return &c
-}
-
-// backendDownError returns true if err is due to backend failure or faulty disk if in server mode
-func backendDownError(err error) bool {
-	_, backendDown := err.(BackendDown)
-	return backendDown || IsErr(err, baseErrs...)
-}
-
-// IsCacheable returns if the object should be saved in the cache.
-func (o ObjectInfo) IsCacheable() bool {
-	if globalCacheKMS != nil {
-		return true
-	}
-	_, ok := crypto.IsEncrypted(o.UserDefined)
-	return !ok
-}
-
-// reads file cached on disk from offset upto length
-func readCacheFileStream(filePath string, offset, length int64) (io.ReadCloser, error) {
-	if filePath == "" || offset < 0 {
-		return nil, errInvalidArgument
-	}
-	if err := checkPathLength(filePath); err != nil {
-		return nil, err
-	}
-
-	fr, err := os.Open(filePath)
-	if err != nil {
-		return nil, osErrToFileErr(err)
-	}
-	// Stat to get the size of the file at path.
-	st, err := fr.Stat()
-	if err != nil {
-		err = osErrToFileErr(err)
-		return nil, err
-	}
-
-	if err = os.Chtimes(filePath, time.Now(), st.ModTime()); err != nil {
-		return nil, err
-	}
-
-	// Verify if its not a regular file, since subsequent Seek is undefined.
-	if !st.Mode().IsRegular() {
-		return nil, errIsNotRegular
-	}
-
-	if err = os.Chtimes(filePath, time.Now(), st.ModTime()); err != nil {
-		return nil, err
-	}
-
-	// Seek to the requested offset.
-	if offset > 0 {
-		_, err = fr.Seek(offset, io.SeekStart)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return struct {
-		io.Reader
-		io.Closer
-	}{Reader: io.LimitReader(fr, length), Closer: fr}, nil
-}
-
-func isCacheEncrypted(meta map[string]string) bool {
-	_, ok := meta[SSECacheEncrypted]
-	return ok
-}
-
-// decryptCacheObjectETag tries to decrypt the ETag saved in encrypted format using the cache KMS
-func decryptCacheObjectETag(info *ObjectInfo) error {
-	if info.IsDir {
-		return nil // Directories are never encrypted.
-	}
-
-	// Depending on the SSE type we handle ETags slightly
-	// differently. ETags encrypted with SSE-S3 must be
-	// decrypted first, since the client expects that
-	// a single-part SSE-S3 ETag is equal to the content MD5.
-	//
-	// For all other SSE types, the ETag is not the content MD5.
-	// Therefore, we don't decrypt but only format it.
-	switch kind, ok := crypto.IsEncrypted(info.UserDefined); {
-	case ok && kind == crypto.S3 && isCacheEncrypted(info.UserDefined):
-		ETag, err := etag.Parse(info.ETag)
-		if err != nil {
-			return err
-		}
-		if !ETag.IsEncrypted() {
-			info.ETag = ETag.Format().String()
-			return nil
-		}
-
-		key, err := crypto.S3.UnsealObjectKey(globalCacheKMS, info.UserDefined, info.Bucket, info.Name)
-		if err != nil {
-			return err
-		}
-		ETag, err = etag.Decrypt(key[:], ETag)
-		if err != nil {
-			return err
-		}
-		info.ETag = ETag.Format().String()
-	case ok && (kind == crypto.S3KMS || kind == crypto.SSEC) && isCacheEncrypted(info.UserDefined):
-		ETag, err := etag.Parse(info.ETag)
-		if err != nil {
-			return err
-		}
-		info.ETag = ETag.Format().String()
-	}
-	return nil
-}
-
-// decryptCacheObjectETag tries to decrypt the ETag saved in encrypted format using the cache KMS
-func decryptCachePartETags(c *cacheMeta) ([]string, error) {
-	// Depending on the SSE type we handle ETags slightly
-	// differently. ETags encrypted with SSE-S3 must be
-	// decrypted first, since the client expects that
-	// a single-part SSE-S3 ETag is equal to the content MD5.
-	//
-	// For all other SSE types, the ETag is not the content MD5.
-	// Therefore, we don't decrypt but only format it.
-	switch kind, ok := crypto.IsEncrypted(c.Meta); {
-	case ok && kind == crypto.S3 && isCacheEncrypted(c.Meta):
-		key, err := crypto.S3.UnsealObjectKey(globalCacheKMS, c.Meta, c.Bucket, c.Object)
-		if err != nil {
-			return nil, err
-		}
-		etags := make([]string, 0, len(c.PartETags))
-		for i := range c.PartETags {
-			ETag, err := etag.Parse(c.PartETags[i])
-			if err != nil {
-				return nil, err
-			}
-			ETag, err = etag.Decrypt(key[:], ETag)
-			if err != nil {
-				return nil, err
-			}
-			etags = append(etags, ETag.Format().String())
-		}
-		return etags, nil
-	case ok && (kind == crypto.S3KMS || kind == crypto.SSEC) && isCacheEncrypted(c.Meta):
-		etags := make([]string, 0, len(c.PartETags))
-		for i := range c.PartETags {
-			ETag, err := etag.Parse(c.PartETags[i])
-			if err != nil {
-				return nil, err
-			}
-			etags = append(etags, ETag.Format().String())
-		}
-		return etags, nil
-	default:
-		return c.PartETags, nil
-	}
-}
-
-func isMetadataSame(m1, m2 map[string]string) bool {
-	if m1 == nil && m2 == nil {
-		return true
-	}
-	if (m1 == nil && m2 != nil) || (m2 == nil && m1 != nil) {
-		return false
-	}
-	if len(m1) != len(m2) {
-		return false
-	}
-	for k1, v1 := range m1 {
-		if v2, ok := m2[k1]; !ok || (v1 != v2) {
-			return false
-		}
-	}
-	return true
-}
-
-type fileScorer struct {
-	saveBytes uint64
-	now       int64
-	maxHits   int
-	// 1/size for consistent score.
-	sizeMult float64
-
-	// queue is a linked list of files we want to delete.
-	// The list is kept sorted according to score, highest at top, lowest at bottom.
-	queue       list.List
-	queuedBytes uint64
-	seenBytes   uint64
-}
-
-type queuedFile struct {
-	name      string
-	versionID string
-	size      uint64
-	score     float64
-}
-
-// newFileScorer allows to collect files to save a specific number of bytes.
-// Each file is assigned a score based on its age, size and number of hits.
-// A list of files is maintained
-func newFileScorer(saveBytes uint64, now int64, maxHits int) (*fileScorer, error) {
-	if saveBytes == 0 {
-		return nil, errors.New("newFileScorer: saveBytes = 0")
-	}
-	if now < 0 {
-		return nil, errors.New("newFileScorer: now < 0")
-	}
-	if maxHits <= 0 {
-		return nil, errors.New("newFileScorer: maxHits <= 0")
-	}
-	f := fileScorer{saveBytes: saveBytes, maxHits: maxHits, now: now, sizeMult: 1 / float64(saveBytes)}
-	f.queue.Init()
-	return &f, nil
-}
-
-func (f *fileScorer) addFile(name string, accTime time.Time, size int64, hits int) {
-	f.addFileWithObjInfo(ObjectInfo{
-		Name:    name,
-		AccTime: accTime,
-		Size:    size,
-	}, hits)
-}
-
-func (f *fileScorer) addFileWithObjInfo(objInfo ObjectInfo, hits int) {
-	// Calculate how much we want to delete this object.
-	file := queuedFile{
-		name:      objInfo.Name,
-		versionID: objInfo.VersionID,
-		size:      uint64(objInfo.Size),
-	}
-	f.seenBytes += uint64(objInfo.Size)
-
-	var score float64
-	if objInfo.ModTime.IsZero() {
-		// Mod time is not available with disk cache use atime.
-		score = float64(f.now - objInfo.AccTime.Unix())
-	} else {
-		// if not used mod time when mod time is available.
-		score = float64(f.now - objInfo.ModTime.Unix())
-	}
-
-	// Size as fraction of how much we want to save, 0->1.
-	szWeight := math.Max(0, (math.Min(1, float64(file.size)*f.sizeMult)))
-	// 0 at f.maxHits, 1 at 0.
-	hitsWeight := (1.0 - math.Max(0, math.Min(1.0, float64(hits)/float64(f.maxHits))))
-	file.score = score * (1 + 0.25*szWeight + 0.25*hitsWeight)
-	// If we still haven't saved enough, just add the file
-	if f.queuedBytes < f.saveBytes {
-		f.insertFile(file)
-		f.trimQueue()
-		return
-	}
-	// If we score less than the worst, don't insert.
-	worstE := f.queue.Back()
-	if worstE != nil && file.score < worstE.Value.(queuedFile).score {
-		return
-	}
-	f.insertFile(file)
-	f.trimQueue()
-}
-
-// adjustSaveBytes allows to adjust the number of bytes to save.
-// This can be used to adjust the count on the fly.
-// Returns true if there still is a need to delete files (n+saveBytes >0),
-// false if no more bytes needs to be saved.
-func (f *fileScorer) adjustSaveBytes(n int64) bool {
-	if f == nil {
-		return false
-	}
-	if int64(f.saveBytes)+n <= 0 {
-		f.saveBytes = 0
-		f.trimQueue()
-		return false
-	}
-	if n < 0 {
-		f.saveBytes -= ^uint64(n - 1)
-	} else {
-		f.saveBytes += uint64(n)
-	}
-	if f.saveBytes == 0 {
-		f.queue.Init()
-		f.saveBytes = 0
-		return false
-	}
-	if n < 0 {
-		f.trimQueue()
-	}
-	return true
-}
-
-// insertFile will insert a file into the list, sorted by its score.
-func (f *fileScorer) insertFile(file queuedFile) {
-	e := f.queue.Front()
-	for e != nil {
-		v := e.Value.(queuedFile)
-		if v.score < file.score {
-			break
-		}
-		e = e.Next()
-	}
-	f.queuedBytes += file.size
-	// We reached the end.
-	if e == nil {
-		f.queue.PushBack(file)
-		return
-	}
-	f.queue.InsertBefore(file, e)
-}
-
-// trimQueue will trim the back of queue and still keep below wantSave.
-func (f *fileScorer) trimQueue() {
-	for {
-		e := f.queue.Back()
-		if e == nil {
-			return
-		}
-		v := e.Value.(queuedFile)
-		if f.queuedBytes-v.size < f.saveBytes {
-			return
-		}
-		f.queue.Remove(e)
-		f.queuedBytes -= v.size
-	}
-}
-
-func (f *fileScorer) purgeFunc(p func(qfile queuedFile)) {
-	e := f.queue.Front()
-	for e != nil {
-		p(e.Value.(queuedFile))
-		e = e.Next()
-	}
-}
-
-// fileNames returns all queued file names.
-func (f *fileScorer) fileNames() []string {
-	res := make([]string, 0, f.queue.Len())
-	e := f.queue.Front()
-	for e != nil {
-		res = append(res, e.Value.(queuedFile).name)
-		e = e.Next()
-	}
-	return res
-}
-
-func (f *fileScorer) reset() {
-	f.queue.Init()
-	f.queuedBytes = 0
-}
-
-func (f *fileScorer) queueString() string {
-	var res strings.Builder
-	e := f.queue.Front()
-	i := 0
-	for e != nil {
-		v := e.Value.(queuedFile)
-		if i > 0 {
-			res.WriteByte('\n')
-		}
-		res.WriteString(fmt.Sprintf("%03d: %s (score: %.3f, bytes: %d)", i, v.name, v.score, v.size))
-		i++
-		e = e.Next()
-	}
-	return res.String()
-}
-
-// bytesToClear() returns the number of bytes to clear to reach low watermark
-// w.r.t quota given disk total and free space, quota in % allocated to cache
-// and low watermark % w.r.t allowed quota.
-// If the high watermark hasn't been reached 0 will be returned.
-func bytesToClear(total, free int64, quotaPct, lowWatermark, highWatermark uint64) uint64 {
-	used := total - free
-	quotaAllowed := total * (int64)(quotaPct) / 100
-	highWMUsage := total * (int64)(highWatermark*quotaPct) / (100 * 100)
-	if used < highWMUsage {
-		return 0
-	}
-	// Return bytes needed to reach low watermark.
-	lowWMUsage := total * (int64)(lowWatermark*quotaPct) / (100 * 100)
-	return (uint64)(math.Min(float64(quotaAllowed), math.Max(0.0, float64(used-lowWMUsage))))
-}
-
-type multiWriter struct {
-	backendWriter io.Writer
-	cacheWriter   *io.PipeWriter
-	pipeClosed    bool
-}
-
-// multiWriter writes to backend and cache - if cache write
-// fails close the pipe, but continue writing to the backend
-func (t *multiWriter) Write(p []byte) (n int, err error) {
-	n, err = t.backendWriter.Write(p)
-	if err == nil && n != len(p) {
-		err = io.ErrShortWrite
-		return
-	}
-	if err != nil {
-		if !t.pipeClosed {
-			t.cacheWriter.CloseWithError(err)
-		}
-		return
-	}
-
-	// ignore errors writing to cache
-	if !t.pipeClosed {
-		_, cerr := t.cacheWriter.Write(p)
-		if cerr != nil {
-			t.pipeClosed = true
-			t.cacheWriter.CloseWithError(cerr)
-		}
-	}
-	return len(p), nil
-}
-
-func cacheMultiWriter(w1 io.Writer, w2 *io.PipeWriter) io.Writer {
-	return &multiWriter{backendWriter: w1, cacheWriter: w2}
-}
-
-// writebackInProgress returns true if writeback commit is not complete
-func writebackInProgress(m map[string]string) bool {
-	if v, ok := m[writeBackStatusHeader]; ok {
-		switch cacheCommitStatus(v) {
-		case CommitPending, CommitFailed:
-			return true
-		}
-	}
-	return false
-}

cmd/disk-cache-utils_test.go

@@ -1,177 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"net/http"
-	"reflect"
-	"testing"
-	"time"
-)
-
-func TestGetCacheControlOpts(t *testing.T) {
-	expiry, _ := time.Parse(http.TimeFormat, "Wed, 21 Oct 2015 07:28:00 GMT")
-
-	testCases := []struct {
-		cacheControlHeaderVal string
-		expiryHeaderVal       time.Time
-		expectedCacheControl  *cacheControl
-		expectedErr           bool
-	}{
-		{"", timeSentinel, nil, false},
-		{"max-age=2592000, public", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
-		{"max-age=2592000, no-store", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, noStore: true, minFresh: 0, expiry: time.Time{}}, false},
-		{"must-revalidate, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
-		{"s-maxAge=2500, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, false},
-		{"s-maxAge=2500, max-age=600", expiry, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 0o7, 28, 0o0, 0o0, time.UTC)}, false},
-		{"s-maxAge=2500, max-age=600s", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, true},
-	}
-
-	for _, testCase := range testCases {
-		t.Run("", func(t *testing.T) {
-			m := make(map[string]string)
-			m["cache-control"] = testCase.cacheControlHeaderVal
-			if !testCase.expiryHeaderVal.Equal(timeSentinel) {
-				m["expires"] = testCase.expiryHeaderVal.String()
-			}
-			c := cacheControlOpts(ObjectInfo{UserDefined: m, Expires: testCase.expiryHeaderVal})
-			if testCase.expectedErr && (c != nil) {
-				t.Errorf("expected err, got <nil>")
-			}
-			if !testCase.expectedErr && !reflect.DeepEqual(c, testCase.expectedCacheControl) {
-				t.Errorf("expected %v, got %v", testCase.expectedCacheControl, c)
-			}
-		})
-	}
-}
-
-func TestIsMetadataSame(t *testing.T) {
-	testCases := []struct {
-		m1       map[string]string
-		m2       map[string]string
-		expected bool
-	}{
-		{nil, nil, true},
-		{nil, map[string]string{}, false},
-		{map[string]string{"k": "v"}, map[string]string{"k": "v"}, true},
-		{map[string]string{"k": "v"}, map[string]string{"a": "b"}, false},
-		{map[string]string{"k1": "v1", "k2": "v2"}, map[string]string{"k1": "v1", "k2": "v1"}, false},
-		{map[string]string{"k1": "v1", "k2": "v2"}, map[string]string{"k1": "v1", "k2": "v2"}, true},
-		{map[string]string{"K1": "v1", "k2": "v2"}, map[string]string{"k1": "v1", "k2": "v2"}, false},
-		{map[string]string{"k1": "v1", "k2": "v2", "k3": "v3"}, map[string]string{"k1": "v1", "k2": "v2"}, false},
-	}
-
-	for i, testCase := range testCases {
-		actual := isMetadataSame(testCase.m1, testCase.m2)
-		if testCase.expected != actual {
-			t.Errorf("test %d expected %v, got %v", i, testCase.expected, actual)
-		}
-	}
-}
-
-func TestNewFileScorer(t *testing.T) {
-	fs, err := newFileScorer(1000, time.Now().Unix(), 10)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(fs.fileNames()) != 0 {
-		t.Fatal("non zero files??")
-	}
-	now := time.Now()
-	fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
-	fs.addFile("older", now.Add(-time.Hour), 1000, 10)
-	if !reflect.DeepEqual(fs.fileNames(), []string{"older"}) {
-		t.Fatal("unexpected file list", fs.queueString())
-	}
-	fs.reset()
-	fs.addFile("bigger", now.Add(-time.Minute), 2000, 10)
-	fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
-	if !reflect.DeepEqual(fs.fileNames(), []string{"bigger"}) {
-		t.Fatal("unexpected file list", fs.queueString())
-	}
-	fs.reset()
-	fs.addFile("less", now.Add(-time.Minute), 1000, 5)
-	fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
-	if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
-		t.Fatal("unexpected file list", fs.queueString())
-	}
-	fs.reset()
-	fs.addFile("small", now.Add(-time.Minute), 200, 10)
-	fs.addFile("medium", now.Add(-time.Minute), 300, 10)
-	if !reflect.DeepEqual(fs.fileNames(), []string{"medium", "small"}) {
-		t.Fatal("unexpected file list", fs.queueString())
-	}
-	fs.addFile("large", now.Add(-time.Minute), 700, 10)
-	fs.addFile("xsmol", now.Add(-time.Minute), 7, 10)
-	if !reflect.DeepEqual(fs.fileNames(), []string{"large", "medium"}) {
-		t.Fatal("unexpected file list", fs.queueString())
-	}
-
-	fs.reset()
-	fs.addFile("less", now.Add(-time.Minute), 500, 5)
-	fs.addFile("recent", now.Add(-time.Minute), 500, 10)
-	if !fs.adjustSaveBytes(-500) {
-		t.Fatal("we should still need more bytes, got false")
-	}
-	// We should only need 500 bytes now.
-	if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
-		t.Fatal("unexpected file list", fs.queueString())
-	}
-	if fs.adjustSaveBytes(-500) {
-		t.Fatal("we shouldn't need any more bytes, got true")
-	}
-	fs, err = newFileScorer(1000, time.Now().Unix(), 10)
-	if err != nil {
-		t.Fatal(err)
-	}
-	fs.addFile("bigger", now.Add(-time.Minute), 50, 10)
-	// sorting should be consistent after adjusting savebytes.
-	fs.adjustSaveBytes(-800)
-	fs.addFile("smaller", now.Add(-time.Minute), 40, 10)
-	if !reflect.DeepEqual(fs.fileNames(), []string{"bigger", "smaller"}) {
-		t.Fatal("unexpected file list", fs.queueString())
-	}
-}
-
-func TestBytesToClear(t *testing.T) {
-	testCases := []struct {
-		total         int64
-		free          int64
-		quotaPct      uint64
-		watermarkLow  uint64
-		watermarkHigh uint64
-		expected      uint64
-	}{
-		{total: 1000, free: 800, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 0},
-		{total: 1000, free: 200, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 400},
-		{total: 1000, free: 400, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 240},
-		{total: 1000, free: 600, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 40},
-		{total: 1000, free: 600, quotaPct: 40, watermarkLow: 70, watermarkHigh: 70, expected: 120},
-		{total: 1000, free: 1000, quotaPct: 90, watermarkLow: 70, watermarkHigh: 70, expected: 0},
-
-		// High not yet reached..
-		{total: 1000, free: 250, quotaPct: 100, watermarkLow: 50, watermarkHigh: 90, expected: 0},
-		{total: 1000, free: 250, quotaPct: 100, watermarkLow: 50, watermarkHigh: 90, expected: 0},
-	}
-	for i, tc := range testCases {
-		toClear := bytesToClear(tc.total, tc.free, tc.quotaPct, tc.watermarkLow, tc.watermarkHigh)
-		if tc.expected != toClear {
-			t.Errorf("test %d expected %v, got %v", i, tc.expected, toClear)
-		}
-	}
-}

cmd/disk-cache_test.go

@@ -1,74 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"testing"
-	"time"
-)
-
-// Tests ToObjectInfo function.
-func TestCacheMetadataObjInfo(t *testing.T) {
-	m := cacheMeta{Meta: nil}
-	objInfo := m.ToObjectInfo()
-	if objInfo.Size != 0 {
-		t.Fatal("Unexpected object info value for Size", objInfo.Size)
-	}
-	if !objInfo.ModTime.Equal(time.Time{}) {
-		t.Fatal("Unexpected object info value for ModTime ", objInfo.ModTime)
-	}
-	if objInfo.IsDir {
-		t.Fatal("Unexpected object info value for IsDir", objInfo.IsDir)
-	}
-	if !objInfo.Expires.IsZero() {
-		t.Fatal("Unexpected object info value for Expires ", objInfo.Expires)
-	}
-}
-
-// test wildcard patterns for excluding entries from cache
-func TestCacheExclusion(t *testing.T) {
-	cobjects := &cacheObjects{
-		cache: nil,
-	}
-
-	testCases := []struct {
-		bucketName     string
-		objectName     string
-		excludePattern string
-		expectedResult bool
-	}{
-		{"testbucket", "testobjectmatch", "testbucket/testobj*", true},
-		{"testbucket", "testobjectnomatch", "testbucet/testobject*", false},
-		{"testbucket", "testobject/pref1/obj1", "*/*", true},
-		{"testbucket", "testobject/pref1/obj1", "*/pref1/*", true},
-		{"testbucket", "testobject/pref1/obj1", "testobject/*", false},
-		{"photos", "image1.jpg", "*.jpg", true},
-		{"photos", "europe/paris/seine.jpg", "seine.jpg", false},
-		{"photos", "europe/paris/seine.jpg", "*/seine.jpg", true},
-		{"phil", "z/likes/coffee", "*/likes/*", true},
-		{"failbucket", "no/slash/prefixes", "/failbucket/no/", false},
-		{"failbucket", "no/slash/prefixes", "/failbucket/no/*", false},
-	}
-
-	for i, testCase := range testCases {
-		cobjects.exclude = []string{testCase.excludePattern}
-		if cobjects.isCacheExclude(testCase.bucketName, testCase.objectName) != testCase.expectedResult {
-			t.Fatal("Cache exclusion test failed for case ", i)
-		}
-	}
-}

cmd/encryption-v1.go

@@ -478,10 +478,7 @@ func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, m
 func decryptObjectMeta(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) {
 	switch kind, _ := crypto.IsEncrypted(metadata); kind {
 	case crypto.S3:
-		var KMS kms.KMS = GlobalKMS
-		if isCacheEncrypted(metadata) {
-			KMS = globalCacheKMS
-		}
+		KMS := GlobalKMS
 		if KMS == nil {
 			return nil, errKMSNotConfigured
 		}

cmd/erasure-multipart.go

@@ -40,7 +40,6 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/mimedb"
 	"github.com/minio/pkg/v2/sync/errgroup"
-	uatomic "go.uber.org/atomic"
 )
 
 func (er erasureObjects) getUploadIDDir(bucket, object, uploadID string) string {
@@ -273,9 +272,15 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 	if len(disks) == 0 {
 		// using er.getLoadBalancedLocalDisks() has one side-affect where
 		// on a pooled setup all disks are remote, add a fallback
-		disks = er.getOnlineDisks()
+		disks = er.getDisks()
 	}
 	for _, disk = range disks {
+		if disk == nil {
+			continue
+		}
+		if !disk.IsOnline() {
+			continue
+		}
 		uploadIDs, err = disk.ListDir(ctx, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
 		if err != nil {
 			if errors.Is(err, errDiskNotFound) {
@@ -399,47 +404,31 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	// If we have offline disks upgrade the number of erasure codes for this object.
 	parityOrig := parityDrives
 
-	atomicParityDrives := uatomic.NewInt64(0)
-	atomicOfflineDrives := uatomic.NewInt64(0)
-
-	// Start with current parityDrives
-	atomicParityDrives.Store(int64(parityDrives))
-
-	var wg sync.WaitGroup
+	var offlineDrives int
 	for _, disk := range onlineDisks {
 		if disk == nil {
-			atomicParityDrives.Inc()
-			atomicOfflineDrives.Inc()
+			parityDrives++
+			offlineDrives++
 			continue
 		}
 		if !disk.IsOnline() {
-			atomicParityDrives.Inc()
-			atomicOfflineDrives.Inc()
+			parityDrives++
+			offlineDrives++
 			continue
 		}
-		wg.Add(1)
-		go func(disk StorageAPI) {
-			defer wg.Done()
-			di, err := disk.DiskInfo(ctx, false)
-			if err != nil || di.ID == "" {
-				atomicOfflineDrives.Inc()
-				atomicParityDrives.Inc()
-			}
-		}(disk)
 	}
-	wg.Wait()
 
-	if int(atomicOfflineDrives.Load()) >= (len(onlineDisks)+1)/2 {
+	if offlineDrives >= (len(onlineDisks)+1)/2 {
 		// if offline drives are more than 50% of the drives
 		// we have no quorum, we shouldn't proceed just
 		// fail at that point.
 		return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
 	}
 
-	parityDrives = int(atomicParityDrives.Load())
 	if parityDrives >= len(onlineDisks)/2 {
 		parityDrives = len(onlineDisks) / 2
 	}
+
 	if parityOrig != parityDrives {
 		userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
 	}

cmd/erasure-object.go

@@ -47,7 +47,6 @@ import (
 	"github.com/minio/pkg/v2/sync/errgroup"
 	"github.com/minio/pkg/v2/wildcard"
 	"github.com/tinylib/msgp/msgp"
-	uatomic "go.uber.org/atomic"
 )
 
 // list all errors which can be ignored in object operations.
@@ -490,6 +489,24 @@ func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID
 	auditLogInternal(ctx, opts)
 }
 
+func joinErrors(errs ...error) error {
+	s := make([]string, 0, len(errs))
+	nonNilErrs := make([]any, 0, len(errs))
+	for _, err := range errs {
+		if err == nil {
+			continue
+		}
+		s = append(s, "[%w]")
+		nonNilErrs = append(nonNilErrs, err)
+	}
+	// If all the errors were nil, return nil.
+	if len(nonNilErrs) == 0 {
+		return nil
+	}
+	allErrs := strings.Join(s, "\n")
+	return fmt.Errorf(allErrs, nonNilErrs...)
+}
+
 func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrs []error, opts ObjectOptions) (FileInfo, error) {
 	_, file, line, cok := runtime.Caller(1)
 	var err error
@@ -498,8 +515,8 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		tags := make(map[string]interface{}, 4)
 		tags["set"] = er.setIndex
 		tags["pool"] = er.poolIndex
-		tags["merrs"] = errors.Join(errs...)
-		tags["derrs"] = errors.Join(dataErrs...)
+		tags["merrs"] = joinErrors(errs...)     // errors.Join(errs...)
+		tags["derrs"] = joinErrors(dataErrs...) // errors.Join(dataErrs...)
 		if m.IsValid() {
 			tags["size"] = m.Size
 			tags["mtime"] = m.ModTime.Format(http.TimeFormat)
@@ -1097,47 +1114,31 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		// If we have offline disks upgrade the number of erasure codes for this object.
 		parityOrig := parityDrives
 
-		atomicParityDrives := uatomic.NewInt64(0)
-		atomicOfflineDrives := uatomic.NewInt64(0)
-
-		// Start with current parityDrives
-		atomicParityDrives.Store(int64(parityDrives))
-
-		var wg sync.WaitGroup
+		var offlineDrives int
 		for _, disk := range storageDisks {
 			if disk == nil {
-				atomicParityDrives.Inc()
-				atomicOfflineDrives.Inc()
+				parityDrives++
+				offlineDrives++
 				continue
 			}
 			if !disk.IsOnline() {
-				atomicParityDrives.Inc()
-				atomicOfflineDrives.Inc()
+				parityDrives++
+				offlineDrives++
 				continue
 			}
-			wg.Add(1)
-			go func(disk StorageAPI) {
-				defer wg.Done()
-				di, err := disk.DiskInfo(ctx, false)
-				if err != nil || di.ID == "" {
-					atomicOfflineDrives.Inc()
-					atomicParityDrives.Inc()
-				}
-			}(disk)
 		}
-		wg.Wait()
 
-		if int(atomicOfflineDrives.Load()) >= (len(storageDisks)+1)/2 {
+		if offlineDrives >= (len(storageDisks)+1)/2 {
 			// if offline drives are more than 50% of the drives
 			// we have no quorum, we shouldn't proceed just
 			// fail at that point.
 			return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object)
 		}
 
-		parityDrives = int(atomicParityDrives.Load())
 		if parityDrives >= len(storageDisks)/2 {
 			parityDrives = len(storageDisks) / 2
 		}
+
 		if parityOrig != parityDrives {
 			userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
 		}
@@ -1628,10 +1629,12 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 
 	var lc *lifecycle.Lifecycle
 	var rcfg lock.Retention
+	var replcfg *replication.Config
 	if opts.Expiration.Expire {
 		// Check if the current bucket has a configured lifecycle policy
 		lc, _ = globalLifecycleSys.Get(bucket)
 		rcfg, _ = globalBucketObjectLockSys.Get(bucket)
+		replcfg, _ = getReplicationConfig(ctx, bucket)
 	}
 
 	// expiration attempted on a bucket with no lifecycle
@@ -1684,7 +1687,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 
 	if opts.Expiration.Expire {
 		if gerr == nil {
-			evt := evalActionFromLifecycle(ctx, *lc, rcfg, goi)
+			evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi)
 			var isErr bool
 			switch evt.Action {
 			case lifecycle.NoneAction:

cmd/erasure-server-pool-decom.go

@@ -160,6 +160,9 @@ func (ps PoolStatus) Clone() PoolStatus {
 type poolMeta struct {
 	Version int          `msg:"v"`
 	Pools   []PoolStatus `msg:"pls"`
+
+	// Value should not be saved when we have not loaded anything yet.
+	dontSave bool `msg:"-"`
 }
 
 // A decommission resumable tells us if decommission is worth
@@ -308,6 +311,10 @@ func (p *poolMeta) Decommission(idx int, pi poolSpaceInfo) error {
 }
 
 func (p poolMeta) IsSuspended(idx int) bool {
+	if idx >= len(p.Pools) {
+		// We don't really know if the pool is suspended or not, since it doesn't exist.
+		return false
+	}
 	return p.Pools[idx].Decommission != nil
 }
 
@@ -445,6 +452,9 @@ func (p *poolMeta) updateAfter(ctx context.Context, idx int, pools []*erasureSet
 }
 
 func (p poolMeta) save(ctx context.Context, pools []*erasureSets) error {
+	if p.dontSave {
+		return nil
+	}
 	data := make([]byte, 4, p.Msgsize()+4)
 
 	// Initialize the header.
@@ -499,36 +509,17 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 
 	// if no update is needed return right away.
 	if !update {
+		z.poolMetaMutex.Lock()
 		z.poolMeta = meta
+		z.poolMetaMutex.Unlock()
 	} else {
-		newMeta := poolMeta{} // to update write poolMeta fresh.
-		// looks like new pool was added we need to update,
-		// or this is a fresh installation (or an existing
-		// installation with pool removed)
-		newMeta.Version = poolMetaVersion
-		for idx, pool := range z.serverPools {
-			var skip bool
-			for _, currentPool := range meta.Pools {
-				// Preserve any current pool status.
-				if currentPool.CmdLine == pool.endpoints.CmdLine {
-					newMeta.Pools = append(newMeta.Pools, currentPool)
-					skip = true
-					break
-				}
-			}
-			if skip {
-				continue
-			}
-			newMeta.Pools = append(newMeta.Pools, PoolStatus{
-				CmdLine:    pool.endpoints.CmdLine,
-				ID:         idx,
-				LastUpdate: UTCNow(),
-			})
-		}
+		newMeta := newPoolMeta(z, meta)
 		if err = newMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
+		z.poolMetaMutex.Lock()
 		z.poolMeta = newMeta
+		z.poolMetaMutex.Unlock()
 	}
 
 	pools := meta.returnResumablePools()
@@ -568,6 +559,34 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 	return nil
 }
 
+func newPoolMeta(z *erasureServerPools, prevMeta poolMeta) poolMeta {
+	newMeta := poolMeta{} // to update write poolMeta fresh.
+	// looks like new pool was added we need to update,
+	// or this is a fresh installation (or an existing
+	// installation with pool removed)
+	newMeta.Version = poolMetaVersion
+	for idx, pool := range z.serverPools {
+		var skip bool
+		for _, currentPool := range prevMeta.Pools {
+			// Preserve any current pool status.
+			if currentPool.CmdLine == pool.endpoints.CmdLine {
+				newMeta.Pools = append(newMeta.Pools, currentPool)
+				skip = true
+				break
+			}
+		}
+		if skip {
+			continue
+		}
+		newMeta.Pools = append(newMeta.Pools, PoolStatus{
+			CmdLine:    pool.endpoints.CmdLine,
+			ID:         idx,
+			LastUpdate: UTCNow(),
+		})
+	}
+	return newMeta
+}
+
 func (z *erasureServerPools) IsDecommissionRunning() bool {
 	z.poolMetaMutex.RLock()
 	defer z.poolMetaMutex.RUnlock()
@@ -734,6 +753,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 
 	// Check if bucket is object locked.
 	lr, _ := globalBucketObjectLockSys.Get(bi.Name)
+	rcfg, _ := getReplicationConfig(ctx, bi.Name)
 
 	for setIdx, set := range pool.sets {
 		set := set
@@ -745,7 +765,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			versioned := vc != nil && vc.Versioned(object)
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)
 
-			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+			evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
 			switch {
 			case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
 				applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom)
@@ -1002,8 +1022,15 @@ func (m *decomMetrics) log(d decomMetric, poolIdx int, paths ...string) func(err
 
 func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx int) error {
 	pool := z.serverPools[idx]
-	for _, bucket := range z.poolMeta.PendingBuckets(idx) {
-		if z.poolMeta.isBucketDecommissioned(idx, bucket.String()) {
+	z.poolMetaMutex.RLock()
+	pending := z.poolMeta.PendingBuckets(idx)
+	z.poolMetaMutex.RUnlock()
+
+	for _, bucket := range pending {
+		z.poolMetaMutex.RLock()
+		isDecommissioned := z.poolMeta.isBucketDecommissioned(idx, bucket.String())
+		z.poolMetaMutex.RUnlock()
+		if isDecommissioned {
 			if serverDebugLog {
 				console.Debugln("decommission: already done, moving on", bucket)
 			}
@@ -1048,6 +1075,7 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error
 
 			// Check if bucket is object locked.
 			lr, _ := globalBucketObjectLockSys.Get(bi.Name)
+			rcfg, _ := getReplicationConfig(ctx, bi.Name)
 
 			filterLifecycle := func(bucket, object string, fi FileInfo) bool {
 				if lc == nil {
@@ -1056,7 +1084,7 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error
 				versioned := vc != nil && vc.Versioned(object)
 				objInfo := fi.ToObjectInfo(bucket, object, versioned)
 
-				evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+				evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
 				switch {
 				case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
 					applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom)

cmd/erasure-server-pool-rebalance.go

@@ -440,6 +440,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 	lc, _ := globalLifecycleSys.Get(bucket)
 	// Check if bucket is object locked.
 	lr, _ := globalBucketObjectLockSys.Get(bucket)
+	rcfg, _ := getReplicationConfig(ctx, bucket)
 
 	pool := z.serverPools[poolIdx]
 	const envRebalanceWorkers = "_MINIO_REBALANCE_WORKERS"
@@ -467,7 +468,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 			versioned := vc != nil && vc.Versioned(object)
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)
 
-			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+			evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
 			if evt.Action.Delete() {
 				globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_Rebal)
 				return true

cmd/erasure-server-pool.go

@@ -25,6 +25,7 @@ import (
 	"io"
 	"math/rand"
 	"net/http"
+	"path"
 	"sort"
 	"strconv"
 	"strings"
@@ -142,6 +143,12 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 
 	z.decommissionCancelers = make([]context.CancelFunc, len(z.serverPools))
 
+	// Initialize the pool meta, but set it to not save.
+	// When z.Init below has loaded the poolmeta will be initialized,
+	// and allowed to save.
+	z.poolMeta = newPoolMeta(z, poolMeta{})
+	z.poolMeta.dontSave = true
+
 	// initialize the object layer.
 	setObjectLayer(z)
 
@@ -1323,6 +1330,85 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	}
 	opts.setBucketMeta(ctx)
 
+	ri := logger.GetReqInfo(ctx)
+	hadoop := ri != nil && strings.Contains(ri.UserAgent, `Hadoop `) && strings.Contains(ri.UserAgent, "scala/")
+	matches := func() bool {
+		if prefix == "" {
+			return false
+		}
+		// List of standard files supported by s3a
+		// that involves a List() on a directory
+		// where directory is actually an object on
+		// namespace.
+		for _, k := range []string{
+			"_SUCCESS/",
+			".parquet/",
+			".csv/",
+			".json/",
+			".avro/",
+			".orc/",
+			".txt/",
+			// Add any other files in future
+		} {
+			if strings.HasSuffix(prefix, k) {
+				return true
+			}
+		}
+		return false
+	}
+	if hadoop && matches() && delimiter == SlashSeparator && maxKeys == 2 && marker == "" {
+		// Optimization for Spark/Hadoop workload where spark sends a garbage
+		// request of this kind
+		//
+		// GET /testbucket/?list-type=2&delimiter=%2F&max-keys=2&prefix=parquet%2F_SUCCESS%2F&fetch-owner=false
+		//
+		// Here spark is expecting that the List() return empty instead, so from MinIO's point
+		// of view if we simply do a GetObjectInfo() on this prefix by treating it as an object
+		// We save a lot of calls over the network.
+		//
+		// This happens repeatedly for all objects that are created concurrently() avoiding this
+		// as a List() call is an important performance improvement.
+		//
+		// Spark based s3a committers are a  big enough use-case to have this optimization.
+		//
+		// A sample code to see the improvements is as follows, this sample code is
+		// simply a read on JSON from MinIO and write it back as "parquet".
+		//
+		// import org.apache.spark.sql.SparkSession
+		// import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
+		// object SparkJSONRead {
+		//   def main(args: Array[String]): Unit = {
+		//     val spark:SparkSession = SparkSession.builder()
+		//       .appName("SparkByExample")
+		//       .master("local[1]").getOrCreate()
+		//
+		//     spark.sparkContext.setLogLevel("ERROR")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.endpoint", "http://minio-lb:9000")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.path.style.access", "true")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.access.key", "minioadmin")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.secret.key", "minioadmin")
+		//
+		//     val df = spark.read.json("s3a://testbucket/s3.json")
+		//
+		//     df.write.parquet("s3a://testbucket/parquet/")
+		//   }
+		// }
+		objInfo, err := z.GetObjectInfo(ctx, bucket, path.Dir(prefix), ObjectOptions{NoLock: true})
+		if err == nil {
+			if opts.Lifecycle != nil {
+				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, opts.Replication.Config, objInfo)
+				if evt.Action.Delete() {
+					globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects)
+					if !evt.Action.DeleteRestored() {
+						// Skip entry if ILM action was DeleteVersionAction or DeleteAction
+						return loi, nil
+					}
+				}
+			}
+			return loi, nil
+		}
+	}
+
 	if len(prefix) > 0 && maxKeys == 1 && marker == "" {
 		// Optimization for certain applications like
 		// - Cohesity
@@ -1334,7 +1420,7 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		objInfo, err := z.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true})
 		if err == nil {
 			if opts.Lifecycle != nil {
-				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo)
+				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, opts.Replication.Config, objInfo)
 				if evt.Action.Delete() {
 					globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects)
 					if !evt.Action.DeleteRestored() {
@@ -2371,12 +2457,12 @@ func (z *erasureServerPools) GetObjectTags(ctx context.Context, bucket, object s
 		return z.serverPools[0].GetObjectTags(ctx, bucket, object, opts)
 	}
 
-	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
+	oi, _, err := z.getLatestObjectInfoWithIdx(ctx, bucket, object, opts)
 	if err != nil {
 		return nil, err
 	}
 
-	return z.serverPools[idx].GetObjectTags(ctx, bucket, object, opts)
+	return tags.ParseObjectTags(oi.UserTags)
 }
 
 // TransitionObject - transition object content to target tier.

cmd/format-disk-cache.go

@@ -1,498 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"path"
-	"path/filepath"
-	"reflect"
-	"strings"
-
-	jsoniter "github.com/json-iterator/go"
-	"github.com/minio/minio/internal/logger"
-	"github.com/minio/sio"
-)
-
-const (
-	// Represents Cache format json holding details on all other cache drives in use.
-	formatCache = "cache"
-
-	// formatCacheV1.Cache.Version
-	formatCacheVersionV1 = "1"
-	formatCacheVersionV2 = "2"
-
-	formatMetaVersion1 = "1"
-
-	formatCacheV1DistributionAlgo = "CRCMOD"
-)
-
-// Represents the current cache structure with list of
-// disks comprising the disk cache
-// formatCacheV1 - structure holds format config version '1'.
-type formatCacheV1 struct {
-	formatMetaV1
-	Cache struct {
-		Version string `json:"version"` // Version of 'cache' format.
-		This    string `json:"this"`    // This field carries assigned disk uuid.
-		// Disks field carries the input disk order generated the first
-		// time when fresh disks were supplied.
-		Disks []string `json:"disks"`
-		// Distribution algorithm represents the hashing algorithm
-		// to pick the right set index for an object.
-		DistributionAlgo string `json:"distributionAlgo"`
-	} `json:"cache"` // Cache field holds cache format.
-}
-
-// formatCacheV2 is same as formatCacheV1
-type formatCacheV2 = formatCacheV1
-
-// Used to detect the version of "cache" format.
-type formatCacheVersionDetect struct {
-	Cache struct {
-		Version string `json:"version"`
-	} `json:"cache"`
-}
-
-// Return a slice of format, to be used to format uninitialized disks.
-func newFormatCacheV2(drives []string) []*formatCacheV2 {
-	diskCount := len(drives)
-	disks := make([]string, diskCount)
-
-	formats := make([]*formatCacheV2, diskCount)
-
-	for i := 0; i < diskCount; i++ {
-		format := &formatCacheV2{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV2
-		format.Cache.DistributionAlgo = formatCacheV1DistributionAlgo
-		format.Cache.This = mustGetUUID()
-		formats[i] = format
-		disks[i] = formats[i].Cache.This
-	}
-	for i := 0; i < diskCount; i++ {
-		format := formats[i]
-		format.Cache.Disks = disks
-	}
-	return formats
-}
-
-// Returns formatCache.Cache.Version
-func formatCacheGetVersion(r io.ReadSeeker) (string, error) {
-	format := &formatCacheVersionDetect{}
-	if err := jsonLoad(r, format); err != nil {
-		return "", err
-	}
-	return format.Cache.Version, nil
-}
-
-// Creates a new cache format.json if unformatted.
-func createFormatCache(fsFormatPath string, format *formatCacheV1) error {
-	// open file using READ & WRITE permission
-	file, err := os.OpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0o666)
-	if err != nil {
-		return err
-	}
-	// Close the locked file upon return.
-	defer file.Close()
-
-	fi, err := file.Stat()
-	if err != nil {
-		return err
-	}
-	if fi.Size() != 0 {
-		// format.json already got created because of another minio process's createFormatCache()
-		return nil
-	}
-	return jsonSave(file, format)
-}
-
-// This function creates a cache format file on disk and returns a slice
-// of format cache config
-func initFormatCache(ctx context.Context, drives []string) (formats []*formatCacheV2, err error) {
-	nformats := newFormatCacheV2(drives)
-	for i, drive := range drives {
-		if err = os.MkdirAll(pathJoin(drive, minioMetaBucket), 0o777); err != nil {
-			logger.GetReqInfo(ctx).AppendTags("drive", drive)
-			logger.LogIf(ctx, err)
-			return nil, err
-		}
-		cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
-		// Fresh disk - create format.json for this cfs
-		if err = createFormatCache(cacheFormatPath, nformats[i]); err != nil {
-			logger.GetReqInfo(ctx).AppendTags("drive", drive)
-			logger.LogIf(ctx, err)
-			return nil, err
-		}
-	}
-	return nformats, nil
-}
-
-func loadFormatCache(ctx context.Context, drives []string) ([]*formatCacheV2, bool, error) {
-	formats := make([]*formatCacheV2, len(drives))
-	var formatV2 *formatCacheV2
-	migrating := false
-	for i, drive := range drives {
-		cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
-		f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0o666)
-		if err != nil {
-			if osIsNotExist(err) {
-				continue
-			}
-			logger.LogIf(ctx, err)
-			return nil, migrating, err
-		}
-		defer f.Close()
-		format, err := formatMetaCacheV1(f)
-		if err != nil {
-			continue
-		}
-		formatV2 = format
-		if format.Cache.Version != formatCacheVersionV2 {
-			migrating = true
-		}
-		formats[i] = formatV2
-	}
-	return formats, migrating, nil
-}
-
-// unmarshalls the cache format.json into formatCacheV1
-func formatMetaCacheV1(r io.ReadSeeker) (*formatCacheV1, error) {
-	format := &formatCacheV1{}
-	if err := jsonLoad(r, format); err != nil {
-		return nil, err
-	}
-	return format, nil
-}
-
-func checkFormatCacheValue(format *formatCacheV2, migrating bool) error {
-	if format.Format != formatCache {
-		return fmt.Errorf("Unsupported cache format [%s] found", format.Format)
-	}
-
-	// during migration one or more cache drive(s) formats can be out of sync
-	if migrating {
-		// Validate format version and format type.
-		if format.Version != formatMetaVersion1 {
-			return fmt.Errorf("Unsupported version of cache format [%s] found", format.Version)
-		}
-		if format.Cache.Version != formatCacheVersionV2 && format.Cache.Version != formatCacheVersionV1 {
-			return fmt.Errorf("Unsupported Cache backend format found [%s]", format.Cache.Version)
-		}
-		return nil
-	}
-	// Validate format version and format type.
-	if format.Version != formatMetaVersion1 {
-		return fmt.Errorf("Unsupported version of cache format [%s] found", format.Version)
-	}
-	if format.Cache.Version != formatCacheVersionV2 {
-		return fmt.Errorf("Unsupported Cache backend format found [%s]", format.Cache.Version)
-	}
-	return nil
-}
-
-func checkFormatCacheValues(migrating bool, formats []*formatCacheV2) (int, error) {
-	for i, formatCache := range formats {
-		if formatCache == nil {
-			continue
-		}
-		if err := checkFormatCacheValue(formatCache, migrating); err != nil {
-			return i, err
-		}
-		if len(formats) != len(formatCache.Cache.Disks) {
-			return i, fmt.Errorf("Expected number of cache drives %d , got  %d",
-				len(formatCache.Cache.Disks), len(formats))
-		}
-	}
-	return -1, nil
-}
-
-// checkCacheDisksConsistency - checks if "This" disk uuid on each disk is consistent with all "Disks" slices
-// across disks.
-func checkCacheDiskConsistency(formats []*formatCacheV2) error {
-	disks := make([]string, len(formats))
-	// Collect currently available disk uuids.
-	for index, format := range formats {
-		if format == nil {
-			disks[index] = ""
-			continue
-		}
-		disks[index] = format.Cache.This
-	}
-	for i, format := range formats {
-		if format == nil {
-			continue
-		}
-		j := findCacheDiskIndex(disks[i], format.Cache.Disks)
-		if j == -1 {
-			return fmt.Errorf("UUID on positions %d:%d do not match with , expected %s", i, j, disks[i])
-		}
-		if i != j {
-			return fmt.Errorf("UUID on positions %d:%d do not match with , expected %s got %s", i, j, disks[i], format.Cache.Disks[j])
-		}
-	}
-	return nil
-}
-
-// checkCacheDisksSliceConsistency - validate cache Disks order if they are consistent.
-func checkCacheDisksSliceConsistency(formats []*formatCacheV2) error {
-	var sentinelDisks []string
-	// Extract first valid Disks slice.
-	for _, format := range formats {
-		if format == nil {
-			continue
-		}
-		sentinelDisks = format.Cache.Disks
-		break
-	}
-	for _, format := range formats {
-		if format == nil {
-			continue
-		}
-		currentDisks := format.Cache.Disks
-		if !reflect.DeepEqual(sentinelDisks, currentDisks) {
-			return errors.New("inconsistent cache drives found")
-		}
-	}
-	return nil
-}
-
-// findCacheDiskIndex returns position of cache disk in JBOD.
-func findCacheDiskIndex(disk string, disks []string) int {
-	for index, uuid := range disks {
-		if uuid == disk {
-			return index
-		}
-	}
-	return -1
-}
-
-// validate whether cache drives order has changed
-func validateCacheFormats(ctx context.Context, migrating bool, formats []*formatCacheV2) error {
-	count := 0
-	for _, format := range formats {
-		if format == nil {
-			count++
-		}
-	}
-	if count == len(formats) {
-		return errors.New("Cache format files missing on all drives")
-	}
-	if _, err := checkFormatCacheValues(migrating, formats); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-	if err := checkCacheDisksSliceConsistency(formats); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-	err := checkCacheDiskConsistency(formats)
-	logger.LogIf(ctx, err)
-	return err
-}
-
-// return true if all of the list of cache drives are
-// fresh disks
-func cacheDrivesUnformatted(drives []string) bool {
-	count := 0
-	for _, drive := range drives {
-		cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
-		if _, err := os.Stat(cacheFormatPath); osIsNotExist(err) {
-			count++
-		}
-	}
-	return count == len(drives)
-}
-
-// create format.json for each cache drive if fresh disk or load format from disk
-// Then validate the format for all drives in the cache to ensure order
-// of cache drives has not changed.
-func loadAndValidateCacheFormat(ctx context.Context, drives []string) (formats []*formatCacheV2, migrating bool, err error) {
-	if cacheDrivesUnformatted(drives) {
-		formats, err = initFormatCache(ctx, drives)
-	} else {
-		formats, migrating, err = loadFormatCache(ctx, drives)
-	}
-	if err != nil {
-		return nil, false, err
-	}
-	if err = validateCacheFormats(ctx, migrating, formats); err != nil {
-		return nil, false, err
-	}
-	return formats, migrating, nil
-}
-
-// reads cached object on disk and writes it back after adding bitrot
-// hashsum per block as per the new disk cache format.
-func migrateCacheData(ctx context.Context, c *diskCache, bucket, object, oldfile, destDir string, metadata map[string]string) error {
-	st, err := os.Stat(oldfile)
-	if err != nil {
-		err = osErrToFileErr(err)
-		return err
-	}
-	readCloser, err := readCacheFileStream(oldfile, 0, st.Size())
-	if err != nil {
-		return err
-	}
-	defer readCloser.Close()
-	var reader io.Reader = readCloser
-
-	actualSize := uint64(st.Size())
-	if globalCacheKMS != nil {
-		reader, err = newCacheEncryptReader(ctx, readCloser, bucket, object, metadata)
-		if err != nil {
-			return err
-		}
-		actualSize, _ = sio.EncryptedSize(uint64(st.Size()))
-	}
-	_, _, err = c.bitrotWriteToCache(destDir, cacheDataFile, reader, actualSize)
-	return err
-}
-
-// migrate cache contents from old cacheFS format to new backend format
-// new format is flat
-//
-//	sha(bucket,object)/  <== dir name
-//	    - part.1         <== data
-//	    - cache.json     <== metadata
-func migrateOldCache(ctx context.Context, c *diskCache) error {
-	oldCacheBucketsPath := path.Join(c.dir, minioMetaBucket, "buckets")
-	cacheFormatPath := pathJoin(c.dir, minioMetaBucket, formatConfigFile)
-
-	if _, err := os.Stat(oldCacheBucketsPath); err != nil {
-		// remove .minio.sys sub directories
-		removeAll(path.Join(c.dir, minioMetaBucket, "multipart"))
-		removeAll(path.Join(c.dir, minioMetaBucket, "tmp"))
-		removeAll(path.Join(c.dir, minioMetaBucket, "trash"))
-		removeAll(path.Join(c.dir, minioMetaBucket, "buckets"))
-		// just migrate cache format
-		return migrateCacheFormatJSON(cacheFormatPath)
-	}
-
-	buckets, err := readDir(oldCacheBucketsPath)
-	if err != nil {
-		return err
-	}
-
-	for _, bucket := range buckets {
-		bucket = strings.TrimSuffix(bucket, SlashSeparator)
-		var objMetaPaths []string
-		root := path.Join(oldCacheBucketsPath, bucket)
-		err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
-			if strings.HasSuffix(path, cacheMetaJSONFile) {
-				objMetaPaths = append(objMetaPaths, path)
-			}
-			return nil
-		})
-		if err != nil {
-			return err
-		}
-		for _, oMeta := range objMetaPaths {
-			objSlice := strings.SplitN(oMeta, cacheMetaJSONFile, 2)
-			object := strings.TrimPrefix(objSlice[0], path.Join(oldCacheBucketsPath, bucket))
-			object = strings.TrimSuffix(object, "/")
-
-			destdir := getCacheSHADir(c.dir, bucket, object)
-			if err := os.MkdirAll(destdir, 0o777); err != nil {
-				return err
-			}
-			prevCachedPath := path.Join(c.dir, bucket, object)
-
-			// get old cached metadata
-			oldMetaPath := pathJoin(oldCacheBucketsPath, bucket, object, cacheMetaJSONFile)
-			metaPath := pathJoin(destdir, cacheMetaJSONFile)
-			metaBytes, err := os.ReadFile(oldMetaPath)
-			if err != nil {
-				return err
-			}
-			// marshal cache metadata after adding version and stat info
-			meta := &cacheMeta{}
-			json := jsoniter.ConfigCompatibleWithStandardLibrary
-			if err = json.Unmarshal(metaBytes, &meta); err != nil {
-				return err
-			}
-			// move cached object to new cache directory path
-			// migrate cache data and add bit-rot protection hash sum
-			// at the start of each block
-			if err := migrateCacheData(ctx, c, bucket, object, prevCachedPath, destdir, meta.Meta); err != nil {
-				continue
-			}
-			stat, err := os.Stat(prevCachedPath)
-			if err != nil {
-				if err == errFileNotFound {
-					continue
-				}
-				logger.LogIf(ctx, err)
-				return err
-			}
-			// old cached file can now be removed
-			if err := os.Remove(prevCachedPath); err != nil {
-				return err
-			}
-			// move cached metadata after changing cache metadata version
-			meta.Checksum = CacheChecksumInfoV1{Algorithm: HighwayHash256S.String(), Blocksize: cacheBlkSize}
-			meta.Version = cacheMetaVersion
-			meta.Stat.Size = stat.Size()
-			meta.Stat.ModTime = stat.ModTime()
-			jsonData, err := json.Marshal(meta)
-			if err != nil {
-				return err
-			}
-
-			if err = os.WriteFile(metaPath, jsonData, 0o644); err != nil {
-				return err
-			}
-		}
-
-		// delete old bucket from cache, now that all contents are cleared
-		removeAll(path.Join(c.dir, bucket))
-	}
-
-	// remove .minio.sys sub directories
-	removeAll(path.Join(c.dir, minioMetaBucket, "multipart"))
-	removeAll(path.Join(c.dir, minioMetaBucket, "tmp"))
-	removeAll(path.Join(c.dir, minioMetaBucket, "trash"))
-	removeAll(path.Join(c.dir, minioMetaBucket, "buckets"))
-
-	return migrateCacheFormatJSON(cacheFormatPath)
-}
-
-func migrateCacheFormatJSON(cacheFormatPath string) error {
-	// now migrate format.json
-	f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0o666)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	formatV1 := formatCacheV1{}
-	if err := jsonLoad(f, &formatV1); err != nil {
-		return err
-	}
-
-	formatV2 := &formatCacheV2{}
-	formatV2.formatMetaV1 = formatV1.formatMetaV1
-	formatV2.Version = formatMetaVersion1
-	formatV2.Cache = formatV1.Cache
-	formatV2.Cache.Version = formatCacheVersionV2
-	return jsonSave(f, formatV2)
-}

cmd/format-disk-cache_test.go

@@ -1,321 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"context"
-	"os"
-	"testing"
-)
-
-// TestDiskCacheFormat - tests initFormatCache, formatMetaGetFormatBackendCache, formatCacheGetVersion.
-func TestDiskCacheFormat(t *testing.T) {
-	ctx := context.Background()
-	fsDirs, err := getRandomDisks(1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = initFormatCache(ctx, fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Do the basic sanity checks to check if initFormatCache() did its job.
-	cacheFormatPath := pathJoin(fsDirs[0], minioMetaBucket, formatConfigFile)
-	f, err := os.OpenFile(cacheFormatPath, os.O_RDWR|os.O_SYNC, 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer f.Close()
-	version, err := formatCacheGetVersion(f)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if version != formatCacheVersionV2 {
-		t.Fatalf(`expected: %s, got: %s`, formatCacheVersionV2, version)
-	}
-
-	// Corrupt the format.json file and test the functions.
-	// formatMetaGetFormatBackendFS, formatFSGetVersion, initFormatFS should return errors.
-	if err = f.Truncate(0); err != nil {
-		t.Fatal(err)
-	}
-	if _, err = f.WriteString("b"); err != nil {
-		t.Fatal(err)
-	}
-
-	if _, _, err = loadAndValidateCacheFormat(context.Background(), fsDirs); err == nil {
-		t.Fatal("expected to fail")
-	}
-
-	// With unknown formatMetaV1.Version formatMetaGetFormatCache, initFormatCache should return error.
-	if err = f.Truncate(0); err != nil {
-		t.Fatal(err)
-	}
-	// Here we set formatMetaV1.Version to "2"
-	if _, err = f.WriteString(`{"version":"2","format":"cache","cache":{"version":"1"}}`); err != nil {
-		t.Fatal(err)
-	}
-
-	if _, _, err = loadAndValidateCacheFormat(context.Background(), fsDirs); err == nil {
-		t.Fatal("expected to fail")
-	}
-}
-
-// generates a valid format.json for Cache backend.
-func genFormatCacheValid() []*formatCacheV2 {
-	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV2, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV1{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV2
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	return formatConfigs
-}
-
-// generates a invalid format.json version for Cache backend.
-func genFormatCacheInvalidVersion() []*formatCacheV2 {
-	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV2, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV1{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV1
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	// Corrupt version numbers.
-	formatConfigs[0].Version = "2"
-	formatConfigs[3].Version = "-1"
-	return formatConfigs
-}
-
-// generates a invalid format.json version for Cache backend.
-func genFormatCacheInvalidFormat() []*formatCacheV2 {
-	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV2, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV2{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV1
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	// Corrupt format.
-	formatConfigs[0].Format = "cach"
-	formatConfigs[3].Format = "cach"
-	return formatConfigs
-}
-
-// generates a invalid format.json version for Cache backend.
-func genFormatCacheInvalidCacheVersion() []*formatCacheV2 {
-	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV2, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV2{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV1
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	// Corrupt version numbers.
-	formatConfigs[0].Cache.Version = "10"
-	formatConfigs[3].Cache.Version = "-1"
-	return formatConfigs
-}
-
-// generates a invalid format.json version for Cache backend.
-func genFormatCacheInvalidDisksCount() []*formatCacheV2 {
-	disks := make([]string, 7)
-	formatConfigs := make([]*formatCacheV2, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV2{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV2
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	return formatConfigs
-}
-
-// generates a invalid format.json Disks for Cache backend.
-func genFormatCacheInvalidDisks() []*formatCacheV2 {
-	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV2, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV1{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV2
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	// Corrupt Disks entries on disk 6 and disk 8.
-	formatConfigs[5].Cache.Disks = disks
-	formatConfigs[7].Cache.Disks = disks
-	return formatConfigs
-}
-
-// generates a invalid format.json This disk UUID for Cache backend.
-func genFormatCacheInvalidThis() []*formatCacheV1 {
-	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV1, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV1{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV2
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	// Make disk 5 and disk 8 have inconsistent disk uuid's.
-	formatConfigs[4].Cache.This = mustGetUUID()
-	formatConfigs[7].Cache.This = mustGetUUID()
-	return formatConfigs
-}
-
-// generates a invalid format.json Disk UUID in wrong order for Cache backend.
-func genFormatCacheInvalidDisksOrder() []*formatCacheV2 {
-	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV2, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV1{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV2
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	// Re order disks for failure case.
-	disks1 := make([]string, 8)
-	copy(disks1, disks)
-	disks1[1], disks1[2] = disks[2], disks[1]
-	formatConfigs[2].Cache.Disks = disks1
-	return formatConfigs
-}
-
-// Wrapper for calling FormatCache tests - validates
-//   - valid format
-//   - unrecognized version number
-//   - unrecognized format tag
-//   - unrecognized cache version
-//   - wrong number of Disks entries
-//   - invalid This uuid
-//   - invalid Disks order
-func TestFormatCache(t *testing.T) {
-	formatInputCases := [][]*formatCacheV1{
-		genFormatCacheValid(),
-		genFormatCacheInvalidVersion(),
-		genFormatCacheInvalidFormat(),
-		genFormatCacheInvalidCacheVersion(),
-		genFormatCacheInvalidDisksCount(),
-		genFormatCacheInvalidDisks(),
-		genFormatCacheInvalidThis(),
-		genFormatCacheInvalidDisksOrder(),
-	}
-	testCases := []struct {
-		formatConfigs []*formatCacheV1
-		shouldPass    bool
-	}{
-		{
-			formatConfigs: formatInputCases[0],
-			shouldPass:    true,
-		},
-		{
-			formatConfigs: formatInputCases[1],
-			shouldPass:    false,
-		},
-		{
-			formatConfigs: formatInputCases[2],
-			shouldPass:    false,
-		},
-		{
-			formatConfigs: formatInputCases[3],
-			shouldPass:    false,
-		},
-		{
-			formatConfigs: formatInputCases[4],
-			shouldPass:    false,
-		},
-		{
-			formatConfigs: formatInputCases[5],
-			shouldPass:    false,
-		},
-		{
-			formatConfigs: formatInputCases[6],
-			shouldPass:    false,
-		},
-		{
-			formatConfigs: formatInputCases[7],
-			shouldPass:    false,
-		},
-	}
-
-	for i, testCase := range testCases {
-		err := validateCacheFormats(context.Background(), false, testCase.formatConfigs)
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass but failed with %s", i+1, err)
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail but passed instead", i+1)
-		}
-	}
-}

cmd/generic-handlers.go

@@ -231,7 +231,8 @@ func guessIsMetricsReq(req *http.Request) bool {
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsPathLegacy ||
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ClusterPath ||
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2NodePath ||
-		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2BucketPath
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2BucketPath ||
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ResourcePath
 }
 
 // guessIsRPCReq - returns true if the request is for an RPC endpoint.

cmd/global-heal.go

@@ -309,7 +309,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 						Remove:   healDeleteDangling,
 					})
 				if err != nil {
-					if isErrObjectNotFound(err) {
+					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 						// queueing happens across namespace, ignore
 						// objects that are not found.
 						return
@@ -335,7 +335,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 						ScanMode: scanMode,
 						Remove:   healDeleteDangling,
 					}); err != nil {
-					if isErrObjectNotFound(err) {
+					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 						// queueing happens across namespace, ignore
 						// objects that are not found.
 						versionNotFound++
@@ -344,9 +344,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					// If not deleted, assume they failed.
 					result = healEntryFailure(uint64(version.Size))
 					if version.VersionID != "" {
-						if !isErrVersionNotFound(err) {
-							logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
-						}
+						logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
 					} else {
 						logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, version.Name, err))
 					}
@@ -358,7 +356,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					return
 				}
 			}
-			// All versions resulted in 'ObjectNotFound'
+			// All versions resulted in 'ObjectNotFound/VersionNotFound'
 			if versionNotFound == len(fivs.Versions) {
 				return
 			}

cmd/globals.go

@@ -38,7 +38,6 @@ import (
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/internal/auth"
-	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/callhome"
 	"github.com/minio/minio/internal/config/compress"
 	"github.com/minio/minio/internal/config/dns"
@@ -273,12 +272,6 @@ var (
 	globalBucketQuotaSys      *BucketQuotaSys
 	globalBucketVersioningSys *BucketVersioningSys
 
-	// Disk cache drives
-	globalCacheConfig cache.Config
-
-	// Initialized KMS configuration for disk cache
-	globalCacheKMS kms.KMS
-
 	// Allocated etcd endpoint for config and bucket DNS.
 	globalEtcdClient *etcd.Client
 

cmd/listen-notification-handlers.go

@@ -116,7 +116,7 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 
 	// Listen Publisher and peer-listen-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	listenCh := make(chan event.Event, 4000)
+	listenCh := make(chan event.Event, globalAPIConfig.getRequestsPoolCapacity()*len(globalEndpoints.Hostnames()))
 
 	peers, _ := newPeerRestClients(globalEndpoints)
 

cmd/metacache-server-pool.go

@@ -370,7 +370,7 @@ func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCa
 
 			objInfo := fi.ToObjectInfo(o.Bucket, obj.name, versioned)
 			if o.Lifecycle != nil {
-				act := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo).Action
+				act := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, o.Replication.Config, objInfo).Action
 				skip = act.Delete()
 				if act.DeleteRestored() {
 					// do not skip DeleteRestored* actions
@@ -398,7 +398,7 @@ func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCa
 			objInfo := version.ToObjectInfo(o.Bucket, obj.name, versioned)
 
 			if o.Lifecycle != nil {
-				evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo)
+				evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, o.Replication.Config, objInfo)
 				if evt.Action.Delete() {
 					globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects)
 					if !evt.Action.DeleteRestored() {

cmd/metacache-walk.go

@@ -251,10 +251,8 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				meta.name = strings.TrimSuffix(meta.name, SlashSeparator)
 				meta.name = pathJoinBuf(sb, current, meta.name)
 				meta.name = decodeDirObject(meta.name)
-				if err := send(meta); err != nil {
-					return err
-				}
-				return nil
+
+				return send(meta)
 			}
 			// Check legacy.
 			if HasSuffix(entry, xlStorageFormatFileV1) && legacy {
@@ -270,10 +268,8 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				meta.name = strings.TrimSuffix(entry, xlStorageFormatFileV1)
 				meta.name = strings.TrimSuffix(meta.name, SlashSeparator)
 				meta.name = pathJoinBuf(sb, current, meta.name)
-				if err := send(meta); err != nil {
-					return err
-				}
-				return nil
+
+				return send(meta)
 			}
 			// Skip all other files.
 		}

cmd/metrics-realtime.go

@@ -19,12 +19,14 @@ package cmd
 
 import (
 	"context"
-	"net/http"
+	"fmt"
 	"time"
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/disk"
 	"github.com/minio/minio/internal/net"
+	c "github.com/shirou/gopsutil/v3/cpu"
+	"github.com/shirou/gopsutil/v3/load"
 )
 
 type collectMetricsOpts struct {
@@ -39,18 +41,6 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 		return
 	}
 
-	byHostName := globalMinioAddr
-	if len(opts.hosts) > 0 {
-		server := getLocalServerProperty(globalEndpoints, &http.Request{
-			Host: globalLocalNodeName,
-		})
-		if _, ok := opts.hosts[server.Endpoint]; ok {
-			byHostName = server.Endpoint
-		} else {
-			return
-		}
-	}
-
 	if types.Contains(madmin.MetricsDisk) {
 		m.ByDisk = make(map[string]madmin.DiskMetric)
 		aggr := madmin.DiskMetric{
@@ -84,16 +74,44 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 		}
 		netStats, err := net.GetInterfaceNetStats(globalInternodeInterface)
 		if err != nil {
-			m.Errors = append(m.Errors, err.Error())
+			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v  (nicstats)", globalMinioAddr, err.Error()))
 		} else {
 			m.Aggregated.Net.NetStats = netStats
 		}
 	}
+	if types.Contains(madmin.MetricsMem) {
+		m.Aggregated.Mem = &madmin.MemMetrics{
+			CollectedAt: UTCNow(),
+		}
+		m.Aggregated.Mem.Info = madmin.GetMemInfo(GlobalContext, globalMinioAddr)
+	}
+	if types.Contains(madmin.MetricsCPU) {
+		m.Aggregated.CPU = &madmin.CPUMetrics{
+			CollectedAt: UTCNow(),
+		}
+		cm, err := c.Times(false)
+		if err != nil {
+			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v  (cputimes)", globalMinioAddr, err.Error()))
+		} else {
+			// not collecting per-cpu stats, so there will be only one element
+			if len(cm) == 1 {
+				m.Aggregated.CPU.TimesStat = &cm[0]
+			} else {
+				m.Errors = append(m.Errors, fmt.Sprintf("%s: Expected one CPU stat, got %d", globalMinioAddr, len(cm)))
+			}
+		}
+		loadStat, err := load.Avg()
+		if err != nil {
+			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (loadStat)", globalMinioAddr, err.Error()))
+		} else {
+			m.Aggregated.CPU.LoadStat = loadStat
+		}
+	}
 	// Add types...
 
 	// ByHost is a shallow reference, so careful about sharing.
-	m.ByHost = map[string]madmin.Metrics{byHostName: m.Aggregated}
-	m.Hosts = append(m.Hosts, byHostName)
+	m.ByHost = map[string]madmin.Metrics{globalMinioAddr: m.Aggregated}
+	m.Hosts = append(m.Hosts, globalMinioAddr)
 
 	return m
 }

cmd/metrics-resource.go

@@ -0,0 +1,453 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/shirou/gopsutil/v3/host"
+)
+
+const (
+	resourceMetricsCollectionInterval = time.Minute
+	resourceMetricsCacheInterval      = time.Minute
+
+	// drive stats
+	totalInodes    MetricName = "total_inodes"
+	readsPerSec    MetricName = "reads_per_sec"
+	writesPerSec   MetricName = "writes_per_sec"
+	readsKBPerSec  MetricName = "reads_kb_per_sec"
+	writesKBPerSec MetricName = "writes_kb_per_sec"
+	readsAwait     MetricName = "reads_await"
+	writesAwait    MetricName = "writes_await"
+	percUtil       MetricName = "perc_util"
+	usedInodes     MetricName = "used_inodes"
+
+	// network stats
+	interfaceRxBytes  MetricName = "rx_bytes"
+	interfaceRxErrors MetricName = "rx_errors"
+	interfaceTxBytes  MetricName = "tx_bytes"
+	interfaceTxErrors MetricName = "tx_errors"
+
+	// memory stats
+	memUsed      MetricName = "used"
+	memFree      MetricName = "free"
+	memShared    MetricName = "shared"
+	memBuffers   MetricName = "buffers"
+	memCache     MetricName = "cache"
+	memAvailable MetricName = "available"
+
+	// cpu stats
+	cpuUser   MetricName = "user"
+	cpuSystem MetricName = "system"
+	cpuIOWait MetricName = "iowait"
+	cpuIdle   MetricName = "idle"
+	cpuNice   MetricName = "nice"
+	cpuSteal  MetricName = "steal"
+	cpuLoad1  MetricName = "load1"
+	cpuLoad5  MetricName = "load5"
+	cpuLoad15 MetricName = "load15"
+)
+
+var (
+	resourceCollector *minioResourceCollector
+	// resourceMetricsMap is a map of subsystem to its metrics
+	resourceMetricsMap map[MetricSubsystem]ResourceMetrics
+	// resourceMetricsHelpMap maps metric name to its help string
+	resourceMetricsHelpMap map[MetricName]string
+	resourceMetricsGroups  []*MetricsGroup
+)
+
+// PeerResourceMetrics represents the resource metrics
+// retrieved from a peer, along with errors if any
+type PeerResourceMetrics struct {
+	Metrics map[MetricSubsystem]ResourceMetrics
+	Errors  []string
+}
+
+// ResourceMetrics is a map of unique key identifying
+// a resource metric (e.g. reads_per_sec_{node}_{drive})
+// to its data
+type ResourceMetrics map[string]ResourceMetric
+
+// ResourceMetric represents a single resource metric
+// The metrics are collected from all servers periodically
+// and stored in the resource metrics map.
+// It also maintains the count of number of times this metric
+// was collected since the server started, and the sum,
+// average and max values across the same.
+type ResourceMetric struct {
+	Name   MetricName
+	Labels map[string]string
+
+	// value captured in current cycle
+	Current float64
+
+	// Used when system provides cumulative (since uptime) values
+	// helps in calculating the current value by comparing the new
+	// cumulative value with previous one
+	Cumulative float64
+
+	Max   float64
+	Avg   float64
+	Sum   float64
+	Count uint64
+}
+
+func init() {
+	interval := fmt.Sprintf("%ds", int(resourceMetricsCollectionInterval.Seconds()))
+	resourceMetricsHelpMap = map[MetricName]string{
+		interfaceRxBytes:  "Bytes received on the interface in " + interval,
+		interfaceRxErrors: "Receive errors in " + interval,
+		interfaceTxBytes:  "Bytes transmitted in " + interval,
+		interfaceTxErrors: "Transmit errors in " + interval,
+		total:             "Total memory on the node",
+		memUsed:           "Used memory on the node",
+		memFree:           "Free memory on the node",
+		memShared:         "Shared memory on the node",
+		memBuffers:        "Buffers memory on the node",
+		memCache:          "Cache memory on the node",
+		memAvailable:      "Available memory on the node",
+		readsPerSec:       "Reads per second on a drive",
+		writesPerSec:      "Writes per second on a drive",
+		readsKBPerSec:     "Kilobytes read per second on a drive",
+		writesKBPerSec:    "Kilobytes written per second on a drive",
+		readsAwait:        "Average time for read requests to be served on a drive",
+		writesAwait:       "Average time for write requests to be served on a drive",
+		percUtil:          "Percentage of time the disk was busy since uptime",
+		usedBytes:         "Used bytes on a drive",
+		totalBytes:        "Total bytes on a drive",
+		usedInodes:        "Total inodes used on a drive",
+		totalInodes:       "Total inodes on a drive",
+		cpuUser:           "CPU user time",
+		cpuSystem:         "CPU system time",
+		cpuIdle:           "CPU idle time",
+		cpuIOWait:         "CPU ioWait time",
+		cpuSteal:          "CPU steal time",
+		cpuNice:           "CPU nice time",
+		cpuLoad1:          "CPU load average 1min",
+		cpuLoad5:          "CPU load average 5min",
+		cpuLoad15:         "CPU load average 15min",
+	}
+	resourceMetricsGroups = []*MetricsGroup{
+		getResourceMetrics(),
+	}
+
+	resourceCollector = newMinioResourceCollector(resourceMetricsGroups)
+}
+
+func updateResourceMetrics(subSys MetricSubsystem, name MetricName, val float64, labels map[string]string, isCumulative bool) {
+	subsysMetrics, found := resourceMetricsMap[subSys]
+	if !found {
+		subsysMetrics = ResourceMetrics{}
+	}
+
+	// labels are used to uniquely identify a metric
+	// e.g. reads_per_sec_{drive} inside the map
+	sfx := ""
+	for _, v := range labels {
+		if len(sfx) > 0 {
+			sfx += "_"
+		}
+		sfx += v
+	}
+
+	key := string(name) + "_" + sfx
+	metric, found := subsysMetrics[key]
+	if !found {
+		metric = ResourceMetric{
+			Name:   name,
+			Labels: labels,
+		}
+	}
+
+	if isCumulative {
+		metric.Current = val - metric.Cumulative
+		metric.Cumulative = val
+	} else {
+		metric.Current = val
+	}
+
+	if metric.Current > metric.Max {
+		metric.Max = val
+	}
+
+	metric.Sum += metric.Current
+	metric.Count++
+
+	metric.Avg = metric.Sum / float64(metric.Count)
+	subsysMetrics[key] = metric
+
+	resourceMetricsMap[subSys] = subsysMetrics
+}
+
+func collectDriveMetrics(m madmin.RealtimeMetrics) {
+	upt, _ := host.Uptime()
+	kib := 1 << 10
+	sectorSize := uint64(512)
+
+	for d, dm := range m.ByDisk {
+		stats := dm.IOStats
+		labels := map[string]string{"drive": d}
+		updateResourceMetrics(driveSubsystem, readsPerSec, float64(stats.ReadIOs)/float64(upt), labels, false)
+
+		readBytes := stats.ReadSectors * sectorSize
+		readKib := float64(readBytes) / float64(kib)
+		readKibPerSec := readKib / float64(upt)
+		updateResourceMetrics(driveSubsystem, readsKBPerSec, readKibPerSec, labels, false)
+
+		updateResourceMetrics(driveSubsystem, writesPerSec, float64(stats.WriteIOs)/float64(upt), labels, false)
+
+		writeBytes := stats.WriteSectors * sectorSize
+		writeKib := float64(writeBytes) / float64(kib)
+		writeKibPerSec := writeKib / float64(upt)
+		updateResourceMetrics(driveSubsystem, writesKBPerSec, writeKibPerSec, labels, false)
+
+		rdAwait := 0.0
+		if stats.ReadIOs > 0 {
+			rdAwait = float64(stats.ReadTicks) / float64(stats.ReadIOs)
+		}
+		updateResourceMetrics(driveSubsystem, readsAwait, rdAwait, labels, false)
+
+		wrAwait := 0.0
+		if stats.WriteIOs > 0 {
+			wrAwait = float64(stats.WriteTicks) / float64(stats.WriteIOs)
+		}
+		updateResourceMetrics(driveSubsystem, writesAwait, wrAwait, labels, false)
+
+		updateResourceMetrics(driveSubsystem, percUtil, float64(stats.TotalTicks)/float64(upt*10), labels, false)
+	}
+
+	globalLocalDrivesMu.RLock()
+	gld := globalLocalDrives
+	globalLocalDrivesMu.RUnlock()
+
+	for _, d := range gld {
+		labels := map[string]string{"drive": d.Endpoint().RawPath}
+		di, err := d.DiskInfo(GlobalContext, false)
+		if err == nil {
+			updateResourceMetrics(driveSubsystem, usedBytes, float64(di.Used), labels, false)
+			updateResourceMetrics(driveSubsystem, totalBytes, float64(di.Total), labels, false)
+			updateResourceMetrics(driveSubsystem, usedInodes, float64(di.UsedInodes), labels, false)
+			updateResourceMetrics(driveSubsystem, totalInodes, float64(di.FreeInodes+di.UsedInodes), labels, false)
+		}
+	}
+}
+
+func collectLocalResourceMetrics() {
+	var types madmin.MetricType = madmin.MetricsDisk | madmin.MetricNet | madmin.MetricsMem | madmin.MetricsCPU
+
+	m := collectLocalMetrics(types, collectMetricsOpts{
+		hosts: map[string]struct{}{
+			globalLocalNodeName: {},
+		},
+	})
+
+	for host, hm := range m.ByHost {
+		if len(host) > 0 {
+			if hm.Net != nil && len(hm.Net.NetStats.Name) > 0 {
+				stats := hm.Net.NetStats
+				labels := map[string]string{"interface": stats.Name}
+				updateResourceMetrics(interfaceSubsystem, interfaceRxBytes, float64(stats.RxBytes), labels, true)
+				updateResourceMetrics(interfaceSubsystem, interfaceRxErrors, float64(stats.RxErrors), labels, true)
+				updateResourceMetrics(interfaceSubsystem, interfaceTxBytes, float64(stats.TxBytes), labels, true)
+				updateResourceMetrics(interfaceSubsystem, interfaceTxErrors, float64(stats.TxErrors), labels, true)
+			}
+			if hm.Mem != nil && len(hm.Mem.Info.Addr) > 0 {
+				labels := map[string]string{}
+				stats := hm.Mem.Info
+				updateResourceMetrics(memSubsystem, total, float64(stats.Total), labels, false)
+				updateResourceMetrics(memSubsystem, memUsed, float64(stats.Used), labels, false)
+				updateResourceMetrics(memSubsystem, memFree, float64(stats.Free), labels, false)
+				updateResourceMetrics(memSubsystem, memShared, float64(stats.Shared), labels, false)
+				updateResourceMetrics(memSubsystem, memBuffers, float64(stats.Buffers), labels, false)
+				updateResourceMetrics(memSubsystem, memAvailable, float64(stats.Available), labels, false)
+				updateResourceMetrics(memSubsystem, memCache, float64(stats.Cache), labels, false)
+			}
+			if hm.CPU != nil {
+				labels := map[string]string{}
+				ts := hm.CPU.TimesStat
+				if ts != nil {
+					tot := ts.User + ts.System + ts.Idle + ts.Iowait + ts.Nice + ts.Steal
+					cpuUserVal := math.Round(ts.User/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuUser, cpuUserVal, labels, false)
+					cpuSystemVal := math.Round(ts.System/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuSystem, cpuSystemVal, labels, false)
+					cpuIdleVal := math.Round(ts.Idle/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuIdle, cpuIdleVal, labels, false)
+					cpuIOWaitVal := math.Round(ts.Iowait/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuIOWait, cpuIOWaitVal, labels, false)
+					cpuNiceVal := math.Round(ts.Nice/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuNice, cpuNiceVal, labels, false)
+					cpuStealVal := math.Round(ts.Steal/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuSteal, cpuStealVal, labels, false)
+				}
+				ls := hm.CPU.LoadStat
+				if ls != nil {
+					updateResourceMetrics(cpuSubsystem, cpuLoad1, ls.Load1, labels, false)
+					updateResourceMetrics(cpuSubsystem, cpuLoad5, ls.Load5, labels, false)
+					updateResourceMetrics(cpuSubsystem, cpuLoad15, ls.Load15, labels, false)
+				}
+			}
+			break // only one host expected
+		}
+	}
+
+	collectDriveMetrics(m)
+}
+
+// startResourceMetricsCollection - starts the job for collecting resource metrics
+func startResourceMetricsCollection() {
+	resourceMetricsMap = map[MetricSubsystem]ResourceMetrics{}
+	metricsTimer := time.NewTimer(resourceMetricsCollectionInterval)
+	defer metricsTimer.Stop()
+
+	collectLocalResourceMetrics()
+
+	for {
+		select {
+		case <-GlobalContext.Done():
+			return
+		case <-metricsTimer.C:
+			collectLocalResourceMetrics()
+
+			// Reset the timer for next cycle.
+			metricsTimer.Reset(resourceMetricsCollectionInterval)
+		}
+	}
+}
+
+// minioResourceCollector is the Collector for resource metrics
+type minioResourceCollector struct {
+	metricsGroups []*MetricsGroup
+	desc          *prometheus.Desc
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+func (c *minioResourceCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.desc
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (c *minioResourceCollector) Collect(out chan<- prometheus.Metric) {
+	var wg sync.WaitGroup
+	publish := func(in <-chan Metric) {
+		defer wg.Done()
+		for metric := range in {
+			labels, values := getOrderedLabelValueArrays(metric.VariableLabels)
+			collectMetric(metric, labels, values, "resource", out)
+		}
+	}
+
+	// Call peer api to fetch metrics
+	wg.Add(2)
+	go publish(ReportMetrics(GlobalContext, c.metricsGroups))
+	go publish(globalNotificationSys.GetResourceMetrics(GlobalContext))
+	wg.Wait()
+}
+
+// newMinioResourceCollector describes the collector
+// and returns reference of minio resource Collector
+// It creates the Prometheus Description which is used
+// to define Metric and  help string
+func newMinioResourceCollector(metricsGroups []*MetricsGroup) *minioResourceCollector {
+	return &minioResourceCollector{
+		metricsGroups: metricsGroups,
+		desc:          prometheus.NewDesc("minio_resource_stats", "Resource statistics exposed by MinIO server", nil, nil),
+	}
+}
+
+func prepareResourceMetrics(rm ResourceMetric, subSys MetricSubsystem, requireAvgMax bool) []Metric {
+	help := resourceMetricsHelpMap[rm.Name]
+	name := rm.Name
+
+	metrics := []Metric{}
+	metrics = append(metrics, Metric{
+		Description:    getResourceMetricDescription(subSys, name, help),
+		Value:          rm.Current,
+		VariableLabels: rm.Labels,
+	})
+
+	if requireAvgMax {
+		avgName := MetricName(fmt.Sprintf("%s_avg", name))
+		avgHelp := fmt.Sprintf("%s (avg)", help)
+		metrics = append(metrics, Metric{
+			Description:    getResourceMetricDescription(subSys, avgName, avgHelp),
+			Value:          math.Round(rm.Avg*100) / 100,
+			VariableLabels: rm.Labels,
+		})
+
+		maxName := MetricName(fmt.Sprintf("%s_max", name))
+		maxHelp := fmt.Sprintf("%s (max)", help)
+		metrics = append(metrics, Metric{
+			Description:    getResourceMetricDescription(subSys, maxName, maxHelp),
+			Value:          rm.Max,
+			VariableLabels: rm.Labels,
+		})
+	}
+
+	return metrics
+}
+
+func getResourceMetricDescription(subSys MetricSubsystem, name MetricName, help string) MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: subSys,
+		Name:      name,
+		Help:      help,
+		Type:      gaugeMetric,
+	}
+}
+
+func getResourceMetrics() *MetricsGroup {
+	mg := &MetricsGroup{
+		cacheInterval: resourceMetricsCacheInterval,
+	}
+	mg.RegisterRead(func(ctx context.Context) []Metric {
+		metrics := []Metric{}
+
+		subSystems := []MetricSubsystem{interfaceSubsystem, memSubsystem, driveSubsystem, cpuSubsystem}
+		for _, subSys := range subSystems {
+			stats, found := resourceMetricsMap[subSys]
+			if found {
+				requireAvgMax := true
+				if subSys == driveSubsystem {
+					requireAvgMax = false
+				}
+				for _, m := range stats {
+					metrics = append(metrics, prepareResourceMetrics(m, subSys, requireAvgMax)...)
+				}
+			}
+		}
+
+		return metrics
+	})
+	return mg
+}
+
+// metricsResourceHandler is the prometheus handler for resource metrics
+func metricsResourceHandler() http.Handler {
+	return metricsHTTPHandler(resourceCollector, "handler.MetricsResource")
+}

cmd/metrics-router.go

@@ -25,10 +25,11 @@ import (
 )
 
 const (
-	prometheusMetricsPathLegacy    = "/prometheus/metrics"
-	prometheusMetricsV2ClusterPath = "/v2/metrics/cluster"
-	prometheusMetricsV2BucketPath  = "/v2/metrics/bucket"
-	prometheusMetricsV2NodePath    = "/v2/metrics/node"
+	prometheusMetricsPathLegacy     = "/prometheus/metrics"
+	prometheusMetricsV2ClusterPath  = "/v2/metrics/cluster"
+	prometheusMetricsV2BucketPath   = "/v2/metrics/bucket"
+	prometheusMetricsV2NodePath     = "/v2/metrics/node"
+	prometheusMetricsV2ResourcePath = "/v2/metrics/resource"
 )
 
 // Standard env prometheus auth type
@@ -57,4 +58,5 @@ func registerMetricsRouter(router *mux.Router) {
 	metricsRouter.Handle(prometheusMetricsV2ClusterPath, auth(metricsServerHandler()))
 	metricsRouter.Handle(prometheusMetricsV2BucketPath, auth(metricsBucketHandler()))
 	metricsRouter.Handle(prometheusMetricsV2NodePath, auth(metricsNodeHandler()))
+	metricsRouter.Handle(prometheusMetricsV2ResourcePath, auth(metricsResourceHandler()))
 }

cmd/metrics-v2.go

@@ -59,7 +59,6 @@ func init() {
 	}
 
 	peerMetricsGroups = []*MetricsGroup{
-		getCacheMetrics(),
 		getGoMetrics(),
 		getHTTPMetrics(false),
 		getNotificationMetrics(),
@@ -85,7 +84,6 @@ func init() {
 
 	nodeGroups := []*MetricsGroup{
 		getNodeHealthMetrics(),
-		getCacheMetrics(),
 		getHTTPMetrics(false),
 		getNetworkMetrics(),
 		getMinioVersionMetrics(),
@@ -132,6 +130,9 @@ const (
 	capacityRawSubsystem      MetricSubsystem = "capacity_raw"
 	capacityUsableSubsystem   MetricSubsystem = "capacity_usable"
 	driveSubsystem            MetricSubsystem = "drive"
+	interfaceSubsystem        MetricSubsystem = "if"
+	memSubsystem              MetricSubsystem = "mem"
+	cpuSubsystem              MetricSubsystem = "cpu_avg"
 	storageClassSubsystem     MetricSubsystem = "storage_class"
 	fileDescriptorSubsystem   MetricSubsystem = "file_descriptor"
 	goRoutines                MetricSubsystem = "go_routine"
@@ -235,8 +236,6 @@ const (
 	latencyMicroSec MetricName = "latency_us"
 	latencyNanoSec  MetricName = "latency_ns"
 
-	usagePercent MetricName = "update_percent"
-
 	commitInfo  MetricName = "commit_info"
 	usageInfo   MetricName = "usage_info"
 	versionInfo MetricName = "version_info"
@@ -539,12 +538,12 @@ func getNodeRRSParityMD() MetricDescription {
 	}
 }
 
-func getNodeDrivesFreeInodes() MetricDescription {
+func getNodeDrivesFreeInodesMD() MetricDescription {
 	return MetricDescription{
 		Namespace: nodeMetricNamespace,
 		Subsystem: driveSubsystem,
 		Name:      freeInodes,
-		Help:      "Total free inodes",
+		Help:      "Free inodes on a drive",
 		Type:      gaugeMetric,
 	}
 }
@@ -1227,76 +1226,6 @@ func getS3RejectedInvalidRequestsTotalMD() MetricDescription {
 	}
 }
 
-func getCacheHitsTotalMD() MetricDescription {
-	return MetricDescription{
-		Namespace: minioNamespace,
-		Subsystem: cacheSubsystem,
-		Name:      hitsTotal,
-		Help:      "Total number of drive cache hits",
-		Type:      counterMetric,
-	}
-}
-
-func getCacheHitsMissedTotalMD() MetricDescription {
-	return MetricDescription{
-		Namespace: minioNamespace,
-		Subsystem: cacheSubsystem,
-		Name:      missedTotal,
-		Help:      "Total number of drive cache misses",
-		Type:      counterMetric,
-	}
-}
-
-func getCacheUsagePercentMD() MetricDescription {
-	return MetricDescription{
-		Namespace: minioNamespace,
-		Subsystem: minioNamespace,
-		Name:      usagePercent,
-		Help:      "Total percentage cache usage",
-		Type:      gaugeMetric,
-	}
-}
-
-func getCacheUsageInfoMD() MetricDescription {
-	return MetricDescription{
-		Namespace: minioNamespace,
-		Subsystem: cacheSubsystem,
-		Name:      usageInfo,
-		Help:      "Total percentage cache usage, value of 1 indicates high and 0 low, label level is set as well",
-		Type:      gaugeMetric,
-	}
-}
-
-func getCacheUsedBytesMD() MetricDescription {
-	return MetricDescription{
-		Namespace: minioNamespace,
-		Subsystem: cacheSubsystem,
-		Name:      usedBytes,
-		Help:      "Current cache usage in bytes",
-		Type:      gaugeMetric,
-	}
-}
-
-func getCacheTotalBytesMD() MetricDescription {
-	return MetricDescription{
-		Namespace: minioNamespace,
-		Subsystem: cacheSubsystem,
-		Name:      totalBytes,
-		Help:      "Total size of cache drive in bytes",
-		Type:      gaugeMetric,
-	}
-}
-
-func getCacheSentBytesMD() MetricDescription {
-	return MetricDescription{
-		Namespace: minioNamespace,
-		Subsystem: cacheSubsystem,
-		Name:      sentBytes,
-		Help:      "Total number of bytes served from cache",
-		Type:      counterMetric,
-	}
-}
-
 func getHealObjectsTotalMD() MetricDescription {
 	return MetricDescription{
 		Namespace: healMetricNamespace,
@@ -2451,56 +2380,6 @@ func getObjectsScanned(seq *healSequence) (m []Metric) {
 	return
 }
 
-func getCacheMetrics() *MetricsGroup {
-	mg := &MetricsGroup{
-		cacheInterval: 10 * time.Second,
-	}
-	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
-		cacheObjLayer := newCachedObjectLayerFn()
-		// Service not initialized yet
-		if cacheObjLayer == nil {
-			return
-		}
-		metrics = make([]Metric, 0, 20)
-		metrics = append(metrics, Metric{
-			Description: getCacheHitsTotalMD(),
-			Value:       float64(cacheObjLayer.CacheStats().getHits()),
-		})
-		metrics = append(metrics, Metric{
-			Description: getCacheHitsMissedTotalMD(),
-			Value:       float64(cacheObjLayer.CacheStats().getMisses()),
-		})
-		metrics = append(metrics, Metric{
-			Description: getCacheSentBytesMD(),
-			Value:       float64(cacheObjLayer.CacheStats().getBytesServed()),
-		})
-		for _, cdStats := range cacheObjLayer.CacheStats().GetDiskStats() {
-			metrics = append(metrics, Metric{
-				Description:    getCacheUsagePercentMD(),
-				Value:          float64(cdStats.UsagePercent),
-				VariableLabels: map[string]string{"drive": cdStats.Dir},
-			})
-			metrics = append(metrics, Metric{
-				Description:    getCacheUsageInfoMD(),
-				Value:          float64(cdStats.UsageState),
-				VariableLabels: map[string]string{"drive": cdStats.Dir, "level": cdStats.GetUsageLevelString()},
-			})
-			metrics = append(metrics, Metric{
-				Description:    getCacheUsedBytesMD(),
-				Value:          float64(cdStats.UsageSize),
-				VariableLabels: map[string]string{"drive": cdStats.Dir},
-			})
-			metrics = append(metrics, Metric{
-				Description:    getCacheTotalBytesMD(),
-				Value:          float64(cdStats.TotalCapacity),
-				VariableLabels: map[string]string{"drive": cdStats.Dir},
-			})
-		}
-		return
-	})
-	return mg
-}
-
 func getDistLockMetrics() *MetricsGroup {
 	mg := &MetricsGroup{
 		cacheInterval: 1 * time.Second,
@@ -2553,30 +2432,53 @@ func getNotificationMetrics() *MetricsGroup {
 		cacheInterval: 10 * time.Second,
 	}
 	mg.RegisterRead(func(ctx context.Context) []Metric {
-		nstats := globalNotifyTargetList.Stats()
-		metrics := make([]Metric, 0, 1+len(nstats.TargetStats))
-		metrics = append(metrics, Metric{
-			Description: MetricDescription{
-				Namespace: minioNamespace,
-				Subsystem: notifySubsystem,
-				Name:      "current_send_in_progress",
-				Help:      "Number of concurrent async Send calls active to all targets",
-				Type:      gaugeMetric,
-			},
-			Value: float64(nstats.CurrentSendCalls),
-		})
-		for _, st := range nstats.TargetStats {
+		metrics := make([]Metric, 0, 3)
+
+		if globalEventNotifier != nil {
+			nstats := globalEventNotifier.targetList.Stats()
 			metrics = append(metrics, Metric{
 				Description: MetricDescription{
 					Namespace: minioNamespace,
 					Subsystem: notifySubsystem,
-					Name:      "target_queue_length",
-					Help:      "Number of unsent notifications in queue for target",
+					Name:      "current_send_in_progress",
+					Help:      "Number of concurrent async Send calls active to all targets",
 					Type:      gaugeMetric,
 				},
-				VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name},
-				Value:          float64(st.CurrentQueue),
+				Value: float64(nstats.CurrentSendCalls),
 			})
+			metrics = append(metrics, Metric{
+				Description: MetricDescription{
+					Namespace: minioNamespace,
+					Subsystem: notifySubsystem,
+					Name:      "events_skipped_total",
+					Help:      "Events that were skipped due to full queue",
+					Type:      counterMetric,
+				},
+				Value: float64(nstats.EventsSkipped),
+			})
+			metrics = append(metrics, Metric{
+				Description: MetricDescription{
+					Namespace: minioNamespace,
+					Subsystem: notifySubsystem,
+					Name:      "events_sent_total",
+					Help:      "Total number of events sent since start",
+					Type:      counterMetric,
+				},
+				Value: float64(nstats.TotalEvents),
+			})
+			for _, st := range nstats.TargetStats {
+				metrics = append(metrics, Metric{
+					Description: MetricDescription{
+						Namespace: minioNamespace,
+						Subsystem: notifySubsystem,
+						Name:      "target_queue_length",
+						Help:      "Number of unsent notifications in queue for target",
+						Type:      gaugeMetric,
+					},
+					VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name},
+					Value:          float64(st.CurrentQueue),
+				})
+			}
 		}
 
 		lstats := globalLambdaTargetList.Stats()
@@ -3207,7 +3109,7 @@ func getLocalStorageMetrics() *MetricsGroup {
 			})
 
 			metrics = append(metrics, Metric{
-				Description:    getNodeDrivesFreeInodes(),
+				Description:    getNodeDrivesFreeInodesMD(),
 				Value:          float64(disk.FreeInodes),
 				VariableLabels: map[string]string{"drive": disk.DrivePath},
 			})
@@ -3546,6 +3448,61 @@ func getKMSMetrics() *MetricsGroup {
 	return mg
 }
 
+func collectMetric(metric Metric, labels []string, values []string, metricName string, out chan<- prometheus.Metric) {
+	if metric.Description.Type == histogramMetric {
+		if metric.Histogram == nil {
+			return
+		}
+		for k, v := range metric.Histogram {
+			pmetric, err := prometheus.NewConstMetric(
+				prometheus.NewDesc(
+					prometheus.BuildFQName(string(metric.Description.Namespace),
+						string(metric.Description.Subsystem),
+						string(metric.Description.Name)),
+					metric.Description.Help,
+					append(labels, metric.HistogramBucketLabel),
+					metric.StaticLabels,
+				),
+				prometheus.GaugeValue,
+				float64(v),
+				append(values, k)...)
+			if err != nil {
+				// Enable for debugging
+				if serverDebugLog {
+					logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v+%v", err, values, metric.Histogram), metricName+"-metrics-histogram")
+				}
+			} else {
+				out <- pmetric
+			}
+		}
+		return
+	}
+	metricType := prometheus.GaugeValue
+	if metric.Description.Type == counterMetric {
+		metricType = prometheus.CounterValue
+	}
+	pmetric, err := prometheus.NewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName(string(metric.Description.Namespace),
+				string(metric.Description.Subsystem),
+				string(metric.Description.Name)),
+			metric.Description.Help,
+			labels,
+			metric.StaticLabels,
+		),
+		metricType,
+		metric.Value,
+		values...)
+	if err != nil {
+		// Enable for debugging
+		if serverDebugLog {
+			logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v", err, values), metricName+"-metrics")
+		}
+	} else {
+		out <- pmetric
+	}
+}
+
 type minioBucketCollector struct {
 	metricsGroups []*MetricsGroup
 	desc          *prometheus.Desc
@@ -3570,52 +3527,7 @@ func (c *minioBucketCollector) Collect(out chan<- prometheus.Metric) {
 		defer wg.Done()
 		for metric := range in {
 			labels, values := getOrderedLabelValueArrays(metric.VariableLabels)
-			if metric.Description.Type == histogramMetric {
-				if metric.Histogram == nil {
-					continue
-				}
-				for k, v := range metric.Histogram {
-					pmetric, err := prometheus.NewConstMetric(
-						prometheus.NewDesc(
-							prometheus.BuildFQName(string(metric.Description.Namespace),
-								string(metric.Description.Subsystem),
-								string(metric.Description.Name)),
-							metric.Description.Help,
-							append(labels, metric.HistogramBucketLabel),
-							metric.StaticLabels,
-						),
-						prometheus.GaugeValue,
-						float64(v),
-						append(values, k)...)
-					if err != nil {
-						logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v+%v", err, values, metric.Histogram), "bucket-metrics-histogram")
-					} else {
-						out <- pmetric
-					}
-				}
-				continue
-			}
-			metricType := prometheus.GaugeValue
-			if metric.Description.Type == counterMetric {
-				metricType = prometheus.CounterValue
-			}
-			pmetric, err := prometheus.NewConstMetric(
-				prometheus.NewDesc(
-					prometheus.BuildFQName(string(metric.Description.Namespace),
-						string(metric.Description.Subsystem),
-						string(metric.Description.Name)),
-					metric.Description.Help,
-					labels,
-					metric.StaticLabels,
-				),
-				metricType,
-				metric.Value,
-				values...)
-			if err != nil {
-				logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v", err, values), "bucket-metrics")
-			} else {
-				out <- pmetric
-			}
+			collectMetric(metric, labels, values, "bucket", out)
 		}
 	}
 
@@ -3650,59 +3562,7 @@ func (c *minioClusterCollector) Collect(out chan<- prometheus.Metric) {
 		defer wg.Done()
 		for metric := range in {
 			labels, values := getOrderedLabelValueArrays(metric.VariableLabels)
-			if metric.Description.Type == histogramMetric {
-				if metric.Histogram == nil {
-					continue
-				}
-				for k, v := range metric.Histogram {
-					pmetric, err := prometheus.NewConstMetric(
-						prometheus.NewDesc(
-							prometheus.BuildFQName(string(metric.Description.Namespace),
-								string(metric.Description.Subsystem),
-								string(metric.Description.Name)),
-							metric.Description.Help,
-							append(labels, metric.HistogramBucketLabel),
-							metric.StaticLabels,
-						),
-						prometheus.GaugeValue,
-						float64(v),
-						append(values, k)...)
-
-					if err != nil {
-						// Enable for debugging
-						if serverDebugLog {
-							logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v:%v", err, values, metric.Histogram), "cluster-metrics-histogram")
-						}
-					} else {
-						out <- pmetric
-					}
-				}
-				continue
-			}
-			metricType := prometheus.GaugeValue
-			if metric.Description.Type == counterMetric {
-				metricType = prometheus.CounterValue
-			}
-			pmetric, err := prometheus.NewConstMetric(
-				prometheus.NewDesc(
-					prometheus.BuildFQName(string(metric.Description.Namespace),
-						string(metric.Description.Subsystem),
-						string(metric.Description.Name)),
-					metric.Description.Help,
-					labels,
-					metric.StaticLabels,
-				),
-				metricType,
-				metric.Value,
-				values...)
-			if err != nil {
-				// Enable for debugging
-				if serverDebugLog {
-					logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v", err, values), "cluster-metrics")
-				}
-			} else {
-				out <- pmetric
-			}
+			collectMetric(metric, labels, values, "cluster", out)
 		}
 	}
 
@@ -3835,11 +3695,11 @@ func newMinioCollectorNode(metricsGroups []*MetricsGroup) *minioNodeCollector {
 	}
 }
 
-func metricsBucketHandler() http.Handler {
+func metricsHTTPHandler(c prometheus.Collector, funcName string) http.Handler {
 	registry := prometheus.NewRegistry()
 
 	// Report all other metrics
-	logger.CriticalIf(GlobalContext, registry.Register(bucketCollector))
+	logger.CriticalIf(GlobalContext, registry.Register(c))
 
 	// DefaultGatherers include golang metrics and process metrics.
 	gatherers := prometheus.Gatherers{
@@ -3849,16 +3709,14 @@ func metricsBucketHandler() http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
 		if ok {
-			tc.FuncName = "handler.MetricsBucket"
+			tc.FuncName = funcName
 			tc.ResponseRecorder.LogErrBody = true
 		}
 
 		mfs, err := gatherers.Gather()
-		if err != nil {
-			if len(mfs) == 0 {
-				writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL)
-				return
-			}
+		if err != nil && len(mfs) == 0 {
+			writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL)
+			return
 		}
 
 		contentType := expfmt.Negotiate(r.Header)
@@ -3878,6 +3736,10 @@ func metricsBucketHandler() http.Handler {
 	})
 }
 
+func metricsBucketHandler() http.Handler {
+	return metricsHTTPHandler(bucketCollector, "handler.MetricsBucket")
+}
+
 func metricsServerHandler() http.Handler {
 	registry := prometheus.NewRegistry()
 

cmd/metrics.go

@@ -108,7 +108,6 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 	bucketUsageMetricsPrometheus(ch)
 	networkMetricsPrometheus(ch)
 	httpMetricsPrometheus(ch)
-	cacheMetricsPrometheus(ch)
 	healingMetricsPrometheus(ch)
 }
 
@@ -188,82 +187,6 @@ func healingMetricsPrometheus(ch chan<- prometheus.Metric) {
 	}
 }
 
-// collects cache metrics for MinIO server in Prometheus specific format
-// and sends to given channel
-func cacheMetricsPrometheus(ch chan<- prometheus.Metric) {
-	cacheObjLayer := newCachedObjectLayerFn()
-	// Service not initialized yet
-	if cacheObjLayer == nil {
-		return
-	}
-
-	ch <- prometheus.MustNewConstMetric(
-		prometheus.NewDesc(
-			prometheus.BuildFQName(cacheNamespace, "hits", "total"),
-			"Total number of drive cache hits in current MinIO instance",
-			nil, nil),
-		prometheus.CounterValue,
-		float64(cacheObjLayer.CacheStats().getHits()),
-	)
-	ch <- prometheus.MustNewConstMetric(
-		prometheus.NewDesc(
-			prometheus.BuildFQName(cacheNamespace, "misses", "total"),
-			"Total number of drive cache misses in current MinIO instance",
-			nil, nil),
-		prometheus.CounterValue,
-		float64(cacheObjLayer.CacheStats().getMisses()),
-	)
-	ch <- prometheus.MustNewConstMetric(
-		prometheus.NewDesc(
-			prometheus.BuildFQName(cacheNamespace, "data", "served"),
-			"Total number of bytes served from cache of current MinIO instance",
-			nil, nil),
-		prometheus.CounterValue,
-		float64(cacheObjLayer.CacheStats().getBytesServed()),
-	)
-	for _, cdStats := range cacheObjLayer.CacheStats().GetDiskStats() {
-		// Cache disk usage percentage
-		ch <- prometheus.MustNewConstMetric(
-			prometheus.NewDesc(
-				prometheus.BuildFQName(cacheNamespace, "usage", "percent"),
-				"Total percentage cache usage",
-				[]string{"disk"}, nil),
-			prometheus.GaugeValue,
-			float64(cdStats.UsagePercent),
-			cdStats.Dir,
-		)
-		ch <- prometheus.MustNewConstMetric(
-			prometheus.NewDesc(
-				prometheus.BuildFQName(cacheNamespace, "usage", "high"),
-				"Indicates cache usage is high or low, relative to current cache 'quota' settings",
-				[]string{"disk"}, nil),
-			prometheus.GaugeValue,
-			float64(cdStats.UsageState),
-			cdStats.Dir,
-		)
-
-		ch <- prometheus.MustNewConstMetric(
-			prometheus.NewDesc(
-				prometheus.BuildFQName("cache", "usage", "size"),
-				"Indicates current cache usage in bytes",
-				[]string{"disk"}, nil),
-			prometheus.GaugeValue,
-			float64(cdStats.UsageSize),
-			cdStats.Dir,
-		)
-
-		ch <- prometheus.MustNewConstMetric(
-			prometheus.NewDesc(
-				prometheus.BuildFQName("cache", "total", "size"),
-				"Indicates total size of cache drive",
-				[]string{"disk"}, nil),
-			prometheus.GaugeValue,
-			float64(cdStats.TotalCapacity),
-			cdStats.Dir,
-		)
-	}
-}
-
 // collects http metrics for MinIO server in Prometheus specific format
 // and sends to given channel
 func httpMetricsPrometheus(ch chan<- prometheus.Metric) {

cmd/notification.go

@@ -777,12 +777,33 @@ func (sys *NotificationSys) GetMetrics(ctx context.Context, t madmin.MetricType,
 
 	for index, err := range g.Wait() {
 		if err != nil {
-			reply[index].Errors = []string{err.Error()}
+			reply[index].Errors = []string{fmt.Sprintf("%s: %s (rpc)", sys.peerClients[index].String(), err.Error())}
 		}
 	}
 	return reply
 }
 
+// GetResourceMetrics - gets the resource metrics from all nodes excluding self.
+func (sys *NotificationSys) GetResourceMetrics(ctx context.Context) <-chan Metric {
+	if sys == nil {
+		return nil
+	}
+	g := errgroup.WithNErrs(len(sys.peerClients))
+	peerChannels := make([]<-chan Metric, len(sys.peerClients))
+	for index := range sys.peerClients {
+		index := index
+		g.Go(func() error {
+			if sys.peerClients[index] == nil {
+				return errPeerNotReachable
+			}
+			var err error
+			peerChannels[index], err = sys.peerClients[index].GetResourceMetrics(ctx)
+			return err
+		}, index)
+	}
+	return sys.collectPeerMetrics(ctx, peerChannels, g)
+}
+
 // GetSysConfig - Get information about system config
 // (only the config that are of concern to minio)
 func (sys *NotificationSys) GetSysConfig(ctx context.Context) []madmin.SysConfig {
@@ -1122,25 +1143,7 @@ func (sys *NotificationSys) GetBandwidthReports(ctx context.Context, buckets ...
 	return consolidatedReport
 }
 
-// GetBucketMetrics - gets the cluster level bucket metrics from all nodes excluding self.
-func (sys *NotificationSys) GetBucketMetrics(ctx context.Context) <-chan Metric {
-	if sys == nil {
-		return nil
-	}
-	g := errgroup.WithNErrs(len(sys.peerClients))
-	peerChannels := make([]<-chan Metric, len(sys.peerClients))
-	for index := range sys.peerClients {
-		index := index
-		g.Go(func() error {
-			if sys.peerClients[index] == nil {
-				return errPeerNotReachable
-			}
-			var err error
-			peerChannels[index], err = sys.peerClients[index].GetPeerBucketMetrics(ctx)
-			return err
-		}, index)
-	}
-
+func (sys *NotificationSys) collectPeerMetrics(ctx context.Context, peerChannels []<-chan Metric, g *errgroup.Group) <-chan Metric {
 	ch := make(chan Metric)
 	var wg sync.WaitGroup
 	for index, err := range g.Wait() {
@@ -1181,6 +1184,27 @@ func (sys *NotificationSys) GetBucketMetrics(ctx context.Context) <-chan Metric
 	return ch
 }
 
+// GetBucketMetrics - gets the cluster level bucket metrics from all nodes excluding self.
+func (sys *NotificationSys) GetBucketMetrics(ctx context.Context) <-chan Metric {
+	if sys == nil {
+		return nil
+	}
+	g := errgroup.WithNErrs(len(sys.peerClients))
+	peerChannels := make([]<-chan Metric, len(sys.peerClients))
+	for index := range sys.peerClients {
+		index := index
+		g.Go(func() error {
+			if sys.peerClients[index] == nil {
+				return errPeerNotReachable
+			}
+			var err error
+			peerChannels[index], err = sys.peerClients[index].GetPeerBucketMetrics(ctx)
+			return err
+		}, index)
+	}
+	return sys.collectPeerMetrics(ctx, peerChannels, g)
+}
+
 // GetClusterMetrics - gets the cluster metrics from all nodes excluding self.
 func (sys *NotificationSys) GetClusterMetrics(ctx context.Context) <-chan Metric {
 	if sys == nil {
@@ -1199,45 +1223,7 @@ func (sys *NotificationSys) GetClusterMetrics(ctx context.Context) <-chan Metric
 			return err
 		}, index)
 	}
-
-	ch := make(chan Metric)
-	var wg sync.WaitGroup
-	for index, err := range g.Wait() {
-		if err != nil {
-			if sys.peerClients[index] != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress",
-					sys.peerClients[index].host.String())
-				logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), err, sys.peerClients[index].host.String())
-			} else {
-				logger.LogOnceIf(ctx, err, "peer-offline")
-			}
-			continue
-		}
-		wg.Add(1)
-		go func(ctx context.Context, peerChannel <-chan Metric, wg *sync.WaitGroup) {
-			defer wg.Done()
-			for {
-				select {
-				case m, ok := <-peerChannel:
-					if !ok {
-						return
-					}
-					select {
-					case ch <- m:
-					case <-ctx.Done():
-						return
-					}
-				case <-ctx.Done():
-					return
-				}
-			}
-		}(ctx, peerChannels[index], &wg)
-	}
-	go func(wg *sync.WaitGroup, ch chan Metric) {
-		wg.Wait()
-		close(ch)
-	}(&wg, ch)
-	return ch
+	return sys.collectPeerMetrics(ctx, peerChannels, g)
 }
 
 // ServiceFreeze freezes all S3 API calls when 'freeze' is true,

cmd/object-api-common.go

@@ -52,9 +52,6 @@ var globalObjLayerMutex sync.RWMutex
 // Global object layer, only accessed by globalObjectAPI.
 var globalObjectAPI ObjectLayer
 
-// Global cacheObjects, only accessed by newCacheObjectsFn().
-var globalCacheObjectAPI CacheObjectLayer
-
 type storageOpts struct {
 	cleanUp     bool
 	healthCheck bool

cmd/object-api-datatypes.go

@@ -149,11 +149,6 @@ type ObjectInfo struct {
 	// Date and time at which the object is no longer able to be cached
 	Expires time.Time
 
-	// CacheStatus sets status of whether this is a cache hit/miss
-	CacheStatus CacheStatusType
-	// CacheLookupStatus sets whether a cacheable response is present in the cache
-	CacheLookupStatus CacheStatusType
-
 	// Specify object storage class
 	StorageClass string
 
@@ -245,8 +240,6 @@ func (o *ObjectInfo) Clone() (cinfo ObjectInfo) {
 		ContentType:                o.ContentType,
 		ContentEncoding:            o.ContentEncoding,
 		Expires:                    o.Expires,
-		CacheStatus:                o.CacheStatus,
-		CacheLookupStatus:          o.CacheLookupStatus,
 		StorageClass:               o.StorageClass,
 		ReplicationStatus:          o.ReplicationStatus,
 		UserTags:                   o.UserTags,

cmd/object-api-options.go

@@ -30,7 +30,6 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 )
 
 func getDefaultOpts(header http.Header, copySource bool, metadata map[string]string) (opts ObjectOptions, err error) {
@@ -115,44 +114,43 @@ func getOpts(ctx context.Context, r *http.Request, bucket, object string) (Objec
 	}
 	opts.PartNumber = partNumber
 	opts.VersionID = vid
-	delMarker := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceDeleteMarker))
-	if delMarker != "" {
-		switch delMarker {
-		case "true":
-			opts.DeleteMarker = true
-		case "false":
-		default:
-			err = fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceDeleteMarker, fmt.Errorf("DeleteMarker should be true or false"))
-			logger.LogIf(ctx, err)
-			return opts, InvalidArgument{
-				Bucket: bucket,
-				Object: object,
-				Err:    err,
-			}
-		}
+
+	delMarker, err := parseBoolHeader(bucket, object, r.Header, xhttp.MinIOSourceDeleteMarker)
+	if err != nil {
+		return opts, err
 	}
-	replReadyCheck := strings.TrimSpace(r.Header.Get(xhttp.MinIOCheckDMReplicationReady))
-	if replReadyCheck != "" {
-		switch replReadyCheck {
-		case "true":
-			opts.CheckDMReplicationReady = true
-		case "false":
-		default:
-			err = fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOCheckDMReplicationReady, fmt.Errorf("should be true or false"))
-			logger.LogIf(ctx, err)
-			return opts, InvalidArgument{
-				Bucket: bucket,
-				Object: object,
-				Err:    err,
-			}
-		}
+	opts.DeleteMarker = delMarker
+
+	replReadyCheck, err := parseBoolHeader(bucket, object, r.Header, xhttp.MinIOCheckDMReplicationReady)
+	if err != nil {
+		return opts, err
 	}
+	opts.CheckDMReplicationReady = replReadyCheck
+
 	opts.Tagging = r.Header.Get(xhttp.AmzTagDirective) == accessDirective
 	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(bucket, object)
 	opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(bucket, object)
 	return opts, nil
 }
 
+func parseBoolHeader(bucket, object string, h http.Header, headerName string) (bool, error) {
+	value := strings.TrimSpace(h.Get(headerName))
+	if value != "" {
+		switch value {
+		case "true":
+			return true, nil
+		case "false":
+		default:
+			return false, InvalidArgument{
+				Bucket: bucket,
+				Object: object,
+				Err:    fmt.Errorf("Unable to parse %s, value should be either 'true' or 'false'", headerName),
+			}
+		}
+	}
+	return false, nil
+}
+
 func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts ObjectOptions, err error) {
 	opts, err = getOpts(ctx, r, bucket, object)
 	if err != nil {
@@ -180,22 +178,11 @@ func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts
 		opts.VersionID = nullVersionID
 	}
 
-	delMarker := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceDeleteMarker))
-	if delMarker != "" {
-		switch delMarker {
-		case "true":
-			opts.DeleteMarker = true
-		case "false":
-		default:
-			err = fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceDeleteMarker, fmt.Errorf("DeleteMarker should be true or false"))
-			logger.LogIf(ctx, err)
-			return opts, InvalidArgument{
-				Bucket: bucket,
-				Object: object,
-				Err:    err,
-			}
-		}
+	delMarker, err := parseBoolHeader(bucket, object, r.Header, xhttp.MinIOSourceDeleteMarker)
+	if err != nil {
+		return opts, err
 	}
+	opts.DeleteMarker = delMarker
 
 	mtime := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceMTime))
 	if mtime != "" {
@@ -222,7 +209,6 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
 	if vid != "" && vid != nullVersionID {
 		_, err := uuid.Parse(vid)
 		if err != nil {
-			logger.LogIf(ctx, err)
 			return opts, InvalidVersionID{
 				Bucket:    bucket,
 				Object:    object,
@@ -233,7 +219,7 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
 			return opts, InvalidArgument{
 				Bucket: bucket,
 				Object: object,
-				Err:    fmt.Errorf("VersionID specified %s, but versioning not enabled on  %s", opts.VersionID, bucket),
+				Err:    fmt.Errorf("VersionID specified %s, but versioning not enabled on bucket=%s", opts.VersionID, bucket),
 			}
 		}
 	}

cmd/object-handlers-common.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"regexp"
 	"strconv"
@@ -29,7 +30,6 @@ import (
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 )
 
 var etagRegex = regexp.MustCompile("\"*?([^\"]*?)\"*?$")
@@ -356,17 +356,8 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD
 			PrefixEnabledFn:  vc.PrefixEnabled,
 			VersionSuspended: vc.Suspended(),
 		})
-		var logged bool
-		for i, err := range errs {
-			if err != nil {
-				if !logged {
-					// log the first error
-					logger.LogIf(ctx, err)
-					logged = true
-				}
-				continue
-			}
-			dobj := deletedObjs[i]
+
+		for i, dobj := range deletedObjs {
 			oi := ObjectInfo{
 				Bucket:    bucket,
 				Name:      dobj.ObjectName,
@@ -382,7 +373,7 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD
 				oi,
 				ILMExpiry, tags, traceFn)
 
-			sendEvent(eventArgs{
+			evArgs := eventArgs{
 				EventName:  event.ObjectRemovedDelete,
 				BucketName: bucket,
 				Object: ObjectInfo{
@@ -391,7 +382,13 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD
 				},
 				UserAgent: "Internal: [ILM-Expiry]",
 				Host:      globalLocalNodeName,
-			})
+			}
+			if errs[i] != nil {
+				evArgs.RespElements = map[string]string{
+					"error": fmt.Sprintf("failed to delete %s(%s), with error %v", dobj.ObjectName, dobj.VersionID, errs[i]),
+				}
+			}
+			sendEvent(evArgs)
 		}
 	}
 }

cmd/object-handlers.go

@@ -59,7 +59,6 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/s3select"
 	"github.com/minio/mux"
-	xnet "github.com/minio/pkg/v2/net"
 	"github.com/minio/pkg/v2/policy"
 )
 
@@ -132,9 +131,6 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 	}
 
 	getObjectInfo := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfo = api.CacheAPI().GetObjectInfo
-	}
 
 	// Check for auth type to return S3 compatible error.
 	// type to return the correct error (NoSuchKey vs AccessDenied)
@@ -193,9 +189,6 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 	defer lock.RUnlock(lkctx)
 
 	getObjectNInfo := objectAPI.GetObjectNInfo
-	if api.CacheAPI() != nil {
-		getObjectNInfo = api.CacheAPI().GetObjectNInfo
-	}
 
 	gopts := opts
 	gopts.NoLock = true // We already have a lock, we can live with it.
@@ -350,9 +343,6 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 				IsOwner:         false,
 			}) {
 				getObjectInfo := objectAPI.GetObjectInfo
-				if api.CacheAPI() != nil {
-					getObjectInfo = api.CacheAPI().GetObjectInfo
-				}
 
 				_, err = getObjectInfo(ctx, bucket, object, opts)
 				if toAPIError(ctx, err).Code == "NoSuchKey" {
@@ -365,9 +355,6 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 	}
 
 	getObjectNInfo := objectAPI.GetObjectNInfo
-	if api.CacheAPI() != nil {
-		getObjectNInfo = api.CacheAPI().GetObjectNInfo
-	}
 
 	// Get request range.
 	var rs *HTTPRangeSpec
@@ -474,7 +461,8 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 		// Automatically remove the object/version if an expiry lifecycle rule can be applied
 		if lc, err := globalLifecycleSys.Get(bucket); err == nil {
 			rcfg, _ := globalBucketObjectLockSys.Get(bucket)
-			event := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
+			replcfg, _ := getReplicationConfig(ctx, bucket)
+			event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo)
 			if event.Action.Delete() {
 				// apply whatever the expiry rule is.
 				applyExpiryRule(event, lcEventSrc_s3GetObject, objInfo)
@@ -546,9 +534,6 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
-		if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients
-			logger.LogOnceIf(ctx, fmt.Errorf("Unable to write all the data to client: %w", err), "get-object-handler-write")
-		}
 		return
 	}
 
@@ -557,9 +542,6 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
-		if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients
-			logger.LogOnceIf(ctx, fmt.Errorf("Unable to write all the data to client: %w", err), "get-object-handler-close")
-		}
 		return
 	}
 
@@ -615,9 +597,6 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 	}
 
 	getObjectInfo := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfo = api.CacheAPI().GetObjectInfo
-	}
 
 	opts, err := getOpts(ctx, r, bucket, object)
 	if err != nil {
@@ -649,9 +628,6 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 				IsOwner:         false,
 			}) {
 				getObjectInfo := objectAPI.GetObjectInfo
-				if api.CacheAPI() != nil {
-					getObjectInfo = api.CacheAPI().GetObjectInfo
-				}
 
 				_, err = getObjectInfo(ctx, bucket, object, opts)
 				if toAPIError(ctx, err).Code == "NoSuchKey" {
@@ -732,7 +708,8 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		// Automatically remove the object/version if an expiry lifecycle rule can be applied
 		if lc, err := globalLifecycleSys.Get(bucket); err == nil {
 			rcfg, _ := globalBucketObjectLockSys.Get(bucket)
-			event := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
+			replcfg, _ := getReplicationConfig(ctx, bucket)
+			event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo)
 			if event.Action.Delete() {
 				// apply whatever the expiry rule is.
 				applyExpiryRule(event, lcEventSrc_s3HeadObject, objInfo)
@@ -778,8 +755,6 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 				writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInvalidRange))
 				return
 			}
-
-			logger.LogIf(ctx, err, logger.Application)
 		}
 	}
 
@@ -1065,7 +1040,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	var srcOpts, dstOpts ObjectOptions
 	srcOpts, err = copySrcOpts(ctx, r, srcBucket, srcObject)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1084,16 +1058,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 
 	dstOpts, err = copyDstOpts(ctx, r, dstBucket, dstObject, nil)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
 
 	getObjectNInfo := objectAPI.GetObjectNInfo
-	if api.CacheAPI() != nil {
-		getObjectNInfo = api.CacheAPI().GetObjectNInfo
-	}
 
 	checkCopyPrecondFn := func(o ObjectInfo) bool {
 		if _, err := DecryptObjectInfo(&o, r); err != nil {
@@ -1376,9 +1346,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectRetentionAction)
 	holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectLegalHoldAction)
 	getObjectInfo := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfo = api.CacheAPI().GetObjectInfo
-	}
 
 	// apply default bucket configuration/governance headers for dest side.
 	retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms, holdPerms)
@@ -1521,9 +1488,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		}
 
 		copyObjectFn := objectAPI.CopyObject
-		if api.CacheAPI() != nil {
-			copyObjectFn = api.CacheAPI().CopyObject
-		}
 
 		// Copy source object to destination, if source and destination
 		// object is same then only metadata is updated.
@@ -1570,7 +1534,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		objInfo.ETag = origETag
 		enqueueTransitionImmediate(objInfo, lcEventSrc_s3CopyObject)
 		// Remove the transitioned object whose object version is being overwritten.
-		logger.LogIf(ctx, os.Sweep())
+		os.Sweep()
 	}
 }
 
@@ -1809,17 +1773,10 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 
-	if api.CacheAPI() != nil {
-		putObject = api.CacheAPI().PutObject
-	}
-
 	retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectRetentionAction)
 	holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectLegalHoldAction)
 
 	getObjectInfo := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfo = api.CacheAPI().GetObjectInfo
-	}
 
 	retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
 	if s3Err == ErrNone && retentionMode.Valid() {
@@ -1970,7 +1927,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		// Schedule object for immediate transition if eligible.
 		objInfo.ETag = origETag
 		enqueueTransitionImmediate(objInfo, lcEventSrc_s3PutObject)
-		logger.LogIf(ctx, os.Sweep())
+		os.Sweep()
 	}
 }
 
@@ -2113,22 +2070,21 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 	retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectRetentionAction)
 	holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectLegalHoldAction)
 
-	if api.CacheAPI() != nil {
-		putObject = api.CacheAPI().PutObject
-	}
-
 	getObjectInfo := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfo = api.CacheAPI().GetObjectInfo
+
+	// These are static for all objects extracted.
+	reqParams := extractReqParams(r)
+	respElements := map[string]string{
+		"requestId": w.Header().Get(xhttp.AmzRequestID),
+		"nodeId":    w.Header().Get(xhttp.AmzRequestHostID),
+	}
+	if sc == "" {
+		sc = storageclass.STANDARD
 	}
 
 	putObjectTar := func(reader io.Reader, info os.FileInfo, object string) error {
 		size := info.Size()
 
-		if sc == "" {
-			sc = storageclass.STANDARD
-		}
-
 		metadata := map[string]string{
 			xhttp.AmzStorageClass: sc, // save same storage-class as incoming stream.
 		}
@@ -2256,8 +2212,8 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 			EventName:    event.ObjectCreatedPut,
 			BucketName:   bucket,
 			Object:       objInfo,
-			ReqParams:    extractReqParams(r),
-			RespElements: extractRespElements(w),
+			ReqParams:    reqParams,
+			RespElements: respElements,
 			UserAgent:    r.UserAgent(),
 			Host:         handlers.GetSourceIP(r),
 		}
@@ -2404,9 +2360,6 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 	})
 
 	deleteObject := objectAPI.DeleteObject
-	if api.CacheAPI() != nil {
-		deleteObject = api.CacheAPI().DeleteObject
-	}
 
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
 	objInfo, err := deleteObject(ctx, bucket, object, opts)
@@ -2597,9 +2550,6 @@ func (api objectAPIHandlers) GetObjectLegalHoldHandler(w http.ResponseWriter, r
 	}
 
 	getObjectInfo := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfo = api.CacheAPI().GetObjectInfo
-	}
 
 	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); !rcfg.LockEnabled {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketObjectLockConfiguration), r.URL)
@@ -2765,9 +2715,6 @@ func (api objectAPIHandlers) GetObjectRetentionHandler(w http.ResponseWriter, r
 	}
 
 	getObjectInfo := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfo = api.CacheAPI().GetObjectInfo
-	}
 
 	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); !rcfg.LockEnabled {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketObjectLockConfiguration), r.URL)
@@ -3065,9 +3012,6 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
 	}
 
 	getObjectInfo := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfo = api.CacheAPI().GetObjectInfo
-	}
 
 	// Check for auth type to return S3 compatible error.
 	if s3Error := checkRequestAuthType(ctx, r, policy.RestoreObjectAction, bucket, object); s3Error != ErrNone {
@@ -3142,7 +3086,6 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
 		}, ObjectOptions{
 			VersionID: objInfo.VersionID,
 		}); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s: %s", objInfo.VersionID, err))
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidObjectState), r.URL)
 			return
 		}
@@ -3222,7 +3165,7 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
 			VersionID: objInfo.VersionID,
 		}
 		if err := objectAPI.RestoreTransitionedObject(rctx, bucket, object, opts); err != nil {
-			logger.LogIf(ctx, err)
+			logger.LogIf(ctx, fmt.Errorf("Unable to restore transitioned bucket/object %s/%s: %w", bucket, object, err))
 			return
 		}
 

cmd/object-multipart-handlers.go

@@ -148,9 +148,6 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 	holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectLegalHoldAction)
 
 	getObjectInfo := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfo = api.CacheAPI().GetObjectInfo
-	}
 
 	retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
 	if s3Err == ErrNone && retentionMode.Valid() {
@@ -210,9 +207,6 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 	}
 
 	newMultipartUpload := objectAPI.NewMultipartUpload
-	if api.CacheAPI() != nil {
-		newMultipartUpload = api.CacheAPI().NewMultipartUpload
-	}
 
 	res, err := newMultipartUpload(ctx, bucket, object, opts)
 	if err != nil {
@@ -329,9 +323,6 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	}
 
 	getObjectNInfo := objectAPI.GetObjectNInfo
-	if api.CacheAPI() != nil {
-		getObjectNInfo = api.CacheAPI().GetObjectNInfo
-	}
 
 	// Get request range.
 	var rs *HTTPRangeSpec
@@ -356,7 +347,6 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			return true
 		}
 		if parseRangeErr != nil {
-			logger.LogIf(ctx, parseRangeErr)
 			writeCopyPartErr(ctx, w, parseRangeErr, r.URL)
 			// Range header mismatch is pre-condition like failure
 			// so return true to indicate Range precondition failed.
@@ -543,9 +533,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 
 	srcInfo.PutObjReader = pReader
 	copyObjectPart := objectAPI.CopyObjectPart
-	if api.CacheAPI() != nil {
-		copyObjectPart = api.CacheAPI().CopyObjectPart
-	}
+
 	// Copy source object to destination, if source and destination
 	// object is same then only metadata is updated.
 	partInfo, err := copyObjectPart(ctx, srcBucket, srcObject, dstBucket, dstObject, uploadID, partID,
@@ -822,9 +810,6 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	opts.IndexCB = idxCb
 
 	putObjectPart := objectAPI.PutObjectPart
-	if api.CacheAPI() != nil {
-		putObjectPart = api.CacheAPI().PutObjectPart
-	}
 
 	partInfo, err := putObjectPart(ctx, bucket, object, uploadID, partID, pReader, opts)
 	if err != nil {
@@ -935,9 +920,6 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	}
 
 	completeMultiPartUpload := objectAPI.CompleteMultipartUpload
-	if api.CacheAPI() != nil {
-		completeMultiPartUpload = api.CacheAPI().CompleteMultipartUpload
-	}
 
 	versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object)
 	suspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
@@ -1035,7 +1017,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	if !globalTierConfigMgr.Empty() {
 		// Schedule object for immediate transition if eligible.
 		enqueueTransitionImmediate(objInfo, lcEventSrc_s3CompleteMultipartUpload)
-		logger.LogIf(ctx, os.Sweep())
+		os.Sweep()
 	}
 }
 
@@ -1059,9 +1041,6 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 		return
 	}
 	abortMultipartUpload := objectAPI.AbortMultipartUpload
-	if api.CacheAPI() != nil {
-		abortMultipartUpload = api.CacheAPI().AbortMultipartUpload
-	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.AbortMultipartUploadAction, bucket, object); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)

cmd/os-instrumented.go

@@ -105,7 +105,8 @@ func osTrace(s osMetric, startTime time.Time, duration time.Duration, path strin
 
 func updateOSMetrics(s osMetric, paths ...string) func(err error) {
 	if globalTrace.NumSubscribers(madmin.TraceOS) == 0 {
-		return func(err error) { globalOSMetrics.time(s) }
+		osAction := globalOSMetrics.time(s)
+		return func(err error) { osAction() }
 	}
 
 	startTime := time.Now()

cmd/peer-rest-client.go

@@ -229,6 +229,33 @@ func (client *peerRESTClient) GetMetrics(ctx context.Context, t madmin.MetricTyp
 	return info, err
 }
 
+func (client *peerRESTClient) GetResourceMetrics(ctx context.Context) (<-chan Metric, error) {
+	respBody, err := client.callWithContext(ctx, peerRESTMethodResourceMetrics, nil, nil, -1)
+	if err != nil {
+		return nil, err
+	}
+	dec := gob.NewDecoder(respBody)
+	ch := make(chan Metric)
+	go func(ch chan<- Metric) {
+		defer func() {
+			xhttp.DrainBody(respBody)
+			close(ch)
+		}()
+		for {
+			var metric Metric
+			if err := dec.Decode(&metric); err != nil {
+				return
+			}
+			select {
+			case <-ctx.Done():
+				return
+			case ch <- metric:
+			}
+		}
+	}(ch)
+	return ch, nil
+}
+
 // GetProcInfo - fetch MinIO process information for a remote node.
 func (client *peerRESTClient) GetProcInfo(ctx context.Context) (info madmin.ProcInfo, err error) {
 	respBody, err := client.callWithContext(ctx, peerRESTMethodProcInfo, nil, nil, -1)

cmd/peer-rest-common.go

@@ -77,6 +77,7 @@ const (
 	peerRESTMethodDevNull                     = "/devnull"
 	peerRESTMethodNetperf                     = "/netperf"
 	peerRESTMethodMetrics                     = "/metrics"
+	peerRESTMethodResourceMetrics             = "/resourcemetrics"
 	peerRESTMethodGetReplicationMRF           = "/getreplicationmrf"
 	peerRESTMethodGetSRMetrics                = "/getsrmetrics"
 )

cmd/peer-rest-server.go

@@ -474,6 +474,22 @@ func (s *peerRESTServer) GetMetricsHandler(w http.ResponseWriter, r *http.Reques
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
 }
 
+func (s *peerRESTServer) GetResourceMetrics(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+
+	enc := gob.NewEncoder(w)
+
+	for m := range ReportMetrics(r.Context(), resourceMetricsGroups) {
+		if err := enc.Encode(m); err != nil {
+			s.writeErrorResponse(w, errors.New("Encoding metric failed: "+err.Error()))
+			return
+		}
+	}
+}
+
 // GetSysConfigHandler - returns system config information.
 // (only the config that are of concern to minio)
 func (s *peerRESTServer) GetSysConfigHandler(w http.ResponseWriter, r *http.Request) {
@@ -939,7 +955,7 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) {
 
 	// Listen Publisher uses nonblocking publish and hence does not wait for slow subscribers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	ch := make(chan event.Event, 2000)
+	ch := make(chan event.Event, globalAPIConfig.getRequestsPoolCapacity())
 
 	err := globalHTTPListen.Subscribe(mask, ch, doneCh, func(ev event.Event) bool {
 		if ev.S3.Bucket.Name != "" && values.Get(peerRESTListenBucket) != "" {
@@ -994,7 +1010,7 @@ func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) {
 
 	// Trace Publisher uses nonblocking publish and hence does not wait for slow subscribers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	ch := make(chan madmin.TraceInfo, 2000)
+	ch := make(chan madmin.TraceInfo, 100000)
 	err = globalTrace.Subscribe(traceOpts.TraceTypes(), ch, r.Context().Done(), func(entry madmin.TraceInfo) bool {
 		return shouldTrace(entry, traceOpts)
 	})
@@ -1143,13 +1159,13 @@ func (s *peerRESTServer) ConsoleLogHandler(w http.ResponseWriter, r *http.Reques
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 
-	ch := make(chan log.Info, 2000)
+	ch := make(chan log.Info, 100000)
 	err := globalConsoleSys.Subscribe(ch, doneCh, "", 0, madmin.LogMaskAll, nil)
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
-	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	keepAliveTicker := time.NewTicker(time.Second)
 	defer keepAliveTicker.Stop()
 
 	enc := gob.NewEncoder(w)
@@ -1438,6 +1454,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcInfo).HandlerFunc(h(server.GetProcInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemInfo).HandlerFunc(h(server.GetMemInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMetrics).HandlerFunc(h(server.GetMetricsHandler)).Queries(restQueries(peerRESTMetricsTypes)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodResourceMetrics).HandlerFunc(h(server.GetResourceMetrics))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysErrors).HandlerFunc(h(server.GetSysErrorsHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysServices).HandlerFunc(h(server.GetSysServicesHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysConfig).HandlerFunc(h(server.GetSysConfigHandler))

cmd/s3-zip-handlers.go

@@ -21,7 +21,6 @@ import (
 	"bytes"
 	"context"
 	"errors"
-	"fmt"
 	"io"
 	"net/http"
 	"sort"
@@ -31,8 +30,6 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
-	xnet "github.com/minio/pkg/v2/net"
 	"github.com/minio/pkg/v2/policy"
 	"github.com/minio/zipindex"
 )
@@ -82,9 +79,6 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
 	}
 
 	getObjectInfo := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfo = api.CacheAPI().GetObjectInfo
-	}
 
 	// Check for auth type to return S3 compatible error.
 	// type to return the correct error (NoSuchKey vs AccessDenied)
@@ -223,9 +217,6 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
-		if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients
-			logger.LogIf(ctx, fmt.Errorf("Unable to write all the data to client: %w", err))
-		}
 		return
 	}
 
@@ -234,9 +225,6 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
-		if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients
-			logger.LogIf(ctx, fmt.Errorf("Unable to write all the data to client: %w", err))
-		}
 		return
 	}
 }
@@ -384,9 +372,6 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context,
 	}
 
 	getObjectInfo := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfo = api.CacheAPI().GetObjectInfo
-	}
 
 	opts, err := getOpts(ctx, r, bucket, zipPath)
 	if err != nil {

cmd/server-main.go

@@ -579,14 +579,17 @@ func serverMain(ctx *cli.Context) {
 
 	setDefaultProfilerRates()
 
-	// Handle all server environment vars.
-	serverHandleEnvVars()
+	// Always load ENV variables from files first.
+	loadEnvVarsFromFiles()
 
 	// Handle all server command args.
 	bootstrapTrace("serverHandleCmdArgs", func() {
 		serverHandleCmdArgs(ctx)
 	})
 
+	// Handle all server environment vars.
+	serverHandleEnvVars()
+
 	// Initialize globalConsoleSys system
 	bootstrapTrace("newConsoleLogger", func() {
 		globalConsoleSys = NewConsoleLogger(GlobalContext)
@@ -853,16 +856,6 @@ func serverMain(ctx *cli.Context) {
 			})
 		}()
 
-		// initialize the new disk cache objects.
-		if globalCacheConfig.Enabled {
-			logger.Info(color.Yellow("WARNING: Drive caching is deprecated for single/multi drive MinIO setups."))
-			var cacheAPI CacheObjectLayer
-			cacheAPI, err = newServerCacheObjects(GlobalContext, globalCacheConfig)
-			logger.FatalIf(err, "Unable to initialize drive caching")
-
-			setCacheObjectLayer(cacheAPI)
-		}
-
 		// Initialize bucket notification system.
 		bootstrapTrace("initBucketTargets", func() {
 			logger.LogIf(GlobalContext, globalEventNotifier.InitBucketTargets(GlobalContext, newObject))
@@ -928,6 +921,10 @@ func serverMain(ctx *cli.Context) {
 		logger.FatalIf(err, "Unable to initialize MinIO client")
 	})
 
+	go bootstrapTrace("startResourceMetricsCollection", func() {
+		startResourceMetricsCollection()
+	})
+
 	// Add User-Agent to differentiate the requests.
 	globalMinioClient.SetAppInfo("minio-perf-test", ReleaseTag)
 

cmd/server-startup-msg.go

@@ -21,12 +21,10 @@ import (
 	"fmt"
 	"net"
 	"net/url"
-	"runtime"
 	"strings"
 
-	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v3"
-	color "github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/logger"
 	xnet "github.com/minio/pkg/v2/net"
 )
@@ -53,11 +51,6 @@ func printStartupMessage(apiEndpoints []string, err error) {
 	}
 
 	strippedAPIEndpoints := stripStandardPorts(apiEndpoints, globalMinioHost)
-	// If cache layer is enabled, print cache capacity.
-	cachedObjAPI := newCachedObjectLayerFn()
-	if cachedObjAPI != nil {
-		printCacheStorageInfo(cachedObjAPI.StorageInfo(GlobalContext))
-	}
 
 	// Object layer is initialized then print StorageInfo.
 	objAPI := newObjectLayerFn()
@@ -129,10 +122,10 @@ func printServerCommonMsg(apiEndpoints []string) {
 	// Colorize the message and print.
 	logger.Info(color.Blue("S3-API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
 	if color.IsTerminal() && (!globalCLIContext.Anonymous && !globalCLIContext.JSON) {
-		logger.Info(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-		logger.Info(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s \n", cred.SecretKey)))
+		logger.Info(color.Blue("RootUser: ") + color.Bold("%s ", cred.AccessKey))
+		logger.Info(color.Blue("RootPass: ") + color.Bold("%s \n", cred.SecretKey))
 		if region != "" {
-			logger.Info(color.Blue("Region: ") + color.Bold(fmt.Sprintf(getFormatStr(len(region), 2), region)))
+			logger.Info(color.Blue("Region: ") + color.Bold("%s", fmt.Sprintf(getFormatStr(len(region), 2), region)))
 		}
 	}
 
@@ -140,8 +133,8 @@ func printServerCommonMsg(apiEndpoints []string) {
 		consoleEndpointStr := strings.Join(stripStandardPorts(getConsoleEndpoints(), globalMinioConsoleHost), " ")
 		logger.Info(color.Blue("Console: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
 		if color.IsTerminal() && (!globalCLIContext.Anonymous && !globalCLIContext.JSON) {
-			logger.Info(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-			logger.Info(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
+			logger.Info(color.Blue("RootUser: ") + color.Bold("%s ", cred.AccessKey))
+			logger.Info(color.Blue("RootPass: ") + color.Bold("%s ", cred.SecretKey))
 		}
 	}
 
@@ -196,15 +189,9 @@ func printCLIAccessMsg(endPoint string, alias string) {
 	// Configure 'mc', following block prints platform specific information for minio client.
 	if color.IsTerminal() && !globalCLIContext.Anonymous {
 		logger.Info(color.Blue("\nCommand-line: ") + mcQuickStartGuide)
-		if runtime.GOOS == globalWindowsOSName {
-			mcMessage := fmt.Sprintf("$ mc.exe alias set %s %s %s %s", alias,
-				endPoint, cred.AccessKey, cred.SecretKey)
-			logger.Info(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
-		} else {
-			mcMessage := fmt.Sprintf("$ mc alias set %s %s %s %s", alias,
-				endPoint, cred.AccessKey, cred.SecretKey)
-			logger.Info(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
-		}
+		mcMessage := fmt.Sprintf("$ mc alias set '%s' '%s' '%s' '%s'", alias,
+			endPoint, cred.AccessKey, cred.SecretKey)
+		logger.Info(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
 	}
 }
 
@@ -233,10 +220,3 @@ func printStorageInfo(storageInfo StorageInfo) {
 		logger.Info(msg)
 	}
 }
-
-func printCacheStorageInfo(storageInfo CacheStorageInfo) {
-	msg := fmt.Sprintf("%s %s Free, %s Total", color.Blue("Cache Capacity:"),
-		humanize.IBytes(storageInfo.Free),
-		humanize.IBytes(storageInfo.Total))
-	logger.Info(msg)
-}

cmd/service.go

@@ -104,11 +104,11 @@ func unfreezeServices() {
 	// Close when we reach 0
 	globalServiceFreezeCnt--
 	if globalServiceFreezeCnt <= 0 {
-		// Ensure we only close once.
-		if val := globalServiceFreeze.Load(); val != nil {
-			var _ch chan struct{}
-			if ch, ok := val.(chan struct{}); ok {
-				globalServiceFreeze.Store(_ch)
+		// Set to a nil channel.
+		var _ch chan struct{}
+		if val := globalServiceFreeze.Swap(_ch); val != nil {
+			if ch, ok := val.(chan struct{}); ok && ch != nil {
+				// Close previous non-nil channel.
 				close(ch)
 			}
 		}

cmd/site-replication-utils.go

@@ -118,11 +118,11 @@ func (sm *siteResyncMetrics) load(ctx context.Context, objAPI ObjectLayer) error
 			return err
 		}
 		sm.Lock()
-		defer sm.Unlock()
 		if _, ok := sm.peerResyncMap[peer.DeploymentID]; !ok {
 			sm.peerResyncMap[peer.DeploymentID] = resyncState{resyncID: rs.ResyncID, LastSaved: time.Time{}}
 			sm.resyncStatus[rs.ResyncID] = rs
 		}
+		sm.Unlock()
 	}
 	return nil
 }
@@ -198,9 +198,9 @@ func (sm *siteResyncMetrics) save(ctx context.Context) {
 }
 
 // update overall site resync state
-func (sm *siteResyncMetrics) updateState(s SiteResyncStatus) {
+func (sm *siteResyncMetrics) updateState(s SiteResyncStatus) error {
 	if !globalSiteReplicationSys.isEnabled() {
-		return
+		return nil
 	}
 	sm.Lock()
 	defer sm.Unlock()
@@ -213,9 +213,12 @@ func (sm *siteResyncMetrics) updateState(s SiteResyncStatus) {
 		if ok {
 			st.LastUpdate = s.LastUpdate
 			st.Status = s.Status
+			return nil
 		}
 		sm.resyncStatus[s.ResyncID] = st
+		return saveSiteResyncMetadata(GlobalContext, st, newObjectLayerFn())
 	}
+	return nil
 }
 
 // increment SyncedBuckets count

cmd/site-replication.go

@@ -5157,8 +5157,7 @@ func (c *SiteReplicationSys) startResync(ctx context.Context, objAPI ObjectLayer
 		}
 	}()
 
-	globalSiteResyncMetrics.updateState(rs)
-	if err := saveSiteResyncMetadata(ctx, rs, objAPI); err != nil {
+	if err := globalSiteResyncMetrics.updateState(rs); err != nil {
 		return res, err
 	}
 
@@ -5408,6 +5407,9 @@ func loadSiteResyncMetadata(ctx context.Context, objAPI ObjectLayer, dID string)
 
 // save resync status of peer to resync/depl-id.meta
 func saveSiteResyncMetadata(ctx context.Context, ss SiteResyncStatus, objectAPI ObjectLayer) error {
+	if objectAPI == nil {
+		return errSRObjectLayerNotReady
+	}
 	data := make([]byte, 4, ss.Msgsize()+4)
 
 	// Initialize the resync meta header.

cmd/sts-errors.go

@@ -27,23 +27,23 @@ import (
 )
 
 // writeSTSErrorRespone writes error headers
-func writeSTSErrorResponse(ctx context.Context, w http.ResponseWriter, errCode STSErrorCode, errCtxt error) {
-	err := stsErrCodes.ToSTSErr(errCode)
+func writeSTSErrorResponse(ctx context.Context, w http.ResponseWriter, errCode STSErrorCode, err error) {
+	stsErr := stsErrCodes.ToSTSErr(errCode)
 
 	// Generate error response.
 	stsErrorResponse := STSErrorResponse{}
-	stsErrorResponse.Error.Code = err.Code
+	stsErrorResponse.Error.Code = stsErr.Code
 	stsErrorResponse.RequestID = w.Header().Get(xhttp.AmzRequestID)
-	stsErrorResponse.Error.Message = err.Description
-	if errCtxt != nil {
-		stsErrorResponse.Error.Message = errCtxt.Error()
+	stsErrorResponse.Error.Message = stsErr.Description
+	if err != nil {
+		stsErrorResponse.Error.Message = err.Error()
 	}
 	switch errCode {
-	case ErrSTSInternalError, ErrSTSNotInitialized, ErrSTSUpstreamError, ErrSTSIAMNotInitialized:
-		logger.LogIf(ctx, errCtxt, logger.Minio)
+	case ErrSTSInternalError, ErrSTSUpstreamError:
+		logger.LogIf(ctx, err, logger.Minio)
 	}
 	encodedErrorResponse := encodeResponse(stsErrorResponse)
-	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
+	writeResponse(w, stsErr.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }
 
 // STSError structure

cmd/test-utils_test.go

@@ -2049,9 +2049,6 @@ func registerAPIFunctions(muxRouter *mux.Router, objLayer ObjectLayer, apiFuncti
 		ObjectAPI: func() ObjectLayer {
 			return globalObjectAPI
 		},
-		CacheAPI: func() CacheObjectLayer {
-			return globalCacheObjectAPI
-		},
 	}
 
 	// Register ListBuckets	handler.

cmd/tier-journal.go

@@ -195,6 +195,10 @@ func (jd *tierDiskJournal) addEntry(je jentry) error {
 	defer jd.Unlock()
 	_, err = jd.file.Write(b)
 	if err != nil {
+		// Do not leak fd here, close the file properly.
+		Fdatasync(jd.file)
+		_ = jd.file.Close()
+
 		jd.file = nil // reset to allow subsequent reopen when file/disk is available.
 	}
 	return err

cmd/utils.go

@@ -702,37 +702,6 @@ func NewRemoteTargetHTTPTransport(insecure bool) func() *http.Transport {
 	}.NewRemoteTargetHTTPTransport(insecure)
 }
 
-// Load the json (typically from disk file).
-func jsonLoad(r io.ReadSeeker, data interface{}) error {
-	if _, err := r.Seek(0, io.SeekStart); err != nil {
-		return err
-	}
-	return json.NewDecoder(r).Decode(data)
-}
-
-// Save to disk file in json format.
-func jsonSave(f interface {
-	io.WriteSeeker
-	Truncate(int64) error
-}, data interface{},
-) error {
-	b, err := json.Marshal(data)
-	if err != nil {
-		return err
-	}
-	if err = f.Truncate(0); err != nil {
-		return err
-	}
-	if _, err = f.Seek(0, io.SeekStart); err != nil {
-		return err
-	}
-	_, err = f.Write(b)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
 // ceilFrac takes a numerator and denominator representing a fraction
 // and returns its ceiling. If denominator is 0, it returns 0 instead
 // of crashing.

cmd/xl-storage-format-v2.go

@@ -328,6 +328,18 @@ func (x xlMetaV2VersionHeader) sortsBefore(o xlMetaV2VersionHeader) bool {
 	return false
 }
 
+func (j xlMetaV2Version) getDataDir() string {
+	if j.Valid() {
+		switch j.Type {
+		case LegacyType:
+			return j.ObjectV1.DataDir
+		case ObjectType:
+			return uuid.UUID(j.ObjectV2.DataDir).String()
+		}
+	}
+	return ""
+}
+
 // Valid xl meta xlMetaV2Version is valid
 func (j xlMetaV2Version) Valid() bool {
 	if !j.Type.valid() {
@@ -1170,6 +1182,20 @@ func (x *xlMetaV2) AppendTo(dst []byte) ([]byte, error) {
 	return append(dst, x.data...), nil
 }
 
+func (x *xlMetaV2) findVersionStr(key string) (idx int, ver *xlMetaV2Version, err error) {
+	if key == nullVersionID {
+		key = ""
+	}
+	var u uuid.UUID
+	if key != "" {
+		u, err = uuid.Parse(key)
+		if err != nil {
+			return -1, nil, errFileVersionNotFound
+		}
+	}
+	return x.findVersion(u)
+}
+
 func (x *xlMetaV2) findVersion(key [16]byte) (idx int, ver *xlMetaV2Version, err error) {
 	for i, ver := range x.versions {
 		if key == ver.header.VersionID {

cmd/xl-storage.go

@@ -290,19 +290,14 @@ func newXLStorage(ep Endpoint, cleanUp bool) (s *xlStorage, err error) {
 		s.formatLegacy = format.Erasure.DistributionAlgo == formatErasureVersionV2DistributionAlgoV1
 	}
 
-	if globalAPIConfig.odirectEnabled() {
-		// Return an error if ODirect is not supported
-		// unless it is a single erasure disk mode
-		if err := s.checkODirectDiskSupport(); err == nil {
-			s.oDirect = true
-		} else {
-			// Allow if unsupported platform or single disk.
-			if errors.Is(err, errUnsupportedDisk) && globalIsErasureSD || !disk.ODirectPlatform {
-				s.oDirect = false
-			} else {
-				return s, err
-			}
-		}
+	// Return an error if ODirect is not supported. Single disk will have
+	// oDirect off.
+	if globalIsErasureSD || !disk.ODirectPlatform {
+		s.oDirect = false
+	} else if err := s.checkODirectDiskSupport(); err == nil {
+		s.oDirect = true
+	} else {
+		return s, err
 	}
 
 	// Success.
@@ -1066,12 +1061,7 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 		return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), buf)
 	}
 
-	// Move xl.meta to trash
-	err = s.moveToTrash(pathJoin(volumeDir, path, xlStorageFormatFile), false, false)
-	if err == nil || err == errFileNotFound {
-		s.deleteFile(volumeDir, pathJoin(volumeDir, path), false, false)
-	}
-	return err
+	return s.deleteFile(volumeDir, pathJoin(volumeDir, path), true, false)
 }
 
 // DeleteVersions deletes slice of versions, it can be same object
@@ -2447,19 +2437,20 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 	}
 
 	// Replace the data of null version or any other existing version-id
-	ofi, err := xlMeta.ToFileInfo(dstVolume, dstPath, reqVID, false, false)
-	if err == nil && !ofi.Deleted {
-		if xlMeta.SharedDataDirCountStr(reqVID, ofi.DataDir) == 0 {
+	_, ver, err := xlMeta.findVersionStr(reqVID)
+	if err == nil {
+		dataDir := ver.getDataDir()
+		if dataDir != "" && (xlMeta.SharedDataDirCountStr(reqVID, dataDir) == 0) {
 			// Purge the destination path as we are not preserving anything
 			// versioned object was not requested.
-			oldDstDataPath = pathJoin(dstVolumeDir, dstPath, ofi.DataDir)
+			oldDstDataPath = pathJoin(dstVolumeDir, dstPath, dataDir)
 			// if old destination path is same as new destination path
 			// there is nothing to purge, this is true in case of healing
 			// avoid setting oldDstDataPath at that point.
 			if oldDstDataPath == dstDataPath {
 				oldDstDataPath = ""
 			} else {
-				xlMeta.data.remove(reqVID, ofi.DataDir)
+				xlMeta.data.remove(reqVID, dataDir)
 			}
 		}
 	}

docs/bucket/replication/setup_3site_replication.sh

@@ -41,7 +41,8 @@ unset MINIO_KMS_KES_KEY_FILE
 unset MINIO_KMS_KES_ENDPOINT
 unset MINIO_KMS_KES_KEY_NAME
 
-go build ./docs/debugging/s3-check-md5/
+go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
+
 wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
 	chmod +x mc
 wget -O mc.RELEASE.2021-03-12T03-36-59Z https://dl.minio.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-03-12T03-36-59Z &&
@@ -194,18 +195,18 @@ head -c 221227088 </dev/urandom >200M
 sleep 10
 
 echo "Verifying ETag for all objects"
-./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9001/ -bucket bucket
-./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9002/ -bucket bucket
-./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9003/ -bucket bucket
-./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9004/ -bucket bucket
-./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9005/ -bucket bucket
-./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9006/ -bucket bucket
+s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9001/ -bucket bucket
+s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9002/ -bucket bucket
+s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9003/ -bucket bucket
+s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9004/ -bucket bucket
+s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9005/ -bucket bucket
+s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9006/ -bucket bucket
 
-./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9001/ -bucket olockbucket
-./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9002/ -bucket olockbucket
-./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9003/ -bucket olockbucket
-./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9004/ -bucket olockbucket
-./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9005/ -bucket olockbucket
-./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9006/ -bucket olockbucket
+s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9001/ -bucket olockbucket
+s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9002/ -bucket olockbucket
+s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9003/ -bucket olockbucket
+s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9004/ -bucket olockbucket
+s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9005/ -bucket olockbucket
+s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9006/ -bucket olockbucket
 
 catch

docs/debugging/pprofgoparser/go.mod

@@ -1,3 +1,3 @@
 module github.com/minio/minio/docs/debugging/pprofgoparser
 
-go 1.21.1
+go 1.21.3

docs/debugging/s3-verify/go.mod

@@ -1,6 +1,6 @@
 module github.com/minio/minio/docs/debugging/s3-verify
 
-go 1.21.1
+go 1.21.3
 
 require github.com/minio/minio-go/v7 v7.0.63
 
@@ -16,9 +16,9 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/rs/xid v1.5.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
-	golang.org/x/crypto v0.12.0 // indirect
-	golang.org/x/net v0.14.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
-	golang.org/x/text v0.12.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 )

docs/debugging/s3-verify/go.sum

@@ -34,16 +34,16 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
-golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
-golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14=
-golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
-golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
-golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

docs/metrics/prometheus/list.md

@@ -303,3 +303,109 @@ For deployments with [Site Replication](https://min.io/docs/minio/linux/operatio
 | `minio_bucket_requests_total`                     | Total number of S3 requests on a bucket.                                        |
 | `minio_bucket_requests_canceled_total`            | Total number S3 requests canceled by the client.                                |
 | `minio_bucket_requests_ttfb_seconds_distribution` | Distribution of time to first byte across API calls per bucket.                 |
+
+# Resource Metrics
+
+MinIO collects the following resource metrics at the node level.
+Each metric includes the `server` label to identify the corresponding node.
+Metrics may include one or more additional labels, such as the drive path, interface name, etc.
+
+These metrics can be obtained from any MinIO server once per collection by using the following URL:
+
+```shell
+https://HOSTNAME:PORT/minio/metrics/v2/resource
+```
+
+Replace `HOSTNAME:PORT` with the hostname of your MinIO deployment.
+For deployments behind a load balancer, use the load balancer hostname instead of a single node hostname.
+
+## Drive Resource Metrics
+
+| Name                                 | Description                                              |
+| :----------------------------------- | :------------------------------------------------------- |
+| `minio_node_drive_total_bytes`       | Total bytes on a drive.                                  |
+| `minio_node_drive_used_bytes`        | Used bytes on a drive.                                   |
+| `minio_node_drive_total_inodes`      | Total inodes on a drive.                                 |
+| `minio_node_drive_used_inodes`       | Total inodes used on a drive.                            |
+| `minio_node_drive_reads_per_sec`     | Reads per second on a drive.                             |
+| `minio_node_drive_reads_kb_per_sec`  | Kilobytes read per second on a drive.                    |
+| `minio_node_drive_reads_await`       | Average time for read requests to be served on a drive.  |
+| `minio_node_drive_writes_per_sec`    | Writes per second on a drive.                            |
+| `minio_node_drive_writes_kb_per_sec` | Kilobytes written per second on a drive.                 |
+| `minio_node_drive_writes_await`      | Average time for write requests to be served on a drive. |
+| `minio_node_drive_perc_util`         | Percentage of time the disk was busy since uptime.       |
+
+## Network Interface Metrics
+
+| Name                          | Description                                   |
+| :---------------------------- | :-------------------------------------------- |
+| `minio_node_if_rx_bytes`      | Bytes received on the interface in 60s.       |
+| `minio_node_if_rx_bytes_avg`  | Bytes received on the interface in 60s (avg). |
+| `minio_node_if_rx_bytes_max`  | Bytes received on the interface in 60s (max). |
+| `minio_node_if_rx_errors`     | Receive errors in 60s.                        |
+| `minio_node_if_rx_errors_avg` | Receive errors in 60s (avg).                  |
+| `minio_node_if_rx_errors_max` | Receive errors in 60s (max).                  |
+| `minio_node_if_tx_bytes`      | Bytes transmitted in 60s.                     |
+| `minio_node_if_tx_bytes_avg`  | Bytes transmitted in 60s (avg).               |
+| `minio_node_if_tx_bytes_max`  | Bytes transmitted in 60s (max).               |
+| `minio_node_if_tx_errors`     | Transmit errors in 60s.                       |
+| `minio_node_if_tx_errors_avg` | Transmit errors in 60s (avg).                 |
+| `minio_node_if_tx_errors_max` | Transmit errors in 60s (max).                 |
+
+## CPU Metrics
+
+| Name                            | Description                   |
+| :------------------------------ | :---------------------------- |
+| `minio_node_cpu_avg_user`       | CPU user time.                |
+| `minio_node_cpu_avg_user_avg`   | CPU user time (avg).          |
+| `minio_node_cpu_avg_user_max`   | CPU user time (max).          |
+| `minio_node_cpu_avg_system`     | CPU system time.              |
+| `minio_node_cpu_avg_system_avg` | CPU system time (avg).        |
+| `minio_node_cpu_avg_system_max` | CPU system time (max).        |
+| `minio_node_cpu_avg_idle`       | CPU idle time.                |
+| `minio_node_cpu_avg_idle_avg`   | CPU idle time (avg).          |
+| `minio_node_cpu_avg_idle_max`   | CPU idle time (max).          |
+| `minio_node_cpu_avg_iowait`     | CPU ioWait time.              |
+| `minio_node_cpu_avg_iowait_avg` | CPU ioWait time (avg).        |
+| `minio_node_cpu_avg_iowait_max` | CPU ioWait time (max).        |
+| `minio_node_cpu_avg_nice`       | CPU nice time.                |
+| `minio_node_cpu_avg_nice_avg`   | CPU nice time (avg).          |
+| `minio_node_cpu_avg_nice_max`   | CPU nice time (max).          |
+| `minio_node_cpu_avg_steal`      | CPU steam time.               |
+| `minio_node_cpu_avg_steal_avg`  | CPU steam time (avg).         |
+| `minio_node_cpu_avg_steal_max`  | CPU steam time (max).         |
+| `minio_node_cpu_avg_load1`      | CPU load average 1min.        |
+| `minio_node_cpu_avg_load1_avg`  | CPU load average 1min (avg).  |
+| `minio_node_cpu_avg_load1_max`  | CPU load average 1min (max).  |
+| `minio_node_cpu_avg_load5`      | CPU load average 5min.        |
+| `minio_node_cpu_avg_load5_avg`  | CPU load average 5min (avg).  |
+| `minio_node_cpu_avg_load5_max`  | CPU load average 5min (max).  |
+| `minio_node_cpu_avg_load15`     | CPU load average 15min.       |
+| `minio_node_cpu_avg_load15_avg` | CPU load average 15min (avg). |
+| `minio_node_cpu_avg_load15_max` | CPU load average 15min (max). |
+
+## Memory Metrics
+
+| Name                           | Description                         |
+| :----------------------------- | :---------------------------------- |
+| `minio_node_mem_available`     | Available memory on the node.       |
+| `minio_node_mem_available_avg` | Available memory on the node (avg). |
+| `minio_node_mem_available_max` | Available memory on the node (max). |
+| `minio_node_mem_buffers`       | Buffers memory on the node.         |
+| `minio_node_mem_buffers_avg`   | Buffers memory on the node (avg).   |
+| `minio_node_mem_buffers_max`   | Buffers memory on the node (max).   |
+| `minio_node_mem_cache`         | Cache memory on the node.           |
+| `minio_node_mem_cache_avg`     | Cache memory on the node (avg).     |
+| `minio_node_mem_cache_max`     | Cache memory on the node (max).     |
+| `minio_node_mem_free`          | Free memory on the node.            |
+| `minio_node_mem_free_avg`      | Free memory on the node (avg).      |
+| `minio_node_mem_free_max`      | Free memory on the node (max).      |
+| `minio_node_mem_shared`        | Shared memory on the node.          |
+| `minio_node_mem_shared_avg`    | Shared memory on the node (avg).    |
+| `minio_node_mem_shared_max`    | Shared memory on the node (max).    |
+| `minio_node_mem_total`         | Total memory on the node.           |
+| `minio_node_mem_total_avg`     | Total memory on the node (avg).     |
+| `minio_node_mem_total_max`     | Total memory on the node (max).     |
+| `minio_node_mem_used`          | Used memory on the node.            |
+| `minio_node_mem_used_avg`      | Used memory on the node (avg).      |
+| `minio_node_mem_used_max`      | Used memory on the node (max).      |

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2023-09-27T15-22-50Z
+  image: quay.io/minio/minio:RELEASE.2023-10-14T05-17-22Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

docs/site-replication/run-multi-site-ldap.sh

@@ -247,7 +247,7 @@ if [ $? -ne 0 ]; then
 	echo "expecting object to be present. exiting.."
 	exit_1
 fi
-./mc tag set --version-id "${vID}" minio2/newbucket/README.md "k=v"
+./mc tag set --version-id "${vID}" minio2/newbucket/README.md "key=val"
 if [ $? -ne 0 ]; then
 	echo "expecting tag set to be successful. exiting.."
 	exit_1
@@ -353,4 +353,21 @@ if [ $? -ne 0 ]; then
 	exit_1
 fi
 
+# force a resync after removing all site replication
+./mc admin replicate rm --all --force minio1
+./mc rb minio2 --force --dangerous
+./mc admin replicate add minio1 minio2
+./mc admin replicate resync start minio1 minio2
+sleep 30
+
+./mc ls -r --versions minio1/newbucket >/tmp/minio1.txt
+./mc ls -r --versions minio2/newbucket >/tmp/minio2.txt
+
+out=$(diff -qpruN /tmp/minio1.txt /tmp/minio2.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+	echo "BUG: expected no missing entries after replication resync: $out"
+	exit 1
+fi
+
 cleanup

docs/site-replication/run-multi-site-minio-idp.sh

@@ -234,7 +234,7 @@ if [ $? -ne 0 ]; then
 	echo "expecting object to be present. exiting.."
 	exit_1
 fi
-./mc tag set --version-id "${vID}" minio2/newbucket/README.md "k=v"
+./mc tag set --version-id "${vID}" minio2/newbucket/README.md "key=val"
 if [ $? -ne 0 ]; then
 	echo "expecting tag set to be successful. exiting.."
 	exit_1
@@ -370,3 +370,20 @@ if [ $? -ne 0 ]; then
 	echo "expected 'bucket2' delete and 'newbucket2' creation to have replicated, exiting..."
 	exit_1
 fi
+
+# force a resync after removing all site replication
+./mc admin replicate rm --all --force minio1
+./mc rb minio2 --force --dangerous
+./mc admin replicate add minio1 minio2
+./mc admin replicate resync start minio1 minio2
+sleep 30
+
+./mc ls -r --versions minio1/newbucket >/tmp/minio1.txt
+./mc ls -r --versions minio2/newbucket >/tmp/minio2.txt
+
+out=$(diff -qpruN /tmp/minio1.txt /tmp/minio2.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+	echo "BUG: expected no missing entries after replication resync: $out"
+	exit 1
+fi

docs/site-replication/run-multi-site-oidc.sh

@@ -293,3 +293,20 @@ if [ $? -ne 0 ]; then
 	echo "expected 'bucket2' delete and 'newbucket2' creation to have replicated, exiting..."
 	exit_1
 fi
+
+# force a resync after removing all site replication
+./mc admin replicate rm --all --force minio1
+./mc rb minio2 --force --dangerous
+./mc admin replicate add minio1 minio2
+./mc admin replicate resync start minio1 minio2
+sleep 30
+
+./mc ls -r --versions minio1/newbucket >/tmp/minio1.txt
+./mc ls -r --versions minio2/newbucket >/tmp/minio2.txt
+
+out=$(diff -qpruN /tmp/minio1.txt /tmp/minio2.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+	echo "BUG: expected no missing entries after replication resync: $out"
+	exit 1
+fi