Send kafka notification messages in batches when queue_dir is enabled (#18164)

Fixes #18124

.github/workflows/mint.yml

@@ -37,7 +37,11 @@ jobs:
 
       - name: build-minio
         run: |
-          TAG="minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
+          TAG="quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
+
+      - name: multipart uploads test
+        run: |
+          ${GITHUB_WORKSPACE}/.github/workflows/multipart/migrate.sh "${{ steps.vars.outputs.sha_short }}"
 
       - name: compress and encrypt
         run: |
@@ -59,7 +63,14 @@ jobs:
              docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml down || true
              docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml rm || true
           done
-          docker rmi -f minio/minio:${{ steps.vars.outputs.sha_short }}
+
+          docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site1.yaml rm -s -f || true
+          docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site2.yaml rm -s -f || true
+          for volume in $(docker volume ls -q | grep minio); do
+            docker volume rm ${volume} || true
+          done
+
+          docker rmi -f quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}
           docker system prune -f || true
           docker volume prune -f || true
           docker volume rm $(docker volume ls -q -f dangling=true) || true

.github/workflows/mint/minio-compress-encrypt.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: minio/minio:${JOB_NAME}
+  image: quay.io/minio/minio:${JOB_NAME}
   command: server --console-address ":9001" http://minio{1...4}/cdata{1...2}
   expose:
     - "9000"

.github/workflows/mint/minio-erasure.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: minio/minio:${JOB_NAME}
+  image: quay.io/minio/minio:${JOB_NAME}
   command: server --console-address ":9001" edata{1...4}
   expose:
     - "9000"

.github/workflows/mint/minio-pools.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: minio/minio:${JOB_NAME}
+  image: quay.io/minio/minio:${JOB_NAME}
   command: server --console-address ":9001" http://minio{1...4}/pdata{1...2} http://minio{5...8}/pdata{1...2}
   expose:
     - "9000"

.github/workflows/multipart/docker-compose-site1.yaml

@@ -0,0 +1,66 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: quay.io/minio/minio:${RELEASE}
+  command: server http://site1-minio{1...4}/data{1...2}
+  environment:
+    - MINIO_PROMETHEUS_AUTH_TYPE=public
+    - CI=true
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  site1-minio1:
+    <<: *minio-common
+    hostname: site1-minio1
+    volumes:
+      - site1-data1-1:/data1
+      - site1-data1-2:/data2
+
+  site1-minio2:
+    <<: *minio-common
+    hostname: site1-minio2
+    volumes:
+      - site1-data2-1:/data1
+      - site1-data2-2:/data2
+
+  site1-minio3:
+    <<: *minio-common
+    hostname: site1-minio3
+    volumes:
+      - site1-data3-1:/data1
+      - site1-data3-2:/data2
+
+  site1-minio4:
+    <<: *minio-common
+    hostname: site1-minio4
+    volumes:
+      - site1-data4-1:/data1
+      - site1-data4-2:/data2
+
+  site1-nginx:
+    image: nginx:1.19.2-alpine
+    hostname: site1-nginx
+    volumes:
+      - ./nginx-site1.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9001:9001"
+    depends_on:
+      - site1-minio1
+      - site1-minio2
+      - site1-minio3
+      - site1-minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  site1-data1-1:
+  site1-data1-2:
+  site1-data2-1:
+  site1-data2-2:
+  site1-data3-1:
+  site1-data3-2:
+  site1-data4-1:
+  site1-data4-2:

.github/workflows/multipart/docker-compose-site2.yaml

@@ -0,0 +1,66 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: quay.io/minio/minio:${RELEASE}
+  command: server http://site2-minio{1...4}/data{1...2}
+  environment:
+    - MINIO_PROMETHEUS_AUTH_TYPE=public
+    - CI=true
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  site2-minio1:
+    <<: *minio-common
+    hostname: site2-minio1
+    volumes:
+      - site2-data1-1:/data1
+      - site2-data1-2:/data2
+
+  site2-minio2:
+    <<: *minio-common
+    hostname: site2-minio2
+    volumes:
+      - site2-data2-1:/data1
+      - site2-data2-2:/data2
+
+  site2-minio3:
+    <<: *minio-common
+    hostname: site2-minio3
+    volumes:
+      - site2-data3-1:/data1
+      - site2-data3-2:/data2
+
+  site2-minio4:
+    <<: *minio-common
+    hostname: site2-minio4
+    volumes:
+      - site2-data4-1:/data1
+      - site2-data4-2:/data2
+
+  site2-nginx:
+    image: nginx:1.19.2-alpine
+    hostname: site2-nginx
+    volumes:
+      - ./nginx-site2.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9002:9002"
+    depends_on:
+      - site2-minio1
+      - site2-minio2
+      - site2-minio3
+      - site2-minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  site2-data1-1:
+  site2-data1-2:
+  site2-data2-1:
+  site2-data2-2:
+  site2-data3-1:
+  site2-data3-2:
+  site2-data4-1:
+  site2-data4-2:

.github/workflows/multipart/migrate.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+
+set -x
+
+## change working directory
+cd .github/workflows/multipart/
+
+docker-compose -f docker-compose-site1.yaml rm -s -f
+docker-compose -f docker-compose-site2.yaml rm -s -f
+for volume in $(docker volume ls -q | grep minio); do
+	docker volume rm ${volume}
+done
+
+if [ ! -f ./mc ]; then
+	wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
+		chmod +x mc
+fi
+
+(
+	cd /tmp
+	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
+)
+
+export RELEASE=RELEASE.2023-08-29T23-07-35Z
+
+docker-compose -f docker-compose-site1.yaml up -d
+docker-compose -f docker-compose-site2.yaml up -d
+
+sleep 30s
+
+./mc alias set site1 http://site1-nginx:9001 minioadmin minioadmin --api s3v4
+./mc alias set site2 http://site2-nginx:9002 minioadmin minioadmin --api s3v4
+
+./mc ready site1/
+./mc ready site2/
+
+./mc admin replicate add site1 site2
+./mc mb site1/testbucket/
+./mc cp -r --quiet /usr/bin site1/testbucket/
+
+sleep 5
+
+s3-check-md5 -h
+
+failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+
+if [ $failed_count_site1 -ne 0 ]; then
+	echo "failed with multipart on site1 uploads"
+	exit 1
+fi
+
+if [ $failed_count_site2 -ne 0 ]; then
+	echo "failed with multipart on site2 uploads"
+	exit 1
+fi
+
+./mc cp -r --quiet /usr/bin site1/testbucket/
+
+sleep 5
+
+failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+
+## we do not need to fail here, since we are going to test
+## upgrading to master, healing and being able to recover
+## the last version.
+if [ $failed_count_site1 -ne 0 ]; then
+	echo "failed with multipart on site1 uploads ${failed_count_site1}"
+fi
+
+if [ $failed_count_site2 -ne 0 ]; then
+	echo "failed with multipart on site2 uploads ${failed_count_site2}"
+fi
+
+export RELEASE=${1}
+
+docker-compose -f docker-compose-site1.yaml up -d
+docker-compose -f docker-compose-site2.yaml up -d
+
+./mc ready site1/
+./mc ready site2/
+
+for i in $(seq 1 10); do
+	# mc admin heal -r --remove when used against a LB endpoint
+	# behaves flaky, let this run 10 times before giving up
+	./mc admin heal -r --remove --json site1/ 2>&1 >/dev/null
+	./mc admin heal -r --remove --json site2/ 2>&1 >/dev/null
+done
+
+failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+
+if [ $failed_count_site1 -ne 0 ]; then
+	echo "failed with multipart on site1 uploads"
+	exit 1
+fi
+
+if [ $failed_count_site2 -ne 0 ]; then
+	echo "failed with multipart on site2 uploads"
+	exit 1
+fi
+
+docker-compose -f docker-compose-site1.yaml rm -s -f
+docker-compose -f docker-compose-site2.yaml rm -s -f
+for volume in $(docker volume ls -q | grep minio); do
+	docker volume rm ${volume}
+done
+
+docker system prune -f || true
+docker volume prune -f || true
+docker volume rm $(docker volume ls -q -f dangling=true) || true
+
+## change working directory
+cd ../../../

.github/workflows/multipart/nginx-site1.conf

@@ -0,0 +1,61 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server site1-minio1:9000;
+        server site1-minio2:9000;
+        server site1-minio3:9000;
+        server site1-minio4:9000;
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+}

.github/workflows/multipart/nginx-site2.conf

@@ -0,0 +1,61 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server site2-minio1:9000;
+        server site2-minio2:9000;
+        server site2-minio3:9000;
+        server site2-minio4:9000;
+    }
+
+    server {
+        listen       9002;
+        listen  [::]:9002;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+}

.github/workflows/run-mint.sh

@@ -16,7 +16,7 @@ docker volume rm $(docker volume ls -f dangling=true) || true
 cd .github/workflows/mint
 
 docker-compose -f minio-${MODE}.yaml up -d
-sleep 5m
+sleep 30s
 
 docker system prune -f || true
 docker volume prune -f || true

Makefile

@@ -6,7 +6,7 @@ GOARCH := $(shell go env GOARCH)
 GOOS := $(shell go env GOOS)
 
 VERSION ?= $(shell git describe --tags)
-TAG ?= "minio/minio:$(VERSION)"
+TAG ?= "quay.io/minio/minio:$(VERSION)"
 
 GOLANGCI_VERSION = v1.51.2
 GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)

cmd/admin-handlers.go

@@ -1581,7 +1581,9 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 
 	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	traceCh := make(chan madmin.TraceInfo, 4000)
+
+	// Keep 100k buffered channel, should be sufficient to ensure we do not lose any events.
+	traceCh := make(chan madmin.TraceInfo, 100000)
 
 	peers, _ := newPeerRestClients(globalEndpoints)
 
@@ -1605,7 +1607,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		peer.Trace(traceCh, ctx.Done(), traceOpts)
 	}
 
-	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	keepAliveTicker := time.NewTicker(time.Second)
 	defer keepAliveTicker.Stop()
 
 	enc := json.NewEncoder(w)

cmd/bucket-replication.go

@@ -376,8 +376,10 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 			} else {
 				// can be the case that other cluster is down and duplicate `mc rm --vid`
 				// is issued - this still needs to be replicated back to the other target
-				replicate = oi.VersionPurgeStatus == Pending || oi.VersionPurgeStatus == Failed
-				dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync))
+				if !oi.VersionPurgeStatus.Empty() {
+					replicate = oi.VersionPurgeStatus == Pending || oi.VersionPurgeStatus == Failed
+					dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync))
+				}
 				continue
 			}
 		}

cmd/data-scanner.go

@@ -929,7 +929,8 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 
 	versionID := oi.VersionID
 	rCfg, _ := globalBucketObjectLockSys.Get(i.bucket)
-	lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi)
+	replcfg, _ := getReplicationConfig(ctx, i.bucket)
+	lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, replcfg, oi)
 	if i.debug {
 		if versionID != "" {
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, lcEvt.Action)
@@ -1129,7 +1130,7 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 	return size
 }
 
-func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo) lifecycle.Event {
+func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, obj ObjectInfo) lifecycle.Event {
 	event := lc.Eval(obj.ToLifecycleOpts())
 	if serverDebugLog {
 		console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", event.Action)
@@ -1161,6 +1162,9 @@ func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr loc
 			}
 			return lifecycle.Event{Action: lifecycle.NoneAction}
 		}
+		if rcfg != nil && !obj.VersionPurgeStatus.Empty() && rcfg.HasActiveRules(obj.Name, true) {
+			return lifecycle.Event{Action: lifecycle.NoneAction}
+		}
 	}
 
 	return event

cmd/data-usage-cache.go

@@ -924,7 +924,7 @@ type objectIO interface {
 // load the cache content with name from minioMetaBackgroundOpsBucket.
 // Only backend errors are returned as errors.
 // The loader is optimistic and has no locking, but tries 5 times before giving up.
-// If the object is not found or unable to deserialize d is cleared and nil error is returned.
+// If the object is not found, a nil error with empty data usage cache is returned.
 func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error {
 	// By defaut, empty data usage cache
 	*d = dataUsageCache{}
@@ -960,13 +960,19 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string)
 		if !retry {
 			break
 		}
-		retry, _ = load(name+".bkp", 30*time.Second)
-		if !retry {
+		retry, err = load(name+".bkp", 30*time.Second)
+		if err == nil && !retry {
+			// Only return when we have valid data from the backup
 			break
 		}
 		retries++
 		time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
 	}
+
+	if retries == 5 {
+		logger.LogOnceIf(ctx, fmt.Errorf("maximum retry reached to load the data usage cache `%s`", name), "retry-loading-data-usage-cache")
+	}
+
 	return nil
 }
 

cmd/erasure-object.go

@@ -1628,10 +1628,12 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 
 	var lc *lifecycle.Lifecycle
 	var rcfg lock.Retention
+	var replcfg *replication.Config
 	if opts.Expiration.Expire {
 		// Check if the current bucket has a configured lifecycle policy
 		lc, _ = globalLifecycleSys.Get(bucket)
 		rcfg, _ = globalBucketObjectLockSys.Get(bucket)
+		replcfg, _ = getReplicationConfig(ctx, bucket)
 	}
 
 	// expiration attempted on a bucket with no lifecycle
@@ -1684,7 +1686,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 
 	if opts.Expiration.Expire {
 		if gerr == nil {
-			evt := evalActionFromLifecycle(ctx, *lc, rcfg, goi)
+			evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi)
 			var isErr bool
 			switch evt.Action {
 			case lifecycle.NoneAction:

cmd/erasure-server-pool-decom.go

@@ -734,6 +734,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 
 	// Check if bucket is object locked.
 	lr, _ := globalBucketObjectLockSys.Get(bi.Name)
+	rcfg, _ := getReplicationConfig(ctx, bi.Name)
 
 	for setIdx, set := range pool.sets {
 		set := set
@@ -745,7 +746,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			versioned := vc != nil && vc.Versioned(object)
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)
 
-			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+			evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
 			switch {
 			case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
 				applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom)
@@ -1048,6 +1049,7 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error
 
 			// Check if bucket is object locked.
 			lr, _ := globalBucketObjectLockSys.Get(bi.Name)
+			rcfg, _ := getReplicationConfig(ctx, bi.Name)
 
 			filterLifecycle := func(bucket, object string, fi FileInfo) bool {
 				if lc == nil {
@@ -1056,7 +1058,7 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error
 				versioned := vc != nil && vc.Versioned(object)
 				objInfo := fi.ToObjectInfo(bucket, object, versioned)
 
-				evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+				evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
 				switch {
 				case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
 					applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom)

cmd/erasure-server-pool-rebalance.go

@@ -440,6 +440,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 	lc, _ := globalLifecycleSys.Get(bucket)
 	// Check if bucket is object locked.
 	lr, _ := globalBucketObjectLockSys.Get(bucket)
+	rcfg, _ := getReplicationConfig(ctx, bucket)
 
 	pool := z.serverPools[poolIdx]
 	const envRebalanceWorkers = "_MINIO_REBALANCE_WORKERS"
@@ -467,7 +468,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 			versioned := vc != nil && vc.Versioned(object)
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)
 
-			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+			evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
 			if evt.Action.Delete() {
 				globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_Rebal)
 				return true

cmd/erasure-server-pool.go

@@ -25,6 +25,7 @@ import (
 	"io"
 	"math/rand"
 	"net/http"
+	"path"
 	"sort"
 	"strconv"
 	"strings"
@@ -1323,6 +1324,85 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	}
 	opts.setBucketMeta(ctx)
 
+	ri := logger.GetReqInfo(ctx)
+	hadoop := ri != nil && strings.Contains(ri.UserAgent, `Hadoop `) && strings.Contains(ri.UserAgent, "scala/")
+	matches := func() bool {
+		if prefix == "" {
+			return false
+		}
+		// List of standard files supported by s3a
+		// that involves a List() on a directory
+		// where directory is actually an object on
+		// namespace.
+		for _, k := range []string{
+			"_SUCCESS/",
+			".parquet/",
+			".csv/",
+			".json/",
+			".avro/",
+			".orc/",
+			".txt/",
+			// Add any other files in future
+		} {
+			if strings.HasSuffix(prefix, k) {
+				return true
+			}
+		}
+		return false
+	}
+	if hadoop && matches() && delimiter == SlashSeparator && maxKeys == 2 && marker == "" {
+		// Optimization for Spark/Hadoop workload where spark sends a garbage
+		// request of this kind
+		//
+		// GET /testbucket/?list-type=2&delimiter=%2F&max-keys=2&prefix=parquet%2F_SUCCESS%2F&fetch-owner=false
+		//
+		// Here spark is expecting that the List() return empty instead, so from MinIO's point
+		// of view if we simply do a GetObjectInfo() on this prefix by treating it as an object
+		// We save a lot of calls over the network.
+		//
+		// This happens repeatedly for all objects that are created concurrently() avoiding this
+		// as a List() call is an important performance improvement.
+		//
+		// Spark based s3a committers are a  big enough use-case to have this optimization.
+		//
+		// A sample code to see the improvements is as follows, this sample code is
+		// simply a read on JSON from MinIO and write it back as "parquet".
+		//
+		// import org.apache.spark.sql.SparkSession
+		// import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
+		// object SparkJSONRead {
+		//   def main(args: Array[String]): Unit = {
+		//     val spark:SparkSession = SparkSession.builder()
+		//       .appName("SparkByExample")
+		//       .master("local[1]").getOrCreate()
+		//
+		//     spark.sparkContext.setLogLevel("ERROR")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.endpoint", "http://minio-lb:9000")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.path.style.access", "true")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.access.key", "minioadmin")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.secret.key", "minioadmin")
+		//
+		//     val df = spark.read.json("s3a://testbucket/s3.json")
+		//
+		//     df.write.parquet("s3a://testbucket/parquet/")
+		//   }
+		// }
+		objInfo, err := z.GetObjectInfo(ctx, bucket, path.Dir(prefix), ObjectOptions{NoLock: true})
+		if err == nil {
+			if opts.Lifecycle != nil {
+				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, opts.Replication.Config, objInfo)
+				if evt.Action.Delete() {
+					globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects)
+					if !evt.Action.DeleteRestored() {
+						// Skip entry if ILM action was DeleteVersionAction or DeleteAction
+						return loi, nil
+					}
+				}
+			}
+			return loi, nil
+		}
+	}
+
 	if len(prefix) > 0 && maxKeys == 1 && marker == "" {
 		// Optimization for certain applications like
 		// - Cohesity
@@ -1334,7 +1414,7 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		objInfo, err := z.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true})
 		if err == nil {
 			if opts.Lifecycle != nil {
-				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo)
+				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, opts.Replication.Config, objInfo)
 				if evt.Action.Delete() {
 					globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects)
 					if !evt.Action.DeleteRestored() {

cmd/generic-handlers.go

@@ -231,7 +231,8 @@ func guessIsMetricsReq(req *http.Request) bool {
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsPathLegacy ||
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ClusterPath ||
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2NodePath ||
-		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2BucketPath
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2BucketPath ||
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ResourcePath
 }
 
 // guessIsRPCReq - returns true if the request is for an RPC endpoint.

cmd/listen-notification-handlers.go

@@ -116,7 +116,7 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 
 	// Listen Publisher and peer-listen-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	listenCh := make(chan event.Event, 4000)
+	listenCh := make(chan event.Event, globalAPIConfig.getRequestsPoolCapacity()*len(globalEndpoints.Hostnames()))
 
 	peers, _ := newPeerRestClients(globalEndpoints)
 

cmd/metacache-server-pool.go

@@ -370,7 +370,7 @@ func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCa
 
 			objInfo := fi.ToObjectInfo(o.Bucket, obj.name, versioned)
 			if o.Lifecycle != nil {
-				act := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo).Action
+				act := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, o.Replication.Config, objInfo).Action
 				skip = act.Delete()
 				if act.DeleteRestored() {
 					// do not skip DeleteRestored* actions
@@ -398,7 +398,7 @@ func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCa
 			objInfo := version.ToObjectInfo(o.Bucket, obj.name, versioned)
 
 			if o.Lifecycle != nil {
-				evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo)
+				evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, o.Replication.Config, objInfo)
 				if evt.Action.Delete() {
 					globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects)
 					if !evt.Action.DeleteRestored() {

cmd/metrics-realtime.go

@@ -19,12 +19,15 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"time"
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/disk"
 	"github.com/minio/minio/internal/net"
+	c "github.com/shirou/gopsutil/v3/cpu"
+	"github.com/shirou/gopsutil/v3/load"
 )
 
 type collectMetricsOpts struct {
@@ -89,6 +92,34 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 			m.Aggregated.Net.NetStats = netStats
 		}
 	}
+	if types.Contains(madmin.MetricsMem) {
+		m.Aggregated.Mem = &madmin.MemMetrics{
+			CollectedAt: UTCNow(),
+		}
+		m.Aggregated.Mem.Info = madmin.GetMemInfo(GlobalContext, globalMinioAddr)
+	}
+	if types.Contains(madmin.MetricsCPU) {
+		m.Aggregated.CPU = &madmin.CPUMetrics{
+			CollectedAt: UTCNow(),
+		}
+		cm, err := c.Times(false)
+		if err != nil {
+			m.Errors = append(m.Errors, err.Error())
+		} else {
+			// not collecting per-cpu stats, so there will be only one element
+			if len(cm) == 1 {
+				m.Aggregated.CPU.TimesStat = &cm[0]
+			} else {
+				m.Errors = append(m.Errors, fmt.Sprintf("Expected one CPU stat, got %d", len(cm)))
+			}
+		}
+		loadStat, err := load.Avg()
+		if err != nil {
+			m.Errors = append(m.Errors, err.Error())
+		} else {
+			m.Aggregated.CPU.LoadStat = loadStat
+		}
+	}
 	// Add types...
 
 	// ByHost is a shallow reference, so careful about sharing.

cmd/metrics-resource.go

@@ -0,0 +1,453 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/shirou/gopsutil/v3/host"
+)
+
+const (
+	resourceMetricsCollectionInterval = time.Minute
+	resourceMetricsCacheInterval      = time.Minute
+
+	// drive stats
+	totalInodes    MetricName = "total_inodes"
+	readsPerSec    MetricName = "reads_per_sec"
+	writesPerSec   MetricName = "writes_per_sec"
+	readsKBPerSec  MetricName = "reads_kb_per_sec"
+	writesKBPerSec MetricName = "writes_kb_per_sec"
+	readsAwait     MetricName = "reads_await"
+	writesAwait    MetricName = "writes_await"
+	percUtil       MetricName = "perc_util"
+	usedInodes     MetricName = "used_inodes"
+
+	// network stats
+	interfaceRxBytes  MetricName = "rx_bytes"
+	interfaceRxErrors MetricName = "rx_errors"
+	interfaceTxBytes  MetricName = "tx_bytes"
+	interfaceTxErrors MetricName = "tx_errors"
+
+	// memory stats
+	memUsed      MetricName = "used"
+	memFree      MetricName = "free"
+	memShared    MetricName = "shared"
+	memBuffers   MetricName = "buffers"
+	memCache     MetricName = "cache"
+	memAvailable MetricName = "available"
+
+	// cpu stats
+	cpuUser   MetricName = "user"
+	cpuSystem MetricName = "system"
+	cpuIOWait MetricName = "iowait"
+	cpuIdle   MetricName = "idle"
+	cpuNice   MetricName = "nice"
+	cpuSteal  MetricName = "steal"
+	cpuLoad1  MetricName = "load1"
+	cpuLoad5  MetricName = "load5"
+	cpuLoad15 MetricName = "load15"
+)
+
+var (
+	resourceCollector *minioResourceCollector
+	// resourceMetricsMap is a map of subsystem to its metrics
+	resourceMetricsMap map[MetricSubsystem]ResourceMetrics
+	// resourceMetricsHelpMap maps metric name to its help string
+	resourceMetricsHelpMap map[MetricName]string
+	resourceMetricsGroups  []*MetricsGroup
+)
+
+// PeerResourceMetrics represents the resource metrics
+// retrieved from a peer, along with errors if any
+type PeerResourceMetrics struct {
+	Metrics map[MetricSubsystem]ResourceMetrics
+	Errors  []string
+}
+
+// ResourceMetrics is a map of unique key identifying
+// a resource metric (e.g. reads_per_sec_{node}_{drive})
+// to its data
+type ResourceMetrics map[string]ResourceMetric
+
+// ResourceMetric represents a single resource metric
+// The metrics are collected from all servers periodically
+// and stored in the resource metrics map.
+// It also maintains the count of number of times this metric
+// was collected since the server started, and the sum,
+// average and max values across the same.
+type ResourceMetric struct {
+	Name   MetricName
+	Labels map[string]string
+
+	// value captured in current cycle
+	Current float64
+
+	// Used when system provides cumulative (since uptime) values
+	// helps in calculating the current value by comparing the new
+	// cumulative value with previous one
+	Cumulative float64
+
+	Max   float64
+	Avg   float64
+	Sum   float64
+	Count uint64
+}
+
+func init() {
+	interval := fmt.Sprintf("%ds", int(resourceMetricsCollectionInterval.Seconds()))
+	resourceMetricsHelpMap = map[MetricName]string{
+		interfaceRxBytes:  "Bytes received on the interface in " + interval,
+		interfaceRxErrors: "Receive errors in " + interval,
+		interfaceTxBytes:  "Bytes transmitted in " + interval,
+		interfaceTxErrors: "Transmit errors in " + interval,
+		total:             "Total memory on the node",
+		memUsed:           "Used memory on the node",
+		memFree:           "Free memory on the node",
+		memShared:         "Shared memory on the node",
+		memBuffers:        "Buffers memory on the node",
+		memCache:          "Cache memory on the node",
+		memAvailable:      "Available memory on the node",
+		readsPerSec:       "Reads per second on a drive",
+		writesPerSec:      "Writes per second on a drive",
+		readsKBPerSec:     "Kilobytes read per second on a drive",
+		writesKBPerSec:    "Kilobytes written per second on a drive",
+		readsAwait:        "Average time for read requests to be served on a drive",
+		writesAwait:       "Average time for write requests to be served on a drive",
+		percUtil:          "Percentage of time the disk was busy since uptime",
+		usedBytes:         "Used bytes on a drive",
+		totalBytes:        "Total bytes on a drive",
+		usedInodes:        "Total inodes used on a drive",
+		totalInodes:       "Total inodes on a drive",
+		cpuUser:           "CPU user time",
+		cpuSystem:         "CPU system time",
+		cpuIdle:           "CPU idle time",
+		cpuIOWait:         "CPU ioWait time",
+		cpuSteal:          "CPU steal time",
+		cpuNice:           "CPU nice time",
+		cpuLoad1:          "CPU load average 1min",
+		cpuLoad5:          "CPU load average 5min",
+		cpuLoad15:         "CPU load average 15min",
+	}
+	resourceMetricsGroups = []*MetricsGroup{
+		getResourceMetrics(),
+	}
+
+	resourceCollector = newMinioResourceCollector(resourceMetricsGroups)
+}
+
+func updateResourceMetrics(subSys MetricSubsystem, name MetricName, val float64, labels map[string]string, isCumulative bool) {
+	subsysMetrics, found := resourceMetricsMap[subSys]
+	if !found {
+		subsysMetrics = ResourceMetrics{}
+	}
+
+	// labels are used to uniquely identify a metric
+	// e.g. reads_per_sec_{drive} inside the map
+	sfx := ""
+	for _, v := range labels {
+		if len(sfx) > 0 {
+			sfx += "_"
+		}
+		sfx += v
+	}
+
+	key := string(name) + "_" + sfx
+	metric, found := subsysMetrics[key]
+	if !found {
+		metric = ResourceMetric{
+			Name:   name,
+			Labels: labels,
+		}
+	}
+
+	if isCumulative {
+		metric.Current = val - metric.Cumulative
+		metric.Cumulative = val
+	} else {
+		metric.Current = val
+	}
+
+	if metric.Current > metric.Max {
+		metric.Max = val
+	}
+
+	metric.Sum += metric.Current
+	metric.Count++
+
+	metric.Avg = metric.Sum / float64(metric.Count)
+	subsysMetrics[key] = metric
+
+	resourceMetricsMap[subSys] = subsysMetrics
+}
+
+func collectDriveMetrics(m madmin.RealtimeMetrics) {
+	upt, _ := host.Uptime()
+	kib := 1 << 10
+	sectorSize := uint64(512)
+
+	for d, dm := range m.ByDisk {
+		stats := dm.IOStats
+		labels := map[string]string{"drive": d}
+		updateResourceMetrics(driveSubsystem, readsPerSec, float64(stats.ReadIOs)/float64(upt), labels, false)
+
+		readBytes := stats.ReadSectors * sectorSize
+		readKib := float64(readBytes) / float64(kib)
+		readKibPerSec := readKib / float64(upt)
+		updateResourceMetrics(driveSubsystem, readsKBPerSec, readKibPerSec, labels, false)
+
+		updateResourceMetrics(driveSubsystem, writesPerSec, float64(stats.WriteIOs)/float64(upt), labels, false)
+
+		writeBytes := stats.WriteSectors * sectorSize
+		writeKib := float64(writeBytes) / float64(kib)
+		writeKibPerSec := writeKib / float64(upt)
+		updateResourceMetrics(driveSubsystem, writesKBPerSec, writeKibPerSec, labels, false)
+
+		rdAwait := 0.0
+		if stats.ReadIOs > 0 {
+			rdAwait = float64(stats.ReadTicks) / float64(stats.ReadIOs)
+		}
+		updateResourceMetrics(driveSubsystem, readsAwait, rdAwait, labels, false)
+
+		wrAwait := 0.0
+		if stats.WriteIOs > 0 {
+			wrAwait = float64(stats.WriteTicks) / float64(stats.WriteIOs)
+		}
+		updateResourceMetrics(driveSubsystem, writesAwait, wrAwait, labels, false)
+
+		updateResourceMetrics(driveSubsystem, percUtil, float64(stats.TotalTicks)/float64(upt*10), labels, false)
+	}
+
+	globalLocalDrivesMu.RLock()
+	gld := globalLocalDrives
+	globalLocalDrivesMu.RUnlock()
+
+	for _, d := range gld {
+		labels := map[string]string{"drive": d.Endpoint().RawPath}
+		di, err := d.DiskInfo(GlobalContext, false)
+		if err == nil {
+			updateResourceMetrics(driveSubsystem, usedBytes, float64(di.Used), labels, false)
+			updateResourceMetrics(driveSubsystem, totalBytes, float64(di.Total), labels, false)
+			updateResourceMetrics(driveSubsystem, usedInodes, float64(di.UsedInodes), labels, false)
+			updateResourceMetrics(driveSubsystem, totalInodes, float64(di.FreeInodes+di.UsedInodes), labels, false)
+		}
+	}
+}
+
+func collectLocalResourceMetrics() {
+	var types madmin.MetricType = madmin.MetricsDisk | madmin.MetricNet | madmin.MetricsMem | madmin.MetricsCPU
+
+	m := collectLocalMetrics(types, collectMetricsOpts{
+		hosts: map[string]struct{}{
+			globalLocalNodeName: {},
+		},
+	})
+
+	for host, hm := range m.ByHost {
+		if len(host) > 0 {
+			if hm.Net != nil && len(hm.Net.NetStats.Name) > 0 {
+				stats := hm.Net.NetStats
+				labels := map[string]string{"interface": stats.Name}
+				updateResourceMetrics(interfaceSubsystem, interfaceRxBytes, float64(stats.RxBytes), labels, true)
+				updateResourceMetrics(interfaceSubsystem, interfaceRxErrors, float64(stats.RxErrors), labels, true)
+				updateResourceMetrics(interfaceSubsystem, interfaceTxBytes, float64(stats.TxBytes), labels, true)
+				updateResourceMetrics(interfaceSubsystem, interfaceTxErrors, float64(stats.TxErrors), labels, true)
+			}
+			if hm.Mem != nil && len(hm.Mem.Info.Addr) > 0 {
+				labels := map[string]string{}
+				stats := hm.Mem.Info
+				updateResourceMetrics(memSubsystem, total, float64(stats.Total), labels, false)
+				updateResourceMetrics(memSubsystem, memUsed, float64(stats.Used), labels, false)
+				updateResourceMetrics(memSubsystem, memFree, float64(stats.Free), labels, false)
+				updateResourceMetrics(memSubsystem, memShared, float64(stats.Shared), labels, false)
+				updateResourceMetrics(memSubsystem, memBuffers, float64(stats.Buffers), labels, false)
+				updateResourceMetrics(memSubsystem, memAvailable, float64(stats.Available), labels, false)
+				updateResourceMetrics(memSubsystem, memCache, float64(stats.Cache), labels, false)
+			}
+			if hm.CPU != nil {
+				labels := map[string]string{}
+				ts := hm.CPU.TimesStat
+				if ts != nil {
+					tot := ts.User + ts.System + ts.Idle + ts.Iowait + ts.Nice + ts.Steal
+					cpuUserVal := math.Round(ts.User/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuUser, cpuUserVal, labels, false)
+					cpuSystemVal := math.Round(ts.System/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuSystem, cpuSystemVal, labels, false)
+					cpuIdleVal := math.Round(ts.Idle/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuIdle, cpuIdleVal, labels, false)
+					cpuIOWaitVal := math.Round(ts.Iowait/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuIOWait, cpuIOWaitVal, labels, false)
+					cpuNiceVal := math.Round(ts.Nice/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuNice, cpuNiceVal, labels, false)
+					cpuStealVal := math.Round(ts.Steal/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuSteal, cpuStealVal, labels, false)
+				}
+				ls := hm.CPU.LoadStat
+				if ls != nil {
+					updateResourceMetrics(cpuSubsystem, cpuLoad1, ls.Load1, labels, false)
+					updateResourceMetrics(cpuSubsystem, cpuLoad5, ls.Load5, labels, false)
+					updateResourceMetrics(cpuSubsystem, cpuLoad15, ls.Load15, labels, false)
+				}
+			}
+			break // only one host expected
+		}
+	}
+
+	collectDriveMetrics(m)
+}
+
+// startResourceMetricsCollection - starts the job for collecting resource metrics
+func startResourceMetricsCollection() {
+	resourceMetricsMap = map[MetricSubsystem]ResourceMetrics{}
+	metricsTimer := time.NewTimer(resourceMetricsCollectionInterval)
+	defer metricsTimer.Stop()
+
+	collectLocalResourceMetrics()
+
+	for {
+		select {
+		case <-GlobalContext.Done():
+			return
+		case <-metricsTimer.C:
+			collectLocalResourceMetrics()
+
+			// Reset the timer for next cycle.
+			metricsTimer.Reset(resourceMetricsCollectionInterval)
+		}
+	}
+}
+
+// minioResourceCollector is the Collector for resource metrics
+type minioResourceCollector struct {
+	metricsGroups []*MetricsGroup
+	desc          *prometheus.Desc
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+func (c *minioResourceCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.desc
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (c *minioResourceCollector) Collect(out chan<- prometheus.Metric) {
+	var wg sync.WaitGroup
+	publish := func(in <-chan Metric) {
+		defer wg.Done()
+		for metric := range in {
+			labels, values := getOrderedLabelValueArrays(metric.VariableLabels)
+			collectMetric(metric, labels, values, "resource", out)
+		}
+	}
+
+	// Call peer api to fetch metrics
+	wg.Add(2)
+	go publish(ReportMetrics(GlobalContext, c.metricsGroups))
+	go publish(globalNotificationSys.GetResourceMetrics(GlobalContext))
+	wg.Wait()
+}
+
+// newMinioResourceCollector describes the collector
+// and returns reference of minio resource Collector
+// It creates the Prometheus Description which is used
+// to define Metric and  help string
+func newMinioResourceCollector(metricsGroups []*MetricsGroup) *minioResourceCollector {
+	return &minioResourceCollector{
+		metricsGroups: metricsGroups,
+		desc:          prometheus.NewDesc("minio_resource_stats", "Resource statistics exposed by MinIO server", nil, nil),
+	}
+}
+
+func prepareResourceMetrics(rm ResourceMetric, subSys MetricSubsystem, requireAvgMax bool) []Metric {
+	help := resourceMetricsHelpMap[rm.Name]
+	name := rm.Name
+
+	metrics := []Metric{}
+	metrics = append(metrics, Metric{
+		Description:    getResourceMetricDescription(subSys, name, help),
+		Value:          rm.Current,
+		VariableLabels: rm.Labels,
+	})
+
+	if requireAvgMax {
+		avgName := MetricName(fmt.Sprintf("%s_avg", name))
+		avgHelp := fmt.Sprintf("%s (avg)", help)
+		metrics = append(metrics, Metric{
+			Description:    getResourceMetricDescription(subSys, avgName, avgHelp),
+			Value:          math.Round(rm.Avg*100) / 100,
+			VariableLabels: rm.Labels,
+		})
+
+		maxName := MetricName(fmt.Sprintf("%s_max", name))
+		maxHelp := fmt.Sprintf("%s (max)", help)
+		metrics = append(metrics, Metric{
+			Description:    getResourceMetricDescription(subSys, maxName, maxHelp),
+			Value:          rm.Max,
+			VariableLabels: rm.Labels,
+		})
+	}
+
+	return metrics
+}
+
+func getResourceMetricDescription(subSys MetricSubsystem, name MetricName, help string) MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: subSys,
+		Name:      name,
+		Help:      help,
+		Type:      gaugeMetric,
+	}
+}
+
+func getResourceMetrics() *MetricsGroup {
+	mg := &MetricsGroup{
+		cacheInterval: resourceMetricsCacheInterval,
+	}
+	mg.RegisterRead(func(ctx context.Context) []Metric {
+		metrics := []Metric{}
+
+		subSystems := []MetricSubsystem{interfaceSubsystem, memSubsystem, driveSubsystem, cpuSubsystem}
+		for _, subSys := range subSystems {
+			stats, found := resourceMetricsMap[subSys]
+			if found {
+				requireAvgMax := true
+				if subSys == driveSubsystem {
+					requireAvgMax = false
+				}
+				for _, m := range stats {
+					metrics = append(metrics, prepareResourceMetrics(m, subSys, requireAvgMax)...)
+				}
+			}
+		}
+
+		return metrics
+	})
+	return mg
+}
+
+// metricsResourceHandler is the prometheus handler for resource metrics
+func metricsResourceHandler() http.Handler {
+	return metricsHTTPHandler(resourceCollector, "handler.MetricsResource")
+}

cmd/metrics-router.go

@@ -25,10 +25,11 @@ import (
 )
 
 const (
-	prometheusMetricsPathLegacy    = "/prometheus/metrics"
-	prometheusMetricsV2ClusterPath = "/v2/metrics/cluster"
-	prometheusMetricsV2BucketPath  = "/v2/metrics/bucket"
-	prometheusMetricsV2NodePath    = "/v2/metrics/node"
+	prometheusMetricsPathLegacy     = "/prometheus/metrics"
+	prometheusMetricsV2ClusterPath  = "/v2/metrics/cluster"
+	prometheusMetricsV2BucketPath   = "/v2/metrics/bucket"
+	prometheusMetricsV2NodePath     = "/v2/metrics/node"
+	prometheusMetricsV2ResourcePath = "/v2/metrics/resource"
 )
 
 // Standard env prometheus auth type
@@ -57,4 +58,5 @@ func registerMetricsRouter(router *mux.Router) {
 	metricsRouter.Handle(prometheusMetricsV2ClusterPath, auth(metricsServerHandler()))
 	metricsRouter.Handle(prometheusMetricsV2BucketPath, auth(metricsBucketHandler()))
 	metricsRouter.Handle(prometheusMetricsV2NodePath, auth(metricsNodeHandler()))
+	metricsRouter.Handle(prometheusMetricsV2ResourcePath, auth(metricsResourceHandler()))
 }

cmd/metrics-v2.go

@@ -132,6 +132,9 @@ const (
 	capacityRawSubsystem      MetricSubsystem = "capacity_raw"
 	capacityUsableSubsystem   MetricSubsystem = "capacity_usable"
 	driveSubsystem            MetricSubsystem = "drive"
+	interfaceSubsystem        MetricSubsystem = "if"
+	memSubsystem              MetricSubsystem = "mem"
+	cpuSubsystem              MetricSubsystem = "cpu_avg"
 	storageClassSubsystem     MetricSubsystem = "storage_class"
 	fileDescriptorSubsystem   MetricSubsystem = "file_descriptor"
 	goRoutines                MetricSubsystem = "go_routine"
@@ -539,12 +542,12 @@ func getNodeRRSParityMD() MetricDescription {
 	}
 }
 
-func getNodeDrivesFreeInodes() MetricDescription {
+func getNodeDrivesFreeInodesMD() MetricDescription {
 	return MetricDescription{
 		Namespace: nodeMetricNamespace,
 		Subsystem: driveSubsystem,
 		Name:      freeInodes,
-		Help:      "Total free inodes",
+		Help:      "Free inodes on a drive",
 		Type:      gaugeMetric,
 	}
 }
@@ -3207,7 +3210,7 @@ func getLocalStorageMetrics() *MetricsGroup {
 			})
 
 			metrics = append(metrics, Metric{
-				Description:    getNodeDrivesFreeInodes(),
+				Description:    getNodeDrivesFreeInodesMD(),
 				Value:          float64(disk.FreeInodes),
 				VariableLabels: map[string]string{"drive": disk.DrivePath},
 			})
@@ -3546,6 +3549,61 @@ func getKMSMetrics() *MetricsGroup {
 	return mg
 }
 
+func collectMetric(metric Metric, labels []string, values []string, metricName string, out chan<- prometheus.Metric) {
+	if metric.Description.Type == histogramMetric {
+		if metric.Histogram == nil {
+			return
+		}
+		for k, v := range metric.Histogram {
+			pmetric, err := prometheus.NewConstMetric(
+				prometheus.NewDesc(
+					prometheus.BuildFQName(string(metric.Description.Namespace),
+						string(metric.Description.Subsystem),
+						string(metric.Description.Name)),
+					metric.Description.Help,
+					append(labels, metric.HistogramBucketLabel),
+					metric.StaticLabels,
+				),
+				prometheus.GaugeValue,
+				float64(v),
+				append(values, k)...)
+			if err != nil {
+				// Enable for debugging
+				if serverDebugLog {
+					logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v+%v", err, values, metric.Histogram), metricName+"-metrics-histogram")
+				}
+			} else {
+				out <- pmetric
+			}
+		}
+		return
+	}
+	metricType := prometheus.GaugeValue
+	if metric.Description.Type == counterMetric {
+		metricType = prometheus.CounterValue
+	}
+	pmetric, err := prometheus.NewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName(string(metric.Description.Namespace),
+				string(metric.Description.Subsystem),
+				string(metric.Description.Name)),
+			metric.Description.Help,
+			labels,
+			metric.StaticLabels,
+		),
+		metricType,
+		metric.Value,
+		values...)
+	if err != nil {
+		// Enable for debugging
+		if serverDebugLog {
+			logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v", err, values), metricName+"-metrics")
+		}
+	} else {
+		out <- pmetric
+	}
+}
+
 type minioBucketCollector struct {
 	metricsGroups []*MetricsGroup
 	desc          *prometheus.Desc
@@ -3570,52 +3628,7 @@ func (c *minioBucketCollector) Collect(out chan<- prometheus.Metric) {
 		defer wg.Done()
 		for metric := range in {
 			labels, values := getOrderedLabelValueArrays(metric.VariableLabels)
-			if metric.Description.Type == histogramMetric {
-				if metric.Histogram == nil {
-					continue
-				}
-				for k, v := range metric.Histogram {
-					pmetric, err := prometheus.NewConstMetric(
-						prometheus.NewDesc(
-							prometheus.BuildFQName(string(metric.Description.Namespace),
-								string(metric.Description.Subsystem),
-								string(metric.Description.Name)),
-							metric.Description.Help,
-							append(labels, metric.HistogramBucketLabel),
-							metric.StaticLabels,
-						),
-						prometheus.GaugeValue,
-						float64(v),
-						append(values, k)...)
-					if err != nil {
-						logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v+%v", err, values, metric.Histogram), "bucket-metrics-histogram")
-					} else {
-						out <- pmetric
-					}
-				}
-				continue
-			}
-			metricType := prometheus.GaugeValue
-			if metric.Description.Type == counterMetric {
-				metricType = prometheus.CounterValue
-			}
-			pmetric, err := prometheus.NewConstMetric(
-				prometheus.NewDesc(
-					prometheus.BuildFQName(string(metric.Description.Namespace),
-						string(metric.Description.Subsystem),
-						string(metric.Description.Name)),
-					metric.Description.Help,
-					labels,
-					metric.StaticLabels,
-				),
-				metricType,
-				metric.Value,
-				values...)
-			if err != nil {
-				logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v", err, values), "bucket-metrics")
-			} else {
-				out <- pmetric
-			}
+			collectMetric(metric, labels, values, "bucket", out)
 		}
 	}
 
@@ -3650,59 +3663,7 @@ func (c *minioClusterCollector) Collect(out chan<- prometheus.Metric) {
 		defer wg.Done()
 		for metric := range in {
 			labels, values := getOrderedLabelValueArrays(metric.VariableLabels)
-			if metric.Description.Type == histogramMetric {
-				if metric.Histogram == nil {
-					continue
-				}
-				for k, v := range metric.Histogram {
-					pmetric, err := prometheus.NewConstMetric(
-						prometheus.NewDesc(
-							prometheus.BuildFQName(string(metric.Description.Namespace),
-								string(metric.Description.Subsystem),
-								string(metric.Description.Name)),
-							metric.Description.Help,
-							append(labels, metric.HistogramBucketLabel),
-							metric.StaticLabels,
-						),
-						prometheus.GaugeValue,
-						float64(v),
-						append(values, k)...)
-
-					if err != nil {
-						// Enable for debugging
-						if serverDebugLog {
-							logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v:%v", err, values, metric.Histogram), "cluster-metrics-histogram")
-						}
-					} else {
-						out <- pmetric
-					}
-				}
-				continue
-			}
-			metricType := prometheus.GaugeValue
-			if metric.Description.Type == counterMetric {
-				metricType = prometheus.CounterValue
-			}
-			pmetric, err := prometheus.NewConstMetric(
-				prometheus.NewDesc(
-					prometheus.BuildFQName(string(metric.Description.Namespace),
-						string(metric.Description.Subsystem),
-						string(metric.Description.Name)),
-					metric.Description.Help,
-					labels,
-					metric.StaticLabels,
-				),
-				metricType,
-				metric.Value,
-				values...)
-			if err != nil {
-				// Enable for debugging
-				if serverDebugLog {
-					logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v", err, values), "cluster-metrics")
-				}
-			} else {
-				out <- pmetric
-			}
+			collectMetric(metric, labels, values, "cluster", out)
 		}
 	}
 
@@ -3835,11 +3796,11 @@ func newMinioCollectorNode(metricsGroups []*MetricsGroup) *minioNodeCollector {
 	}
 }
 
-func metricsBucketHandler() http.Handler {
+func metricsHTTPHandler(c prometheus.Collector, funcName string) http.Handler {
 	registry := prometheus.NewRegistry()
 
 	// Report all other metrics
-	logger.CriticalIf(GlobalContext, registry.Register(bucketCollector))
+	logger.CriticalIf(GlobalContext, registry.Register(c))
 
 	// DefaultGatherers include golang metrics and process metrics.
 	gatherers := prometheus.Gatherers{
@@ -3849,16 +3810,14 @@ func metricsBucketHandler() http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
 		if ok {
-			tc.FuncName = "handler.MetricsBucket"
+			tc.FuncName = funcName
 			tc.ResponseRecorder.LogErrBody = true
 		}
 
 		mfs, err := gatherers.Gather()
-		if err != nil {
-			if len(mfs) == 0 {
-				writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL)
-				return
-			}
+		if err != nil && len(mfs) == 0 {
+			writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL)
+			return
 		}
 
 		contentType := expfmt.Negotiate(r.Header)
@@ -3878,6 +3837,10 @@ func metricsBucketHandler() http.Handler {
 	})
 }
 
+func metricsBucketHandler() http.Handler {
+	return metricsHTTPHandler(bucketCollector, "handler.MetricsBucket")
+}
+
 func metricsServerHandler() http.Handler {
 	registry := prometheus.NewRegistry()
 

cmd/notification.go

@@ -783,6 +783,27 @@ func (sys *NotificationSys) GetMetrics(ctx context.Context, t madmin.MetricType,
 	return reply
 }
 
+// GetResourceMetrics - gets the resource metrics from all nodes excluding self.
+func (sys *NotificationSys) GetResourceMetrics(ctx context.Context) <-chan Metric {
+	if sys == nil {
+		return nil
+	}
+	g := errgroup.WithNErrs(len(sys.peerClients))
+	peerChannels := make([]<-chan Metric, len(sys.peerClients))
+	for index := range sys.peerClients {
+		index := index
+		g.Go(func() error {
+			if sys.peerClients[index] == nil {
+				return errPeerNotReachable
+			}
+			var err error
+			peerChannels[index], err = sys.peerClients[index].GetResourceMetrics(ctx)
+			return err
+		}, index)
+	}
+	return sys.collectPeerMetrics(ctx, peerChannels, g)
+}
+
 // GetSysConfig - Get information about system config
 // (only the config that are of concern to minio)
 func (sys *NotificationSys) GetSysConfig(ctx context.Context) []madmin.SysConfig {
@@ -1122,25 +1143,7 @@ func (sys *NotificationSys) GetBandwidthReports(ctx context.Context, buckets ...
 	return consolidatedReport
 }
 
-// GetBucketMetrics - gets the cluster level bucket metrics from all nodes excluding self.
-func (sys *NotificationSys) GetBucketMetrics(ctx context.Context) <-chan Metric {
-	if sys == nil {
-		return nil
-	}
-	g := errgroup.WithNErrs(len(sys.peerClients))
-	peerChannels := make([]<-chan Metric, len(sys.peerClients))
-	for index := range sys.peerClients {
-		index := index
-		g.Go(func() error {
-			if sys.peerClients[index] == nil {
-				return errPeerNotReachable
-			}
-			var err error
-			peerChannels[index], err = sys.peerClients[index].GetPeerBucketMetrics(ctx)
-			return err
-		}, index)
-	}
-
+func (sys *NotificationSys) collectPeerMetrics(ctx context.Context, peerChannels []<-chan Metric, g *errgroup.Group) <-chan Metric {
 	ch := make(chan Metric)
 	var wg sync.WaitGroup
 	for index, err := range g.Wait() {
@@ -1181,6 +1184,27 @@ func (sys *NotificationSys) GetBucketMetrics(ctx context.Context) <-chan Metric
 	return ch
 }
 
+// GetBucketMetrics - gets the cluster level bucket metrics from all nodes excluding self.
+func (sys *NotificationSys) GetBucketMetrics(ctx context.Context) <-chan Metric {
+	if sys == nil {
+		return nil
+	}
+	g := errgroup.WithNErrs(len(sys.peerClients))
+	peerChannels := make([]<-chan Metric, len(sys.peerClients))
+	for index := range sys.peerClients {
+		index := index
+		g.Go(func() error {
+			if sys.peerClients[index] == nil {
+				return errPeerNotReachable
+			}
+			var err error
+			peerChannels[index], err = sys.peerClients[index].GetPeerBucketMetrics(ctx)
+			return err
+		}, index)
+	}
+	return sys.collectPeerMetrics(ctx, peerChannels, g)
+}
+
 // GetClusterMetrics - gets the cluster metrics from all nodes excluding self.
 func (sys *NotificationSys) GetClusterMetrics(ctx context.Context) <-chan Metric {
 	if sys == nil {
@@ -1199,45 +1223,7 @@ func (sys *NotificationSys) GetClusterMetrics(ctx context.Context) <-chan Metric
 			return err
 		}, index)
 	}
-
-	ch := make(chan Metric)
-	var wg sync.WaitGroup
-	for index, err := range g.Wait() {
-		if err != nil {
-			if sys.peerClients[index] != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress",
-					sys.peerClients[index].host.String())
-				logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), err, sys.peerClients[index].host.String())
-			} else {
-				logger.LogOnceIf(ctx, err, "peer-offline")
-			}
-			continue
-		}
-		wg.Add(1)
-		go func(ctx context.Context, peerChannel <-chan Metric, wg *sync.WaitGroup) {
-			defer wg.Done()
-			for {
-				select {
-				case m, ok := <-peerChannel:
-					if !ok {
-						return
-					}
-					select {
-					case ch <- m:
-					case <-ctx.Done():
-						return
-					}
-				case <-ctx.Done():
-					return
-				}
-			}
-		}(ctx, peerChannels[index], &wg)
-	}
-	go func(wg *sync.WaitGroup, ch chan Metric) {
-		wg.Wait()
-		close(ch)
-	}(&wg, ch)
-	return ch
+	return sys.collectPeerMetrics(ctx, peerChannels, g)
 }
 
 // ServiceFreeze freezes all S3 API calls when 'freeze' is true,

cmd/object-handlers.go

@@ -474,7 +474,8 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 		// Automatically remove the object/version if an expiry lifecycle rule can be applied
 		if lc, err := globalLifecycleSys.Get(bucket); err == nil {
 			rcfg, _ := globalBucketObjectLockSys.Get(bucket)
-			event := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
+			replcfg, _ := getReplicationConfig(ctx, bucket)
+			event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo)
 			if event.Action.Delete() {
 				// apply whatever the expiry rule is.
 				applyExpiryRule(event, lcEventSrc_s3GetObject, objInfo)
@@ -732,7 +733,8 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		// Automatically remove the object/version if an expiry lifecycle rule can be applied
 		if lc, err := globalLifecycleSys.Get(bucket); err == nil {
 			rcfg, _ := globalBucketObjectLockSys.Get(bucket)
-			event := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
+			replcfg, _ := getReplicationConfig(ctx, bucket)
+			event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo)
 			if event.Action.Delete() {
 				// apply whatever the expiry rule is.
 				applyExpiryRule(event, lcEventSrc_s3HeadObject, objInfo)

cmd/peer-rest-client.go

@@ -229,6 +229,33 @@ func (client *peerRESTClient) GetMetrics(ctx context.Context, t madmin.MetricTyp
 	return info, err
 }
 
+func (client *peerRESTClient) GetResourceMetrics(ctx context.Context) (<-chan Metric, error) {
+	respBody, err := client.callWithContext(ctx, peerRESTMethodResourceMetrics, nil, nil, -1)
+	if err != nil {
+		return nil, err
+	}
+	dec := gob.NewDecoder(respBody)
+	ch := make(chan Metric)
+	go func(ch chan<- Metric) {
+		defer func() {
+			xhttp.DrainBody(respBody)
+			close(ch)
+		}()
+		for {
+			var metric Metric
+			if err := dec.Decode(&metric); err != nil {
+				return
+			}
+			select {
+			case <-ctx.Done():
+				return
+			case ch <- metric:
+			}
+		}
+	}(ch)
+	return ch, nil
+}
+
 // GetProcInfo - fetch MinIO process information for a remote node.
 func (client *peerRESTClient) GetProcInfo(ctx context.Context) (info madmin.ProcInfo, err error) {
 	respBody, err := client.callWithContext(ctx, peerRESTMethodProcInfo, nil, nil, -1)

cmd/peer-rest-common.go

@@ -77,6 +77,7 @@ const (
 	peerRESTMethodDevNull                     = "/devnull"
 	peerRESTMethodNetperf                     = "/netperf"
 	peerRESTMethodMetrics                     = "/metrics"
+	peerRESTMethodResourceMetrics             = "/resourcemetrics"
 	peerRESTMethodGetReplicationMRF           = "/getreplicationmrf"
 	peerRESTMethodGetSRMetrics                = "/getsrmetrics"
 )

cmd/peer-rest-server.go

@@ -474,6 +474,22 @@ func (s *peerRESTServer) GetMetricsHandler(w http.ResponseWriter, r *http.Reques
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
 }
 
+func (s *peerRESTServer) GetResourceMetrics(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+
+	enc := gob.NewEncoder(w)
+
+	for m := range ReportMetrics(r.Context(), resourceMetricsGroups) {
+		if err := enc.Encode(m); err != nil {
+			s.writeErrorResponse(w, errors.New("Encoding metric failed: "+err.Error()))
+			return
+		}
+	}
+}
+
 // GetSysConfigHandler - returns system config information.
 // (only the config that are of concern to minio)
 func (s *peerRESTServer) GetSysConfigHandler(w http.ResponseWriter, r *http.Request) {
@@ -939,7 +955,7 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) {
 
 	// Listen Publisher uses nonblocking publish and hence does not wait for slow subscribers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	ch := make(chan event.Event, 2000)
+	ch := make(chan event.Event, globalAPIConfig.getRequestsPoolCapacity())
 
 	err := globalHTTPListen.Subscribe(mask, ch, doneCh, func(ev event.Event) bool {
 		if ev.S3.Bucket.Name != "" && values.Get(peerRESTListenBucket) != "" {
@@ -994,7 +1010,7 @@ func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) {
 
 	// Trace Publisher uses nonblocking publish and hence does not wait for slow subscribers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	ch := make(chan madmin.TraceInfo, 2000)
+	ch := make(chan madmin.TraceInfo, 100000)
 	err = globalTrace.Subscribe(traceOpts.TraceTypes(), ch, r.Context().Done(), func(entry madmin.TraceInfo) bool {
 		return shouldTrace(entry, traceOpts)
 	})
@@ -1143,13 +1159,13 @@ func (s *peerRESTServer) ConsoleLogHandler(w http.ResponseWriter, r *http.Reques
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 
-	ch := make(chan log.Info, 2000)
+	ch := make(chan log.Info, 100000)
 	err := globalConsoleSys.Subscribe(ch, doneCh, "", 0, madmin.LogMaskAll, nil)
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
-	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	keepAliveTicker := time.NewTicker(time.Second)
 	defer keepAliveTicker.Stop()
 
 	enc := gob.NewEncoder(w)
@@ -1438,6 +1454,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcInfo).HandlerFunc(h(server.GetProcInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemInfo).HandlerFunc(h(server.GetMemInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMetrics).HandlerFunc(h(server.GetMetricsHandler)).Queries(restQueries(peerRESTMetricsTypes)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodResourceMetrics).HandlerFunc(h(server.GetResourceMetrics))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysErrors).HandlerFunc(h(server.GetSysErrorsHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysServices).HandlerFunc(h(server.GetSysServicesHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysConfig).HandlerFunc(h(server.GetSysConfigHandler))

cmd/server-main.go

@@ -928,6 +928,10 @@ func serverMain(ctx *cli.Context) {
 		logger.FatalIf(err, "Unable to initialize MinIO client")
 	})
 
+	go bootstrapTrace("startResourceMetricsCollection", func() {
+		startResourceMetricsCollection()
+	})
+
 	// Add User-Agent to differentiate the requests.
 	globalMinioClient.SetAppInfo("minio-perf-test", ReleaseTag)
 

cmd/server-startup-msg.go

@@ -21,12 +21,11 @@ import (
 	"fmt"
 	"net"
 	"net/url"
-	"runtime"
 	"strings"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v3"
-	color "github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/logger"
 	xnet "github.com/minio/pkg/v2/net"
 )
@@ -129,10 +128,10 @@ func printServerCommonMsg(apiEndpoints []string) {
 	// Colorize the message and print.
 	logger.Info(color.Blue("S3-API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
 	if color.IsTerminal() && (!globalCLIContext.Anonymous && !globalCLIContext.JSON) {
-		logger.Info(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-		logger.Info(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s \n", cred.SecretKey)))
+		logger.Info(color.Blue("RootUser: ") + color.Bold("%s ", cred.AccessKey))
+		logger.Info(color.Blue("RootPass: ") + color.Bold("%s \n", cred.SecretKey))
 		if region != "" {
-			logger.Info(color.Blue("Region: ") + color.Bold(fmt.Sprintf(getFormatStr(len(region), 2), region)))
+			logger.Info(color.Blue("Region: ") + color.Bold("%s", fmt.Sprintf(getFormatStr(len(region), 2), region)))
 		}
 	}
 
@@ -140,8 +139,8 @@ func printServerCommonMsg(apiEndpoints []string) {
 		consoleEndpointStr := strings.Join(stripStandardPorts(getConsoleEndpoints(), globalMinioConsoleHost), " ")
 		logger.Info(color.Blue("Console: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
 		if color.IsTerminal() && (!globalCLIContext.Anonymous && !globalCLIContext.JSON) {
-			logger.Info(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-			logger.Info(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
+			logger.Info(color.Blue("RootUser: ") + color.Bold("%s ", cred.AccessKey))
+			logger.Info(color.Blue("RootPass: ") + color.Bold("%s ", cred.SecretKey))
 		}
 	}
 
@@ -196,15 +195,9 @@ func printCLIAccessMsg(endPoint string, alias string) {
 	// Configure 'mc', following block prints platform specific information for minio client.
 	if color.IsTerminal() && !globalCLIContext.Anonymous {
 		logger.Info(color.Blue("\nCommand-line: ") + mcQuickStartGuide)
-		if runtime.GOOS == globalWindowsOSName {
-			mcMessage := fmt.Sprintf("$ mc.exe alias set %s %s %s %s", alias,
-				endPoint, cred.AccessKey, cred.SecretKey)
-			logger.Info(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
-		} else {
-			mcMessage := fmt.Sprintf("$ mc alias set %s %s %s %s", alias,
-				endPoint, cred.AccessKey, cred.SecretKey)
-			logger.Info(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
-		}
+		mcMessage := fmt.Sprintf("$ mc alias set '%s' '%s' '%s' '%s'", alias,
+			endPoint, cred.AccessKey, cred.SecretKey)
+		logger.Info(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
 	}
 }
 

cmd/service.go

@@ -104,11 +104,11 @@ func unfreezeServices() {
 	// Close when we reach 0
 	globalServiceFreezeCnt--
 	if globalServiceFreezeCnt <= 0 {
-		// Ensure we only close once.
-		if val := globalServiceFreeze.Load(); val != nil {
-			var _ch chan struct{}
-			if ch, ok := val.(chan struct{}); ok {
-				globalServiceFreeze.Store(_ch)
+		// Set to a nil channel.
+		var _ch chan struct{}
+		if val := globalServiceFreeze.Swap(_ch); val != nil {
+			if ch, ok := val.(chan struct{}); ok && ch != nil {
+				// Close previous non-nil channel.
 				close(ch)
 			}
 		}

cmd/xl-storage.go

@@ -290,18 +290,16 @@ func newXLStorage(ep Endpoint, cleanUp bool) (s *xlStorage, err error) {
 		s.formatLegacy = format.Erasure.DistributionAlgo == formatErasureVersionV2DistributionAlgoV1
 	}
 
-	if globalAPIConfig.odirectEnabled() {
-		// Return an error if ODirect is not supported
-		// unless it is a single erasure disk mode
-		if err := s.checkODirectDiskSupport(); err == nil {
-			s.oDirect = true
+	// Return an error if ODirect is not supported unless it is a single erasure
+	// disk mode
+	if err := s.checkODirectDiskSupport(); err == nil {
+		s.oDirect = true
+	} else {
+		// Allow if unsupported platform or single disk.
+		if errors.Is(err, errUnsupportedDisk) && globalIsErasureSD || !disk.ODirectPlatform {
+			s.oDirect = false
 		} else {
-			// Allow if unsupported platform or single disk.
-			if errors.Is(err, errUnsupportedDisk) && globalIsErasureSD || !disk.ODirectPlatform {
-				s.oDirect = false
-			} else {
-				return s, err
-			}
+			return s, err
 		}
 	}
 

docs/metrics/prometheus/list.md

@@ -303,3 +303,109 @@ For deployments with [Site Replication](https://min.io/docs/minio/linux/operatio
 | `minio_bucket_requests_total`                     | Total number of S3 requests on a bucket.                                        |
 | `minio_bucket_requests_canceled_total`            | Total number S3 requests canceled by the client.                                |
 | `minio_bucket_requests_ttfb_seconds_distribution` | Distribution of time to first byte across API calls per bucket.                 |
+
+# Resource Metrics
+
+MinIO collects the following resource metrics at the node level.
+Each metric includes the `server` label to identify the corresponding node.
+Metrics may include one or more additional labels, such as the drive path, interface name, etc.
+
+These metrics can be obtained from any MinIO server once per collection by using the following URL:
+
+```shell
+https://HOSTNAME:PORT/minio/metrics/v2/resource
+```
+
+Replace `HOSTNAME:PORT` with the hostname of your MinIO deployment.
+For deployments behind a load balancer, use the load balancer hostname instead of a single node hostname.
+
+## Drive Resource Metrics
+
+| Name                                 | Description                                              |
+| :----------------------------------- | :------------------------------------------------------- |
+| `minio_node_drive_total_bytes`       | Total bytes on a drive.                                  |
+| `minio_node_drive_used_bytes`        | Used bytes on a drive.                                   |
+| `minio_node_drive_total_inodes`      | Total inodes on a drive.                                 |
+| `minio_node_drive_used_inodes`       | Total inodes used on a drive.                            |
+| `minio_node_drive_reads_per_sec`     | Reads per second on a drive.                             |
+| `minio_node_drive_reads_kb_per_sec`  | Kilobytes read per second on a drive.                    |
+| `minio_node_drive_reads_await`       | Average time for read requests to be served on a drive.  |
+| `minio_node_drive_writes_per_sec`    | Writes per second on a drive.                            |
+| `minio_node_drive_writes_kb_per_sec` | Kilobytes written per second on a drive.                 |
+| `minio_node_drive_writes_await`      | Average time for write requests to be served on a drive. |
+| `minio_node_drive_perc_util`         | Percentage of time the disk was busy since uptime.       |
+
+## Network Interface Metrics
+
+| Name                          | Description                                   |
+| :---------------------------- | :-------------------------------------------- |
+| `minio_node_if_rx_bytes`      | Bytes received on the interface in 60s.       |
+| `minio_node_if_rx_bytes_avg`  | Bytes received on the interface in 60s (avg). |
+| `minio_node_if_rx_bytes_max`  | Bytes received on the interface in 60s (max). |
+| `minio_node_if_rx_errors`     | Receive errors in 60s.                        |
+| `minio_node_if_rx_errors_avg` | Receive errors in 60s (avg).                  |
+| `minio_node_if_rx_errors_max` | Receive errors in 60s (max).                  |
+| `minio_node_if_tx_bytes`      | Bytes transmitted in 60s.                     |
+| `minio_node_if_tx_bytes_avg`  | Bytes transmitted in 60s (avg).               |
+| `minio_node_if_tx_bytes_max`  | Bytes transmitted in 60s (max).               |
+| `minio_node_if_tx_errors`     | Transmit errors in 60s.                       |
+| `minio_node_if_tx_errors_avg` | Transmit errors in 60s (avg).                 |
+| `minio_node_if_tx_errors_max` | Transmit errors in 60s (max).                 |
+
+## CPU Metrics
+
+| Name                            | Description                   |
+| :------------------------------ | :---------------------------- |
+| `minio_node_cpu_avg_user`       | CPU user time.                |
+| `minio_node_cpu_avg_user_avg`   | CPU user time (avg).          |
+| `minio_node_cpu_avg_user_max`   | CPU user time (max).          |
+| `minio_node_cpu_avg_system`     | CPU system time.              |
+| `minio_node_cpu_avg_system_avg` | CPU system time (avg).        |
+| `minio_node_cpu_avg_system_max` | CPU system time (max).        |
+| `minio_node_cpu_avg_idle`       | CPU idle time.                |
+| `minio_node_cpu_avg_idle_avg`   | CPU idle time (avg).          |
+| `minio_node_cpu_avg_idle_max`   | CPU idle time (max).          |
+| `minio_node_cpu_avg_iowait`     | CPU ioWait time.              |
+| `minio_node_cpu_avg_iowait_avg` | CPU ioWait time (avg).        |
+| `minio_node_cpu_avg_iowait_max` | CPU ioWait time (max).        |
+| `minio_node_cpu_avg_nice`       | CPU nice time.                |
+| `minio_node_cpu_avg_nice_avg`   | CPU nice time (avg).          |
+| `minio_node_cpu_avg_nice_max`   | CPU nice time (max).          |
+| `minio_node_cpu_avg_steal`      | CPU steam time.               |
+| `minio_node_cpu_avg_steal_avg`  | CPU steam time (avg).         |
+| `minio_node_cpu_avg_steal_max`  | CPU steam time (max).         |
+| `minio_node_cpu_avg_load1`      | CPU load average 1min.        |
+| `minio_node_cpu_avg_load1_avg`  | CPU load average 1min (avg).  |
+| `minio_node_cpu_avg_load1_max`  | CPU load average 1min (max).  |
+| `minio_node_cpu_avg_load5`      | CPU load average 5min.        |
+| `minio_node_cpu_avg_load5_avg`  | CPU load average 5min (avg).  |
+| `minio_node_cpu_avg_load5_max`  | CPU load average 5min (max).  |
+| `minio_node_cpu_avg_load15`     | CPU load average 15min.       |
+| `minio_node_cpu_avg_load15_avg` | CPU load average 15min (avg). |
+| `minio_node_cpu_avg_load15_max` | CPU load average 15min (max). |
+
+## Memory Metrics
+
+| Name                           | Description                         |
+| :----------------------------- | :---------------------------------- |
+| `minio_node_mem_available`     | Available memory on the node.       |
+| `minio_node_mem_available_avg` | Available memory on the node (avg). |
+| `minio_node_mem_available_max` | Available memory on the node (max). |
+| `minio_node_mem_buffers`       | Buffers memory on the node.         |
+| `minio_node_mem_buffers_avg`   | Buffers memory on the node (avg).   |
+| `minio_node_mem_buffers_max`   | Buffers memory on the node (max).   |
+| `minio_node_mem_cache`         | Cache memory on the node.           |
+| `minio_node_mem_cache_avg`     | Cache memory on the node (avg).     |
+| `minio_node_mem_cache_max`     | Cache memory on the node (max).     |
+| `minio_node_mem_free`          | Free memory on the node.            |
+| `minio_node_mem_free_avg`      | Free memory on the node (avg).      |
+| `minio_node_mem_free_max`      | Free memory on the node (max).      |
+| `minio_node_mem_shared`        | Shared memory on the node.          |
+| `minio_node_mem_shared_avg`    | Shared memory on the node (avg).    |
+| `minio_node_mem_shared_max`    | Shared memory on the node (max).    |
+| `minio_node_mem_total`         | Total memory on the node.           |
+| `minio_node_mem_total_avg`     | Total memory on the node (avg).     |
+| `minio_node_mem_total_max`     | Total memory on the node (max).     |
+| `minio_node_mem_used`          | Used memory on the node.            |
+| `minio_node_mem_used_avg`      | Used memory on the node (avg).      |
+| `minio_node_mem_used_max`      | Used memory on the node (max).      |

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2023-09-27T15-22-50Z
+  image: quay.io/minio/minio:RELEASE.2023-09-30T07-02-29Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

go.mod

@@ -43,14 +43,14 @@ require (
 	github.com/lithammer/shortuuid/v4 v4.0.0
 	github.com/miekg/dns v1.1.56
 	github.com/minio/cli v1.24.2
-	github.com/minio/console v0.39.0
+	github.com/minio/console v0.40.0
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/dnscache v0.1.1
 	github.com/minio/dperf v0.5.0
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes-go v0.2.0
 	github.com/minio/madmin-go/v3 v3.0.21
-	github.com/minio/minio-go/v7 v7.0.63
+	github.com/minio/minio-go/v7 v7.0.64-0.20230920204636-e783c9ba11b3
 	github.com/minio/mux v1.9.0
 	github.com/minio/pkg/v2 v2.0.2
 	github.com/minio/selfupdate v0.6.0

go.sum

@@ -460,8 +460,8 @@ github.com/minio/cli v1.24.2 h1:J+fCUh9mhPLjN3Lj/YhklXvxj8mnyE/D6FpFduXJ2jg=
 github.com/minio/cli v1.24.2/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
 github.com/minio/colorjson v1.0.6 h1:m7TUvpvt0u7FBmVIEQNIa0T4NBQlxrcMBp4wJKsg2Ik=
 github.com/minio/colorjson v1.0.6/go.mod h1:LUXwS5ZGNb6Eh9f+t+3uJiowD3XsIWtsvTriUBeqgYs=
-github.com/minio/console v0.39.0 h1:Ta3C5Magg9bLp1C2LwFX98QZWV282Z+X0ICJ+xH7J4k=
-github.com/minio/console v0.39.0/go.mod h1:lg8HbAvha71oKDxWI5la0v22kX8zNpitvw80etkhOpk=
+github.com/minio/console v0.40.0 h1:zldPHumWA5SN6uAUyAsctUFQ80LvLDv3n0VlONmBJMc=
+github.com/minio/console v0.40.0/go.mod h1:ra5pDF0mFQ9DI8K5iyk6Q5gV6KQuZq3MKPGYU66a+vc=
 github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU=
 github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM=
 github.com/minio/dnscache v0.1.1 h1:AMYLqomzskpORiUA1ciN9k7bZT1oB3YZN4cEIi88W5o=
@@ -481,8 +481,8 @@ github.com/minio/mc v0.0.0-20230922050746-ae05d451739b/go.mod h1:+4OzbgOsVsKvdmL
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
-github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ=
-github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4=
+github.com/minio/minio-go/v7 v7.0.64-0.20230920204636-e783c9ba11b3 h1:0DtfDxg67S/IRcGnIBKgzyjZ0GXyk3jN1Fy5/+8CQlM=
+github.com/minio/minio-go/v7 v7.0.64-0.20230920204636-e783c9ba11b3/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4=
 github.com/minio/mux v1.9.0 h1:dWafQFyEfGhJvK6AwLOt83bIG5bxKxKJnKMCi0XAaoA=
 github.com/minio/mux v1.9.0/go.mod h1:1pAare17ZRL5GpmNL+9YmqHoWnLmMZF9C/ioUCfy0BQ=
 github.com/minio/pkg v1.7.5 h1:UOUJjewE5zoaDPlCMJtNx/swc1jT1ZR+IajT7hrLd44=

helm/minio/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: Multi-Cloud Object Storage
 name: minio
-version: 5.0.13
-appVersion: RELEASE.2023-07-07T07-13-57Z
+version: 5.0.14
+appVersion: RELEASE.2023-09-30T07-02-29Z
 keywords:
   - minio
   - storage

helm/minio/templates/deployment.yaml

@@ -120,7 +120,14 @@ spec:
             - name: MINIO_IDENTITY_OPENID_CLIENT_ID
               value: {{ .Values.oidc.clientId }}
             - name: MINIO_IDENTITY_OPENID_CLIENT_SECRET
+            {{- if and .Values.oidc.existingClientSecretName .Values.oidc.existingClientSecretKey }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.oidc.existingClientSecretName }}
+                  key: {{ .Values.oidc.existingClientSecretKey }}
+            {{- else }}
               value: {{ .Values.oidc.clientSecret }}
+            {{- end }}
             - name: MINIO_IDENTITY_OPENID_CLAIM_NAME
               value: {{ .Values.oidc.claimName }}
             - name: MINIO_IDENTITY_OPENID_CLAIM_PREFIX

helm/minio/templates/statefulset.yaml

@@ -158,7 +158,14 @@ spec:
             - name: MINIO_IDENTITY_OPENID_CLIENT_ID
               value: {{ .Values.oidc.clientId }}
             - name: MINIO_IDENTITY_OPENID_CLIENT_SECRET
+            {{- if and .Values.oidc.existingClientSecretName .Values.oidc.existingClientSecretKey }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.oidc.existingClientSecretName }}
+                  key: {{ .Values.oidc.existingClientSecretKey }}
+            {{- else }}
               value: {{ .Values.oidc.clientSecret }}
+            {{- end }}
             - name: MINIO_IDENTITY_OPENID_CLAIM_NAME
               value: {{ .Values.oidc.claimName }}
             - name: MINIO_IDENTITY_OPENID_CLAIM_PREFIX

helm/minio/values.yaml

@@ -14,7 +14,7 @@ clusterDomain: cluster.local
 ##
 image:
   repository: quay.io/minio/minio
-  tag: RELEASE.2023-07-07T07-13-57Z
+  tag: RELEASE.2023-09-30T07-02-29Z
   pullPolicy: IfNotPresent
 
 imagePullSecrets: []
@@ -25,7 +25,7 @@ imagePullSecrets: []
 ##
 mcImage:
   repository: quay.io/minio/mc
-  tag: RELEASE.2023-06-28T21-54-17Z
+  tag: RELEASE.2023-09-29T16-41-22Z
   pullPolicy: IfNotPresent
 
 ## minio mode, i.e. standalone or distributed
@@ -486,6 +486,9 @@ oidc:
   configUrl: "https://identity-provider-url/.well-known/openid-configuration"
   clientId: "minio"
   clientSecret: ""
+  # Provide existing client secret from the Kubernetes Secret resource, existing secret will have priority over `clientSecret`
+  existingClientSecretName: ""
+  existingClientSecretKey: ""
   claimName: "policy"
   scopes: "openid,profile,email"
   redirectUri: "https://console-endpoint-url/oauth_callback"

index.yaml

@@ -1,9 +1,31 @@
 apiVersion: v1
 entries:
   minio:
+  - apiVersion: v1
+    appVersion: RELEASE.2023-09-30T07-02-29Z
+    created: "2023-09-30T13:45:44.318107697-07:00"
+    description: Multi-Cloud Object Storage
+    digest: 6c3656924fbad2cb17f810cd78f352f9b60626aaec64b837c96829095b215ad3
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-5.0.14.tgz
+    version: 5.0.14
   - apiVersion: v1
     appVersion: RELEASE.2023-07-07T07-13-57Z
-    created: "2023-07-09T00:12:52.083613719-07:00"
+    created: "2023-09-30T13:45:44.317015357-07:00"
     description: Multi-Cloud Object Storage
     digest: 3c18f7381efe6d86497f952e6d5f59003ee5a009c54778ddea1ee8d3c7bed9c8
     home: https://min.io
@@ -25,7 +47,7 @@ entries:
     version: 5.0.13
   - apiVersion: v1
     appVersion: RELEASE.2023-07-07T07-13-57Z
-    created: "2023-07-09T00:12:52.082626304-07:00"
+    created: "2023-09-30T13:45:44.315355964-07:00"
     description: Multi-Cloud Object Storage
     digest: 5318bc56c73a8f4539c3dd178f4d55c7f41bee4a25d7dc02ac6a5843eeee7976
     home: https://min.io
@@ -47,7 +69,7 @@ entries:
     version: 5.0.12
   - apiVersion: v1
     appVersion: RELEASE.2023-06-19T19-52-50Z
-    created: "2023-07-09T00:12:52.081596401-07:00"
+    created: "2023-09-30T13:45:44.314320466-07:00"
     description: Multi-Cloud Object Storage
     digest: cba44c8cddcda1fb5c082dce82004a39f53cc20677ab9698a6998f01efefd8db
     home: https://min.io
@@ -69,7 +91,7 @@ entries:
     version: 5.0.11
   - apiVersion: v1
     appVersion: RELEASE.2023-05-18T00-05-36Z
-    created: "2023-07-09T00:12:52.07978903-07:00"
+    created: "2023-09-30T13:45:44.313340376-07:00"
     description: Multi-Cloud Object Storage
     digest: a3d55b12f38a2049ddf3efe35b38b6dc4e59777452b72d18d5a82f3378deb9cd
     home: https://min.io
@@ -91,7 +113,7 @@ entries:
     version: 5.0.10
   - apiVersion: v1
     appVersion: RELEASE.2023-04-28T18-11-17Z
-    created: "2023-07-09T00:12:52.092296696-07:00"
+    created: "2023-09-30T13:45:44.327034407-07:00"
     description: Multi-Cloud Object Storage
     digest: cf98985e32675e4ce327304ea9ac61046a788b3d5190d6b501330f7803d41a11
     home: https://min.io
@@ -113,7 +135,7 @@ entries:
     version: 5.0.9
   - apiVersion: v1
     appVersion: RELEASE.2023-04-13T03-08-07Z
-    created: "2023-07-09T00:12:52.091349939-07:00"
+    created: "2023-09-30T13:45:44.326090811-07:00"
     description: Multi-Cloud Object Storage
     digest: 034d68f85799f6693836975797f85a91842cf2d003a6c4ff401bd4ea4c946af6
     home: https://min.io
@@ -135,7 +157,7 @@ entries:
     version: 5.0.8
   - apiVersion: v1
     appVersion: RELEASE.2023-02-10T18-48-39Z
-    created: "2023-07-09T00:12:52.090389234-07:00"
+    created: "2023-09-30T13:45:44.32510969-07:00"
     description: Multi-Cloud Object Storage
     digest: 3f935a310e1b5b873052629b66005c160356ca7b2bd394cb07b34dbaf9905e3f
     home: https://min.io
@@ -157,7 +179,7 @@ entries:
     version: 5.0.7
   - apiVersion: v1
     appVersion: RELEASE.2023-02-10T18-48-39Z
-    created: "2023-07-09T00:12:52.089429881-07:00"
+    created: "2023-09-30T13:45:44.32366994-07:00"
     description: Multi-Cloud Object Storage
     digest: 82ef858ce483c2d736444792986cb36bd0fb4fc90a80b97fe30d7b2f2034d24a
     home: https://min.io
@@ -179,7 +201,7 @@ entries:
     version: 5.0.6
   - apiVersion: v1
     appVersion: RELEASE.2023-01-31T02-24-19Z
-    created: "2023-07-09T00:12:52.087989387-07:00"
+    created: "2023-09-30T13:45:44.322210657-07:00"
     description: Multi-Cloud Object Storage
     digest: fefeea10e4e525e45f82fb80a03900d34605ec432dd92f56d94eaf4fb1b98c41
     home: https://min.io
@@ -201,7 +223,7 @@ entries:
     version: 5.0.5
   - apiVersion: v1
     appVersion: RELEASE.2022-12-12T19-27-27Z
-    created: "2023-07-09T00:12:52.0865042-07:00"
+    created: "2023-09-30T13:45:44.321254746-07:00"
     description: Multi-Cloud Object Storage
     digest: 6b305783c98b0b97ffab079ff4430094fd0ca6e98e82bb8153cb93033a1bf40f
     home: https://min.io
@@ -223,7 +245,7 @@ entries:
     version: 5.0.4
   - apiVersion: v1
     appVersion: RELEASE.2022-12-12T19-27-27Z
-    created: "2023-07-09T00:12:52.08555416-07:00"
+    created: "2023-09-30T13:45:44.320169821-07:00"
     description: Multi-Cloud Object Storage
     digest: bac89157c53b324aece263c294aa49f5c9b64f426b4b06c9bca3d72e77e244f2
     home: https://min.io
@@ -245,7 +267,7 @@ entries:
     version: 5.0.3
   - apiVersion: v1
     appVersion: RELEASE.2022-12-12T19-27-27Z
-    created: "2023-07-09T00:12:52.084560152-07:00"
+    created: "2023-09-30T13:45:44.319086339-07:00"
     description: Multi-Cloud Object Storage
     digest: 935ce4f09366231b11d414d626f887fa6fa6024dd30a42e81e810ca1438d5904
     home: https://min.io
@@ -267,7 +289,7 @@ entries:
     version: 5.0.2
   - apiVersion: v1
     appVersion: RELEASE.2022-11-11T03-44-20Z
-    created: "2023-07-09T00:12:52.078824549-07:00"
+    created: "2023-09-30T13:45:44.312066393-07:00"
     description: Multi-Cloud Object Storage
     digest: 3e952c5d737980b8ccdfb819021eafb4b4e8da226f764a1dc3de1ba63ceb1ffa
     home: https://min.io
@@ -289,7 +311,7 @@ entries:
     version: 5.0.1
   - apiVersion: v1
     appVersion: RELEASE.2022-10-24T18-35-07Z
-    created: "2023-07-09T00:12:52.077895753-07:00"
+    created: "2023-09-30T13:45:44.311113045-07:00"
     description: Multi-Cloud Object Storage
     digest: 6215c800d84fd4c40e4fb4142645fc1c6a039c251776a3cc8c11a24b9e3b59c7
     home: https://min.io
@@ -311,7 +333,7 @@ entries:
     version: 5.0.0
   - apiVersion: v1
     appVersion: RELEASE.2022-10-24T18-35-07Z
-    created: "2023-07-09T00:12:52.076911932-07:00"
+    created: "2023-09-30T13:45:44.310137758-07:00"
     description: Multi-Cloud Object Storage
     digest: 2d3d884490ea1127742f938bc9382844bae713caae08b3308f766f3c9000659a
     home: https://min.io
@@ -333,7 +355,7 @@ entries:
     version: 4.1.0
   - apiVersion: v1
     appVersion: RELEASE.2022-09-17T00-09-45Z
-    created: "2023-07-09T00:12:52.068200433-07:00"
+    created: "2023-09-30T13:45:44.299926747-07:00"
     description: Multi-Cloud Object Storage
     digest: 6f16f2dbfed91ab81a7fae60b6ea32f554365bd27bf5fda55b64a0fa264f4252
     home: https://min.io
@@ -355,7 +377,7 @@ entries:
     version: 4.0.15
   - apiVersion: v1
     appVersion: RELEASE.2022-09-01T23-53-36Z
-    created: "2023-07-09T00:12:52.06625577-07:00"
+    created: "2023-09-30T13:45:44.298411641-07:00"
     description: Multi-Cloud Object Storage
     digest: 35d89d8f49d53ea929466fb88ee26123431326033f1387e6b2d536a629c0a398
     home: https://min.io
@@ -377,7 +399,7 @@ entries:
     version: 4.0.14
   - apiVersion: v1
     appVersion: RELEASE.2022-08-22T23-53-06Z
-    created: "2023-07-09T00:12:52.065251383-07:00"
+    created: "2023-09-30T13:45:44.29745713-07:00"
     description: Multi-Cloud Object Storage
     digest: 5b86937ca88d9f6046141fdc2b1cc54760435ed92d289cd0a115fa7148781d4e
     home: https://min.io
@@ -399,7 +421,7 @@ entries:
     version: 4.0.13
   - apiVersion: v1
     appVersion: RELEASE.2022-08-13T21-54-44Z
-    created: "2023-07-09T00:12:52.064278973-07:00"
+    created: "2023-09-30T13:45:44.296233683-07:00"
     description: Multi-Cloud Object Storage
     digest: 2d9c227c0f46ea8bdef4d760c212156fd4c6623ddc5406779c569fe925527787
     home: https://min.io
@@ -421,7 +443,7 @@ entries:
     version: 4.0.12
   - apiVersion: v1
     appVersion: RELEASE.2022-08-05T23-27-09Z
-    created: "2023-07-09T00:12:52.063368673-07:00"
+    created: "2023-09-30T13:45:44.295016135-07:00"
     description: Multi-Cloud Object Storage
     digest: 6caaffcb636e040cd7e8bc4883a1674a673757f4781c32d53b5ec0f41fea3944
     home: https://min.io
@@ -443,7 +465,7 @@ entries:
     version: 4.0.11
   - apiVersion: v1
     appVersion: RELEASE.2022-08-02T23-59-16Z
-    created: "2023-07-09T00:12:52.062416509-07:00"
+    created: "2023-09-30T13:45:44.29308946-07:00"
     description: Multi-Cloud Object Storage
     digest: 841d87788fb094d6a7d8a91e91821fe1e847bc952e054c781fc93742d112e18a
     home: https://min.io
@@ -465,7 +487,7 @@ entries:
     version: 4.0.10
   - apiVersion: v1
     appVersion: RELEASE.2022-08-02T23-59-16Z
-    created: "2023-07-09T00:12:52.075955122-07:00"
+    created: "2023-09-30T13:45:44.308504002-07:00"
     description: Multi-Cloud Object Storage
     digest: 6f1a78382df3215deac07495a5e7de7009a1153b4cf6cb565630652a69aec4cf
     home: https://min.io
@@ -487,7 +509,7 @@ entries:
     version: 4.0.9
   - apiVersion: v1
     appVersion: RELEASE.2022-07-29T19-40-48Z
-    created: "2023-07-09T00:12:52.075062029-07:00"
+    created: "2023-09-30T13:45:44.307480879-07:00"
     description: Multi-Cloud Object Storage
     digest: d11db37963636922cb778b6bc0ad2ca4724cb391ea7b785995ada52467d7dd83
     home: https://min.io
@@ -509,7 +531,7 @@ entries:
     version: 4.0.8
   - apiVersion: v1
     appVersion: RELEASE.2022-07-26T00-53-03Z
-    created: "2023-07-09T00:12:52.073779859-07:00"
+    created: "2023-09-30T13:45:44.306527164-07:00"
     description: Multi-Cloud Object Storage
     digest: ca775e08c84331bb5029d4d29867d30c16e2c62e897788eb432212a756e91e4e
     home: https://min.io
@@ -531,7 +553,7 @@ entries:
     version: 4.0.7
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-07-09T00:12:52.07269585-07:00"
+    created: "2023-09-30T13:45:44.305405238-07:00"
     description: Multi-Cloud Object Storage
     digest: 06542b8f3d149d5908b15de9a8d6f8cf304af0213830be56dc315785d14f9ccd
     home: https://min.io
@@ -553,7 +575,7 @@ entries:
     version: 4.0.6
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-07-09T00:12:52.071826014-07:00"
+    created: "2023-09-30T13:45:44.304317131-07:00"
     description: Multi-Cloud Object Storage
     digest: dd2676362f067454a496cdd293609d0c904b08f521625af49f95402a024ba1f5
     home: https://min.io
@@ -575,7 +597,7 @@ entries:
     version: 4.0.5
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-07-09T00:12:52.070968807-07:00"
+    created: "2023-09-30T13:45:44.303451898-07:00"
     description: Multi-Cloud Object Storage
     digest: bab9ef192d4eda4c572ad0ce0cf551736c847f582d1837d6833ee10543c23167
     home: https://min.io
@@ -597,7 +619,7 @@ entries:
     version: 4.0.4
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-07-09T00:12:52.070118552-07:00"
+    created: "2023-09-30T13:45:44.302546931-07:00"
     description: Multi-Cloud Object Storage
     digest: c770bb9841c76576e4e8573f78b0ec33e0d729504c9667e67ad62d48df5ed64c
     home: https://min.io
@@ -619,7 +641,7 @@ entries:
     version: 4.0.3
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-07-09T00:12:52.069249607-07:00"
+    created: "2023-09-30T13:45:44.301269526-07:00"
     description: Multi-Cloud Object Storage
     digest: 95835f4199d963e2a23a2493610b348e6f2ff8b71c1a648c4a3b84af9b7a83eb
     home: https://min.io
@@ -641,7 +663,7 @@ entries:
     version: 4.0.2
   - apiVersion: v1
     appVersion: RELEASE.2022-04-30T22-23-53Z
-    created: "2023-07-09T00:12:52.061513072-07:00"
+    created: "2023-09-30T13:45:44.29213066-07:00"
     description: Multi-Cloud Object Storage
     digest: 55a088c403b056e1f055a97426aa11759c3d6cbad38face170fe6cbbec7d568f
     home: https://min.io
@@ -663,7 +685,7 @@ entries:
     version: 4.0.1
   - apiVersion: v1
     appVersion: RELEASE.2022-04-26T01-20-24Z
-    created: "2023-07-09T00:12:52.060505181-07:00"
+    created: "2023-09-30T13:45:44.291284045-07:00"
     description: Multi-Cloud Object Storage
     digest: f541237e24336ec3f7f45ae0d523fef694e3a2f9ef648c5b11c15734db6ba2b2
     home: https://min.io
@@ -685,7 +707,7 @@ entries:
     version: 4.0.0
   - apiVersion: v1
     appVersion: RELEASE.2022-04-16T04-26-02Z
-    created: "2023-07-09T00:12:52.058907841-07:00"
+    created: "2023-09-30T13:45:44.290426627-07:00"
     description: Multi-Cloud Object Storage
     digest: edc0c3dd6d5246a06b74ba16bb4aff80a6d7225dc9aecf064fd89a8af371b9c1
     home: https://min.io
@@ -707,7 +729,7 @@ entries:
     version: 3.6.6
   - apiVersion: v1
     appVersion: RELEASE.2022-04-12T06-55-35Z
-    created: "2023-07-09T00:12:52.058000086-07:00"
+    created: "2023-09-30T13:45:44.289556731-07:00"
     description: Multi-Cloud Object Storage
     digest: 211e89f6b9eb0b9a3583abaa127be60e1f9717a098e6b2858cb9dc1cc50c1650
     home: https://min.io
@@ -729,7 +751,7 @@ entries:
     version: 3.6.5
   - apiVersion: v1
     appVersion: RELEASE.2022-04-09T15-09-52Z
-    created: "2023-07-09T00:12:52.056972348-07:00"
+    created: "2023-09-30T13:45:44.28851016-07:00"
     description: Multi-Cloud Object Storage
     digest: 534a879d73b370a18b554b93d0930e1c115419619c4ce4ec7dbaae632acacf06
     home: https://min.io
@@ -751,7 +773,7 @@ entries:
     version: 3.6.4
   - apiVersion: v1
     appVersion: RELEASE.2022-03-24T00-43-44Z
-    created: "2023-07-09T00:12:52.056123789-07:00"
+    created: "2023-09-30T13:45:44.287632827-07:00"
     description: Multi-Cloud Object Storage
     digest: 99508b20eb0083a567dcccaf9a6c237e09575ed1d70cd2e8333f89c472d13d75
     home: https://min.io
@@ -773,7 +795,7 @@ entries:
     version: 3.6.3
   - apiVersion: v1
     appVersion: RELEASE.2022-03-17T06-34-49Z
-    created: "2023-07-09T00:12:52.055271276-07:00"
+    created: "2023-09-30T13:45:44.28672558-07:00"
     description: Multi-Cloud Object Storage
     digest: b4cd25611ca322b1d23d23112fdfa6b068fd91eefe0b0663b88ff87ea4282495
     home: https://min.io
@@ -795,7 +817,7 @@ entries:
     version: 3.6.2
   - apiVersion: v1
     appVersion: RELEASE.2022-03-14T18-25-24Z
-    created: "2023-07-09T00:12:52.054400832-07:00"
+    created: "2023-09-30T13:45:44.285161842-07:00"
     description: Multi-Cloud Object Storage
     digest: d75b88162bfe54740a233bcecf87328bba2ae23d170bec3a35c828bc6fdc224c
     home: https://min.io
@@ -817,7 +839,7 @@ entries:
     version: 3.6.1
   - apiVersion: v1
     appVersion: RELEASE.2022-03-11T23-57-45Z
-    created: "2023-07-09T00:12:52.053508093-07:00"
+    created: "2023-09-30T13:45:44.284217509-07:00"
     description: Multi-Cloud Object Storage
     digest: 22e53a1184a21a679bc7d8b94e955777f3506340fc29da5ab0cb6d729bdbde8d
     home: https://min.io
@@ -839,7 +861,7 @@ entries:
     version: 3.6.0
   - apiVersion: v1
     appVersion: RELEASE.2022-03-03T21-21-16Z
-    created: "2023-07-09T00:12:52.051823002-07:00"
+    created: "2023-09-30T13:45:44.28321379-07:00"
     description: Multi-Cloud Object Storage
     digest: 6fda968d3fdfd60470c0055a4e1a3bd8e5aee9ad0af5ba2fb7b7b926fdc9e4a0
     home: https://min.io
@@ -861,7 +883,7 @@ entries:
     version: 3.5.9
   - apiVersion: v1
     appVersion: RELEASE.2022-02-26T02-54-46Z
-    created: "2023-07-09T00:12:52.050984544-07:00"
+    created: "2023-09-30T13:45:44.282331055-07:00"
     description: Multi-Cloud Object Storage
     digest: 8e015369048a3a82bbd53ad36696786f18561c6b25d14eee9e2c93a7336cef46
     home: https://min.io
@@ -883,7 +905,7 @@ entries:
     version: 3.5.8
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2023-07-09T00:12:52.050137914-07:00"
+    created: "2023-09-30T13:45:44.281421196-07:00"
     description: Multi-Cloud Object Storage
     digest: cb3543fe748e5f0d59b3ccf4ab9af8e10b731405ae445d1f5715e30013632373
     home: https://min.io
@@ -905,7 +927,7 @@ entries:
     version: 3.5.7
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2023-07-09T00:12:52.049298332-07:00"
+    created: "2023-09-30T13:45:44.280388028-07:00"
     description: Multi-Cloud Object Storage
     digest: f2e359fa5eefffc59abb3d14a8fa94b11ddeaa99f6cd8dd5f40f4e04121000d6
     home: https://min.io
@@ -927,7 +949,7 @@ entries:
     version: 3.5.6
   - apiVersion: v1
     appVersion: RELEASE.2022-02-16T00-35-27Z
-    created: "2023-07-09T00:12:52.048393471-07:00"
+    created: "2023-09-30T13:45:44.279117054-07:00"
     description: Multi-Cloud Object Storage
     digest: 529d56cca9d83a3d0e5672e63b6e87b5bcbe10a6b45f7a55ba998cceb32f9c81
     home: https://min.io
@@ -949,7 +971,7 @@ entries:
     version: 3.5.5
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2023-07-09T00:12:52.047593644-07:00"
+    created: "2023-09-30T13:45:44.277558085-07:00"
     description: Multi-Cloud Object Storage
     digest: 3d530598f8ece67bec5b7f990d206584893987c713502f9228e4ee24b5535414
     home: https://min.io
@@ -971,7 +993,7 @@ entries:
     version: 3.5.4
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2023-07-09T00:12:52.046792295-07:00"
+    created: "2023-09-30T13:45:44.276727539-07:00"
     description: Multi-Cloud Object Storage
     digest: 53937031348b29615f07fc4869b2d668391d8ba9084630a497abd7a7dea9dfb0
     home: https://min.io
@@ -993,7 +1015,7 @@ entries:
     version: 3.5.3
   - apiVersion: v1
     appVersion: RELEASE.2022-02-07T08-17-33Z
-    created: "2023-07-09T00:12:52.046012432-07:00"
+    created: "2023-09-30T13:45:44.275957276-07:00"
     description: Multi-Cloud Object Storage
     digest: 68d643414ff0d565716c5715034fcbf1af262e041915a5c02eb51ec1a65c1ea0
     home: https://min.io
@@ -1015,7 +1037,7 @@ entries:
     version: 3.5.2
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2023-07-09T00:12:52.044649337-07:00"
+    created: "2023-09-30T13:45:44.275214578-07:00"
     description: Multi-Cloud Object Storage
     digest: a3e855ed0f31233b989fffd775a29d6fbfa0590089010ff16783fd7f142ef6e7
     home: https://min.io
@@ -1037,7 +1059,7 @@ entries:
     version: 3.5.1
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2023-07-09T00:12:52.043927219-07:00"
+    created: "2023-09-30T13:45:44.274471492-07:00"
     description: Multi-Cloud Object Storage
     digest: b1b0ae3c54b4260a698753e11d7781bb8ddc67b7e3fbf0af82796e4cd4ef92a3
     home: https://min.io
@@ -1059,7 +1081,7 @@ entries:
     version: 3.5.0
   - apiVersion: v1
     appVersion: RELEASE.2022-01-28T02-28-16Z
-    created: "2023-07-09T00:12:52.043197969-07:00"
+    created: "2023-09-30T13:45:44.273733788-07:00"
     description: Multi-Cloud Object Storage
     digest: fecf25d2d3fb208c6f894fed642a60780a570b7f6d0adddde846af7236dc80aa
     home: https://min.io
@@ -1081,7 +1103,7 @@ entries:
     version: 3.4.8
   - apiVersion: v1
     appVersion: RELEASE.2022-01-25T19-56-04Z
-    created: "2023-07-09T00:12:52.042459778-07:00"
+    created: "2023-09-30T13:45:44.272981708-07:00"
     description: Multi-Cloud Object Storage
     digest: c78008caa5ce98f64c887630f59d0cbd481cb3f19a7d4e9d3e81bf4e1e45cadc
     home: https://min.io
@@ -1103,7 +1125,7 @@ entries:
     version: 3.4.7
   - apiVersion: v1
     appVersion: RELEASE.2022-01-08T03-11-54Z
-    created: "2023-07-09T00:12:52.041744623-07:00"
+    created: "2023-09-30T13:45:44.272032593-07:00"
     description: Multi-Cloud Object Storage
     digest: 8f2e2691bf897f74ff094dd370ec56ba9d417e5e8926710c14c2ba346330238d
     home: https://min.io
@@ -1125,7 +1147,7 @@ entries:
     version: 3.4.6
   - apiVersion: v1
     appVersion: RELEASE.2022-01-04T07-41-07Z
-    created: "2023-07-09T00:12:52.040983842-07:00"
+    created: "2023-09-30T13:45:44.270594751-07:00"
     description: Multi-Cloud Object Storage
     digest: bacd140f0016fab35f516bde787da6449b3a960c071fad9e4b6563118033ac84
     home: https://min.io
@@ -1147,7 +1169,7 @@ entries:
     version: 3.4.5
   - apiVersion: v1
     appVersion: RELEASE.2021-12-29T06-49-06Z
-    created: "2023-07-09T00:12:52.040253734-07:00"
+    created: "2023-09-30T13:45:44.269865392-07:00"
     description: Multi-Cloud Object Storage
     digest: 48a453ea5ffeef25933904caefd9470bfb26224dfc2d1096bd0031467ba53007
     home: https://min.io
@@ -1169,7 +1191,7 @@ entries:
     version: 3.4.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2023-07-09T00:12:52.039441814-07:00"
+    created: "2023-09-30T13:45:44.269127567-07:00"
     description: Multi-Cloud Object Storage
     digest: 47ef4a930713b98f9438ceca913c6e700f85bb25dba5624b056486254b5f0c60
     home: https://min.io
@@ -1191,7 +1213,7 @@ entries:
     version: 3.4.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2023-07-09T00:12:52.037915028-07:00"
+    created: "2023-09-30T13:45:44.268163723-07:00"
     description: Multi-Cloud Object Storage
     digest: d6763f7e2ea66810bd55eb225579a9c3b968f9ae1256f45fd469362e55d846ff
     home: https://min.io
@@ -1213,7 +1235,7 @@ entries:
     version: 3.4.2
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-07-09T00:12:52.037190929-07:00"
+    created: "2023-09-30T13:45:44.267246354-07:00"
     description: Multi-Cloud Object Storage
     digest: 2fb822c87216ba3fc2ae51a54a0a3e239aa560d86542991504a841cc2a2b9a37
     home: https://min.io
@@ -1235,7 +1257,7 @@ entries:
     version: 3.4.1
   - apiVersion: v1
     appVersion: RELEASE.2021-12-18T04-42-33Z
-    created: "2023-07-09T00:12:52.036385019-07:00"
+    created: "2023-09-30T13:45:44.266343771-07:00"
     description: Multi-Cloud Object Storage
     digest: fa8ba1aeb1a15316c6be8403416a5e6b5e6139b7166592087e7bddc9e6db5453
     home: https://min.io
@@ -1257,7 +1279,7 @@ entries:
     version: 3.4.0
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-07-09T00:12:52.035682883-07:00"
+    created: "2023-09-30T13:45:44.265456326-07:00"
     description: Multi-Cloud Object Storage
     digest: b9b0af9ca50b8d00868e1f1b989dca275829d9110af6de91bb9b3a398341e894
     home: https://min.io
@@ -1279,7 +1301,7 @@ entries:
     version: 3.3.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-07-09T00:12:52.034982834-07:00"
+    created: "2023-09-30T13:45:44.264304002-07:00"
     description: Multi-Cloud Object Storage
     digest: f8b22a5b8fe95a7ddf61b825e17d11c9345fb10e4c126b0d78381608aa300a08
     home: https://min.io
@@ -1301,7 +1323,7 @@ entries:
     version: 3.3.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-07-09T00:12:52.034279304-07:00"
+    created: "2023-09-30T13:45:44.259435328-07:00"
     description: Multi-Cloud Object Storage
     digest: c48d474f269427abe5ab446f00687d0625b3d1adfc5c73bdb4b21ca9e42853fb
     home: https://min.io
@@ -1323,7 +1345,7 @@ entries:
     version: 3.3.2
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2023-07-09T00:12:52.033504928-07:00"
+    created: "2023-09-30T13:45:44.258448201-07:00"
     description: Multi-Cloud Object Storage
     digest: 7c3da39d9b0090cbf5efedf0cc163a1e2df05becc5152c3add8e837384690bc4
     home: https://min.io
@@ -1345,7 +1367,7 @@ entries:
     version: 3.3.1
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2023-07-09T00:12:52.032082552-07:00"
+    created: "2023-09-30T13:45:44.257734262-07:00"
     description: Multi-Cloud Object Storage
     digest: 50d6590b4cc779c40f81cc13b1586fbe508aa7f3230036c760bfc5f4154fbce4
     home: https://min.io
@@ -1367,7 +1389,7 @@ entries:
     version: 3.3.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-13T00-23-17Z
-    created: "2023-07-09T00:12:52.031187915-07:00"
+    created: "2023-09-30T13:45:44.257013636-07:00"
     description: Multi-Cloud Object Storage
     digest: 5b797b7208cd904c11a76cd72938c8652160cb5fcd7f09fa41e4e703e6d64054
     home: https://min.io
@@ -1389,7 +1411,7 @@ entries:
     version: 3.2.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-10T16-53-30Z
-    created: "2023-07-09T00:12:52.030483162-07:00"
+    created: "2023-09-30T13:45:44.25605328-07:00"
     description: Multi-Cloud Object Storage
     digest: e084ac4bb095f071e59f8f08bd092e4ab2404c1ddadacfdce7dbe248f1bafff8
     home: https://min.io
@@ -1411,7 +1433,7 @@ entries:
     version: 3.1.9
   - apiVersion: v1
     appVersion: RELEASE.2021-10-06T23-36-31Z
-    created: "2023-07-09T00:12:52.02979236-07:00"
+    created: "2023-09-30T13:45:44.255324762-07:00"
     description: Multi-Cloud Object Storage
     digest: 2890430a8d9487d1fa5508c26776e4881d0086b2c052aa6bdc65c0e4423b9159
     home: https://min.io
@@ -1433,7 +1455,7 @@ entries:
     version: 3.1.8
   - apiVersion: v1
     appVersion: RELEASE.2021-10-02T16-31-05Z
-    created: "2023-07-09T00:12:52.029062485-07:00"
+    created: "2023-09-30T13:45:44.254617009-07:00"
     description: Multi-Cloud Object Storage
     digest: 01a92196af6c47e3a01e1c68d7cf693a8bc487cba810c2cecff155071e4d6a11
     home: https://min.io
@@ -1455,7 +1477,7 @@ entries:
     version: 3.1.7
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2023-07-09T00:12:52.028178186-07:00"
+    created: "2023-09-30T13:45:44.253895027-07:00"
     description: Multi-Cloud Object Storage
     digest: e779d73f80b75f33b9c9d995ab10fa455c9c57ee575ebc54e06725a64cd04310
     home: https://min.io
@@ -1477,7 +1499,7 @@ entries:
     version: 3.1.6
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2023-07-09T00:12:52.027297471-07:00"
+    created: "2023-09-30T13:45:44.252741769-07:00"
     description: Multi-Cloud Object Storage
     digest: 19de4bbc8a400f0c2a94c5e85fc25c9bfc666e773fb3e368dd621d5a57dd1c2a
     home: https://min.io
@@ -1499,7 +1521,7 @@ entries:
     version: 3.1.5
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2023-07-09T00:12:52.02646493-07:00"
+    created: "2023-09-30T13:45:44.251426132-07:00"
     description: Multi-Cloud Object Storage
     digest: f789d93a171296dd01af0105a5ce067c663597afbb2432faeda293b752b355c0
     home: https://min.io
@@ -1521,7 +1543,7 @@ entries:
     version: 3.1.4
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-07-09T00:12:52.025609511-07:00"
+    created: "2023-09-30T13:45:44.2504309-07:00"
     description: Multi-Cloud Object Storage
     digest: e2eb34d31560b012ef6581f0ff6004ea4376c968cbe0daed2d8f3a614a892afb
     home: https://min.io
@@ -1543,7 +1565,7 @@ entries:
     version: 3.1.3
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-07-09T00:12:52.02463492-07:00"
+    created: "2023-09-30T13:45:44.249671479-07:00"
     description: Multi-Cloud Object Storage
     digest: 8d7e0cc46b3583abd71b97dc0c071f98321101f90eca17348f1e9e0831be64cd
     home: https://min.io
@@ -1565,7 +1587,7 @@ entries:
     version: 3.1.2
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-07-09T00:12:52.016273999-07:00"
+    created: "2023-09-30T13:45:44.248904262-07:00"
     description: Multi-Cloud Object Storage
     digest: 50dcbf366b1b21f4a6fc429d0b884c0c7ff481d0fb95c5e9b3ae157c348dd124
     home: https://min.io
@@ -1587,7 +1609,7 @@ entries:
     version: 3.1.1
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-07-09T00:12:52.015535262-07:00"
+    created: "2023-09-30T13:45:44.248011769-07:00"
     description: Multi-Cloud Object Storage
     digest: 6c01af55d2e2e5f716eabf6fef3a92a8464d0674529e9bacab292e5478a73b7a
     home: https://min.io
@@ -1609,7 +1631,7 @@ entries:
     version: 3.1.0
   - apiVersion: v1
     appVersion: RELEASE.2021-09-03T03-56-13Z
-    created: "2023-07-09T00:12:52.014848017-07:00"
+    created: "2023-09-30T13:45:44.246954326-07:00"
     description: Multi-Cloud Object Storage
     digest: 18e10be4d0458bc590ca9abf753227e0c70f60511495387b8d4fb15a4daf932e
     home: https://min.io
@@ -1631,7 +1653,7 @@ entries:
     version: 3.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2023-07-09T00:12:52.01416606-07:00"
+    created: "2023-09-30T13:45:44.246078126-07:00"
     description: Multi-Cloud Object Storage
     digest: f5b6e7f6272a9e71aef3b75555f6f756a39eef65cb78873f26451dba79b19906
     home: https://min.io
@@ -1653,7 +1675,7 @@ entries:
     version: 3.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2023-07-09T00:12:52.013496497-07:00"
+    created: "2023-09-30T13:45:44.24452265-07:00"
     description: Multi-Cloud Object Storage
     digest: 6d2ee1336c412affaaf209fdb80215be2a6ebb23ab2443adbaffef9e7df13fab
     home: https://min.io
@@ -1675,7 +1697,7 @@ entries:
     version: 3.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2023-07-09T00:12:52.012743494-07:00"
+    created: "2023-09-30T13:45:44.243841906-07:00"
     description: Multi-Cloud Object Storage
     digest: 0a004aaf5bb61deed6a5c88256d1695ebe2f9ff1553874a93e4acfd75e8d339b
     home: https://min.io
@@ -1695,7 +1717,7 @@ entries:
     version: 2.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2023-07-09T00:12:52.011973175-07:00"
+    created: "2023-09-30T13:45:44.242951969-07:00"
     description: Multi-Cloud Object Storage
     digest: fcd944e837ee481307de6aa3d387ea18c234f995a84c15abb211aab4a4054afc
     home: https://min.io
@@ -1715,7 +1737,7 @@ entries:
     version: 2.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2023-07-09T00:12:52.011186882-07:00"
+    created: "2023-09-30T13:45:44.242280535-07:00"
     description: Multi-Cloud Object Storage
     digest: 7b6c033d43a856479eb493ab8ca05b230f77c3e42e209e8f298fac6af1a9796f
     home: https://min.io
@@ -1735,7 +1757,7 @@ entries:
     version: 1.0.5
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2023-07-09T00:12:52.010322776-07:00"
+    created: "2023-09-30T13:45:44.241614992-07:00"
     description: Multi-Cloud Object Storage
     digest: abd221245ace16c8e0c6c851cf262d1474a5219dcbf25c4b2e7b77142f9c59ed
     home: https://min.io
@@ -1755,7 +1777,7 @@ entries:
     version: 1.0.4
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2023-07-09T00:12:52.00894929-07:00"
+    created: "2023-09-30T13:45:44.24052836-07:00"
     description: Multi-Cloud Object Storage
     digest: 922a333f5413d1042f7aa81929f43767f6ffca9b260c46713f04ce1dda86d57d
     home: https://min.io
@@ -1775,7 +1797,7 @@ entries:
     version: 1.0.3
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2023-07-09T00:12:52.008126414-07:00"
+    created: "2023-09-30T13:45:44.23955753-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 10e22773506bbfb1c66442937956534cf4057b94f06a977db78b8cd223588388
     home: https://min.io
@@ -1795,7 +1817,7 @@ entries:
     version: 1.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2023-07-09T00:12:52.007422376-07:00"
+    created: "2023-09-30T13:45:44.234084367-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: ef86ab6df23d6942705da9ef70991b649638c51bc310587d37a425268ba4a06c
     home: https://min.io
@@ -1815,7 +1837,7 @@ entries:
     version: 1.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-17T20-53-08Z
-    created: "2023-07-09T00:12:52.006698484-07:00"
+    created: "2023-09-30T13:45:44.233235408-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 1add7608692cbf39aaf9b1252530e566f7b2f306a14e390b0f49b97a20f2b188
     home: https://min.io
@@ -1833,4 +1855,4 @@ entries:
     urls:
     - https://charts.min.io/helm-releases/minio-1.0.0.tgz
     version: 1.0.0
-generated: "2023-07-09T00:12:52.00579794-07:00"
+generated: "2023-09-30T13:45:44.232215184-07:00"

internal/config/notify/parse.go

@@ -362,6 +362,10 @@ var (
 			Key:   target.KafkaVersion,
 			Value: "",
 		},
+		config.KV{
+			Key:   target.KafkaBatchSize,
+			Value: "0",
+		},
 	}
 )
 
@@ -434,6 +438,15 @@ func GetNotifyKafka(kafkaKVS map[string]config.KVS) (map[string]target.KafkaArgs
 			versionEnv = versionEnv + config.Default + k
 		}
 
+		batchSizeEnv := target.EnvKafkaBatchSize
+		if k != config.Default {
+			batchSizeEnv = batchSizeEnv + config.Default + k
+		}
+		batchSize, err := strconv.ParseUint(env.Get(batchSizeEnv, kv.Get(target.KafkaBatchSize)), 10, 32)
+		if err != nil {
+			return nil, err
+		}
+
 		kafkaArgs := target.KafkaArgs{
 			Enable:     enabled,
 			Brokers:    brokers,
@@ -441,6 +454,7 @@ func GetNotifyKafka(kafkaKVS map[string]config.KVS) (map[string]target.KafkaArgs
 			QueueDir:   env.Get(queueDirEnv, kv.Get(target.KafkaQueueDir)),
 			QueueLimit: queueLimit,
 			Version:    env.Get(versionEnv, kv.Get(target.KafkaVersion)),
+			BatchSize:  uint32(batchSize),
 		}
 
 		tlsEnableEnv := target.EnvKafkaTLS

internal/event/target/amqp.go

@@ -291,7 +291,7 @@ func (target *AMQPTarget) Save(eventData event.Event) error {
 }
 
 // SendFromStore - reads an event from store and sends it to AMQP091.
-func (target *AMQPTarget) SendFromStore(eventKey string) error {
+func (target *AMQPTarget) SendFromStore(key store.Key) error {
 	if err := target.init(); err != nil {
 		return err
 	}
@@ -302,7 +302,7 @@ func (target *AMQPTarget) SendFromStore(eventKey string) error {
 	}
 	defer ch.Close()
 
-	eventData, eErr := target.store.Get(eventKey)
+	eventData, eErr := target.store.Get(key.Name)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and wouldve been already been sent successfully.
@@ -317,7 +317,7 @@ func (target *AMQPTarget) SendFromStore(eventKey string) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(eventKey)
+	return target.store.Del(key.Name)
 }
 
 // Close - does nothing and available for interface compatibility.

internal/event/target/elasticsearch.go

@@ -264,7 +264,7 @@ func (target *ElasticsearchTarget) send(eventData event.Event) error {
 }
 
 // SendFromStore - reads an event from store and sends it to Elasticsearch.
-func (target *ElasticsearchTarget) SendFromStore(eventKey string) error {
+func (target *ElasticsearchTarget) SendFromStore(key store.Key) error {
 	if err := target.init(); err != nil {
 		return err
 	}
@@ -277,7 +277,7 @@ func (target *ElasticsearchTarget) SendFromStore(eventKey string) error {
 		return err
 	}
 
-	eventData, eErr := target.store.Get(eventKey)
+	eventData, eErr := target.store.Get(key.Name)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and wouldve been already been sent successfully.
@@ -295,7 +295,7 @@ func (target *ElasticsearchTarget) SendFromStore(eventKey string) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(eventKey)
+	return target.store.Del(key.Name)
 }
 
 // Close - does nothing and available for interface compatibility.

internal/event/target/kafka.go

@@ -57,6 +57,7 @@ const (
 	KafkaClientTLSCert = "client_tls_cert"
 	KafkaClientTLSKey  = "client_tls_key"
 	KafkaVersion       = "version"
+	KafkaBatchSize     = "batch_size"
 
 	EnvKafkaEnable        = "MINIO_NOTIFY_KAFKA_ENABLE"
 	EnvKafkaBrokers       = "MINIO_NOTIFY_KAFKA_BROKERS"
@@ -73,6 +74,9 @@ const (
 	EnvKafkaClientTLSCert = "MINIO_NOTIFY_KAFKA_CLIENT_TLS_CERT"
 	EnvKafkaClientTLSKey  = "MINIO_NOTIFY_KAFKA_CLIENT_TLS_KEY"
 	EnvKafkaVersion       = "MINIO_NOTIFY_KAFKA_VERSION"
+	EnvKafkaBatchSize     = "MINIO_NOTIFY_KAFKA_BATCH_SIZE"
+
+	maxBatchLimit = 100
 )
 
 // KafkaArgs - Kafka target arguments.
@@ -83,6 +87,7 @@ type KafkaArgs struct {
 	QueueDir   string      `json:"queueDir"`
 	QueueLimit uint64      `json:"queueLimit"`
 	Version    string      `json:"version"`
+	BatchSize  uint32      `json:"batchSize"`
 	TLS        struct {
 		Enable        bool               `json:"enable"`
 		RootCAs       *x509.CertPool     `json:"-"`
@@ -122,6 +127,14 @@ func (k KafkaArgs) Validate() error {
 			return err
 		}
 	}
+	if k.BatchSize > 1 {
+		if k.QueueDir == "" {
+			return errors.New("batch should be enabled only if queue dir is enabled")
+		}
+		if k.BatchSize > maxBatchLimit {
+			return fmt.Errorf("batch limit should not exceed %d", maxBatchLimit)
+		}
+	}
 	return nil
 }
 
@@ -134,6 +147,7 @@ type KafkaTarget struct {
 	producer   sarama.SyncProducer
 	config     *sarama.Config
 	store      store.Store[event.Event]
+	batch      *store.Batch[string, *sarama.ProducerMessage]
 	loggerOnce logger.LogOnce
 	quitCh     chan struct{}
 }
@@ -188,55 +202,33 @@ func (target *KafkaTarget) send(eventData event.Event) error {
 	if target.producer == nil {
 		return store.ErrNotConnected
 	}
-	objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+	msg, err := target.toProducerMessage(eventData)
 	if err != nil {
 		return err
 	}
-	key := eventData.S3.Bucket.Name + "/" + objectName
-
-	data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}})
-	if err != nil {
-		return err
-	}
-
-	msg := sarama.ProducerMessage{
-		Topic: target.args.Topic,
-		Key:   sarama.StringEncoder(key),
-		Value: sarama.ByteEncoder(data),
-	}
-
-	_, _, err = target.producer.SendMessage(&msg)
-
+	_, _, err = target.producer.SendMessage(msg)
 	return err
 }
 
 // SendFromStore - reads an event from store and sends it to Kafka.
-func (target *KafkaTarget) SendFromStore(eventKey string) error {
+func (target *KafkaTarget) SendFromStore(key store.Key) error {
 	if err := target.init(); err != nil {
 		return err
 	}
 
+	// If batch is enabled, the event will be batched in memory
+	// and will be committed once the batch is full.
+	if target.batch != nil {
+		return target.addToBatch(key)
+	}
+
 	var err error
 	_, err = target.isActive()
 	if err != nil {
 		return err
 	}
 
-	if target.producer == nil {
-		brokers := []string{}
-		for _, broker := range target.args.Brokers {
-			brokers = append(brokers, broker.String())
-		}
-		target.producer, err = sarama.NewSyncProducer(brokers, target.config)
-		if err != nil {
-			if err != sarama.ErrOutOfBrokers {
-				return err
-			}
-			return store.ErrNotConnected
-		}
-	}
-
-	eventData, eErr := target.store.Get(eventKey)
+	eventData, eErr := target.store.Get(key.Name)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and wouldve been already been sent successfully.
@@ -248,15 +240,79 @@ func (target *KafkaTarget) SendFromStore(eventKey string) error {
 
 	err = target.send(eventData)
 	if err != nil {
-		// Sarama opens the ciruit breaker after 3 consecutive connection failures.
-		if err == sarama.ErrLeaderNotAvailable || err.Error() == "circuit breaker is open" {
+		if isKafkaConnErr(err) {
 			return store.ErrNotConnected
 		}
 		return err
 	}
 
 	// Delete the event from store.
-	return target.store.Del(eventKey)
+	return target.store.Del(key.Name)
+}
+
+func (target *KafkaTarget) addToBatch(key store.Key) error {
+	if target.batch.IsFull() {
+		if err := target.commitBatch(); err != nil {
+			return err
+		}
+	}
+	if _, ok := target.batch.GetByKey(key.Name); !ok {
+		eventData, err := target.store.Get(key.Name)
+		if err != nil {
+			if os.IsNotExist(err) {
+				return nil
+			}
+			return err
+		}
+		msg, err := target.toProducerMessage(eventData)
+		if err != nil {
+			return err
+		}
+		if err = target.batch.Add(key.Name, msg); err != nil {
+			return err
+		}
+	}
+	// commit the batch if the key is the last one present in the store.
+	if key.IsLast || target.batch.IsFull() {
+		return target.commitBatch()
+	}
+	return nil
+}
+
+func (target *KafkaTarget) commitBatch() error {
+	if _, err := target.isActive(); err != nil {
+		return err
+	}
+	keys, msgs, err := target.batch.GetAll()
+	if err != nil {
+		return err
+	}
+	if err = target.producer.SendMessages(msgs); err != nil {
+		if isKafkaConnErr(err) {
+			return store.ErrNotConnected
+		}
+		return err
+	}
+	return target.store.DelList(keys)
+}
+
+func (target *KafkaTarget) toProducerMessage(eventData event.Event) (*sarama.ProducerMessage, error) {
+	objectName, err := url.QueryUnescape(eventData.S3.Object.Key)
+	if err != nil {
+		return nil, err
+	}
+
+	key := eventData.S3.Bucket.Name + "/" + objectName
+	data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}})
+	if err != nil {
+		return nil, err
+	}
+
+	return &sarama.ProducerMessage{
+		Topic: target.args.Topic,
+		Key:   sarama.StringEncoder(key),
+		Value: sarama.ByteEncoder(data),
+	}, nil
 }
 
 // Close - closes underneath kafka connection.
@@ -335,16 +391,16 @@ func (target *KafkaTarget) initKafka() error {
 	// These settings are needed to ensure that kafka client doesn't hang on brokers
 	// refer https://github.com/IBM/sarama/issues/765#issuecomment-254333355
 	config.Producer.Retry.Max = 2
-	config.Producer.Retry.Backoff = (10 * time.Second)
+	config.Producer.Retry.Backoff = (1 * time.Second)
 	config.Producer.Return.Successes = true
 	config.Producer.Return.Errors = true
 	config.Producer.RequiredAcks = 1
-	config.Producer.Timeout = (10 * time.Second)
-	config.Net.ReadTimeout = (10 * time.Second)
-	config.Net.DialTimeout = (10 * time.Second)
-	config.Net.WriteTimeout = (10 * time.Second)
+	config.Producer.Timeout = (5 * time.Second)
+	config.Net.ReadTimeout = (5 * time.Second)
+	config.Net.DialTimeout = (5 * time.Second)
+	config.Net.WriteTimeout = (5 * time.Second)
 	config.Metadata.Retry.Max = 1
-	config.Metadata.Retry.Backoff = (10 * time.Second)
+	config.Metadata.Retry.Backoff = (1 * time.Second)
 	config.Metadata.RefreshFrequency = (15 * time.Minute)
 
 	target.config = config
@@ -394,8 +450,16 @@ func NewKafkaTarget(id string, args KafkaArgs, loggerOnce logger.LogOnce) (*Kafk
 	}
 
 	if target.store != nil {
+		if args.BatchSize > 1 {
+			target.batch = store.NewBatch[string, *sarama.ProducerMessage](args.BatchSize)
+		}
 		store.StreamItems(target.store, target, target.quitCh, target.loggerOnce)
 	}
 
 	return target, nil
 }
+
+func isKafkaConnErr(err error) bool {
+	// Sarama opens the ciruit breaker after 3 consecutive connection failures.
+	return err == sarama.ErrLeaderNotAvailable || err.Error() == "circuit breaker is open"
+}

internal/event/target/mqtt.go

@@ -169,7 +169,7 @@ func (target *MQTTTarget) send(eventData event.Event) error {
 }
 
 // SendFromStore - reads an event from store and sends it to MQTT.
-func (target *MQTTTarget) SendFromStore(eventKey string) error {
+func (target *MQTTTarget) SendFromStore(key store.Key) error {
 	if err := target.init(); err != nil {
 		return err
 	}
@@ -180,7 +180,7 @@ func (target *MQTTTarget) SendFromStore(eventKey string) error {
 		return err
 	}
 
-	eventData, err := target.store.Get(eventKey)
+	eventData, err := target.store.Get(key.Name)
 	if err != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and wouldve been already been sent successfully.
@@ -195,7 +195,7 @@ func (target *MQTTTarget) SendFromStore(eventKey string) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(eventKey)
+	return target.store.Del(key.Name)
 }
 
 // Save - saves the events to the store if queuestore is configured, which will

internal/event/target/mysql.go

@@ -254,7 +254,7 @@ func (target *MySQLTarget) send(eventData event.Event) error {
 }
 
 // SendFromStore - reads an event from store and sends it to MySQL.
-func (target *MySQLTarget) SendFromStore(eventKey string) error {
+func (target *MySQLTarget) SendFromStore(key store.Key) error {
 	if err := target.init(); err != nil {
 		return err
 	}
@@ -273,7 +273,7 @@ func (target *MySQLTarget) SendFromStore(eventKey string) error {
 		}
 	}
 
-	eventData, eErr := target.store.Get(eventKey)
+	eventData, eErr := target.store.Get(key.Name)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and wouldve been already been sent successfully.
@@ -291,7 +291,7 @@ func (target *MySQLTarget) SendFromStore(eventKey string) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(eventKey)
+	return target.store.Del(key.Name)
 }
 
 // Close - closes underneath connections to MySQL database.

internal/event/target/nats.go

@@ -335,7 +335,7 @@ func (target *NATSTarget) send(eventData event.Event) error {
 }
 
 // SendFromStore - reads an event from store and sends it to Nats.
-func (target *NATSTarget) SendFromStore(eventKey string) error {
+func (target *NATSTarget) SendFromStore(key store.Key) error {
 	if err := target.init(); err != nil {
 		return err
 	}
@@ -345,7 +345,7 @@ func (target *NATSTarget) SendFromStore(eventKey string) error {
 		return err
 	}
 
-	eventData, eErr := target.store.Get(eventKey)
+	eventData, eErr := target.store.Get(key.Name)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and wouldve been already been sent successfully.
@@ -359,7 +359,7 @@ func (target *NATSTarget) SendFromStore(eventKey string) error {
 		return err
 	}
 
-	return target.store.Del(eventKey)
+	return target.store.Del(key.Name)
 }
 
 // Close - closes underneath connections to NATS server.

internal/event/target/nsq.go

@@ -178,7 +178,7 @@ func (target *NSQTarget) send(eventData event.Event) error {
 }
 
 // SendFromStore - reads an event from store and sends it to NSQ.
-func (target *NSQTarget) SendFromStore(eventKey string) error {
+func (target *NSQTarget) SendFromStore(key store.Key) error {
 	if err := target.init(); err != nil {
 		return err
 	}
@@ -188,7 +188,7 @@ func (target *NSQTarget) SendFromStore(eventKey string) error {
 		return err
 	}
 
-	eventData, eErr := target.store.Get(eventKey)
+	eventData, eErr := target.store.Get(key.Name)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and wouldve been already been sent successfully.
@@ -203,7 +203,7 @@ func (target *NSQTarget) SendFromStore(eventKey string) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(eventKey)
+	return target.store.Del(key.Name)
 }
 
 // Close - closes underneath connections to NSQD server.

internal/event/target/postgresql.go

@@ -251,7 +251,7 @@ func (target *PostgreSQLTarget) send(eventData event.Event) error {
 }
 
 // SendFromStore - reads an event from store and sends it to PostgreSQL.
-func (target *PostgreSQLTarget) SendFromStore(eventKey string) error {
+func (target *PostgreSQLTarget) SendFromStore(key store.Key) error {
 	if err := target.init(); err != nil {
 		return err
 	}
@@ -269,7 +269,7 @@ func (target *PostgreSQLTarget) SendFromStore(eventKey string) error {
 		}
 	}
 
-	eventData, eErr := target.store.Get(eventKey)
+	eventData, eErr := target.store.Get(key.Name)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and wouldve been already been sent successfully.
@@ -287,7 +287,7 @@ func (target *PostgreSQLTarget) SendFromStore(eventKey string) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(eventKey)
+	return target.store.Del(key.Name)
 }
 
 // Close - closes underneath connections to PostgreSQL database.

internal/event/target/redis.go

@@ -223,7 +223,7 @@ func (target *RedisTarget) send(eventData event.Event) error {
 }
 
 // SendFromStore - reads an event from store and sends it to redis.
-func (target *RedisTarget) SendFromStore(eventKey string) error {
+func (target *RedisTarget) SendFromStore(key store.Key) error {
 	if err := target.init(); err != nil {
 		return err
 	}
@@ -249,7 +249,7 @@ func (target *RedisTarget) SendFromStore(eventKey string) error {
 		target.firstPing = true
 	}
 
-	eventData, eErr := target.store.Get(eventKey)
+	eventData, eErr := target.store.Get(key.Name)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and would've been already been sent successfully.
@@ -267,7 +267,7 @@ func (target *RedisTarget) SendFromStore(eventKey string) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(eventKey)
+	return target.store.Del(key.Name)
 }
 
 // Close - releases the resources used by the pool.

internal/event/target/webhook.go

@@ -207,12 +207,12 @@ func (target *WebhookTarget) send(eventData event.Event) error {
 }
 
 // SendFromStore - reads an event from store and sends it to webhook.
-func (target *WebhookTarget) SendFromStore(eventKey string) error {
+func (target *WebhookTarget) SendFromStore(key store.Key) error {
 	if err := target.init(); err != nil {
 		return err
 	}
 
-	eventData, eErr := target.store.Get(eventKey)
+	eventData, eErr := target.store.Get(key.Name)
 	if eErr != nil {
 		// The last event key in a successful batch will be sent in the channel atmost once by the replayEvents()
 		// Such events will not exist and would've been already been sent successfully.
@@ -230,7 +230,7 @@ func (target *WebhookTarget) SendFromStore(eventKey string) error {
 	}
 
 	// Delete the event from store.
-	return target.store.Del(eventKey)
+	return target.store.Del(key.Name)
 }
 
 // Close - does nothing and available for interface compatibility.

internal/event/targetlist.go

@@ -22,6 +22,8 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+
+	"github.com/minio/minio/internal/store"
 )
 
 const (
@@ -34,7 +36,7 @@ type Target interface {
 	ID() TargetID
 	IsActive() (bool, error)
 	Save(Event) error
-	SendFromStore(string) error
+	SendFromStore(store.Key) error
 	Close() error
 	Store() TargetStore
 }

internal/event/targetlist_test.go

@@ -23,6 +23,8 @@ import (
 	"reflect"
 	"testing"
 	"time"
+
+	"github.com/minio/minio/internal/store"
 )
 
 type ExampleTarget struct {
@@ -61,7 +63,7 @@ func (target ExampleTarget) send(eventData Event) error {
 }
 
 // SendFromStore - interface compatible method does no-op.
-func (target *ExampleTarget) SendFromStore(eventKey string) error {
+func (target *ExampleTarget) SendFromStore(_ store.Key) error {
 	return nil
 }
 

internal/logger/console.go

@@ -49,8 +49,16 @@ func consoleLog(console Logger, msg string, args ...interface{}) {
 		msg = ansiRE.ReplaceAllLiteralString(msg, "")
 		console.json(msg, args...)
 	case quietFlag:
+		if len(msg) != 0 && len(args) == 0 {
+			args = append(args, msg)
+			msg = "%s"
+		}
 		console.quiet(msg+"\n", args...)
 	default:
+		if len(msg) != 0 && len(args) == 0 {
+			args = append(args, msg)
+			msg = "%s"
+		}
 		console.pretty(msg+"\n", args...)
 	}
 }

internal/logger/target/http/http.go

@@ -349,9 +349,9 @@ func New(config Config) *Target {
 }
 
 // SendFromStore - reads the log from store and sends it to webhook.
-func (h *Target) SendFromStore(key string) (err error) {
+func (h *Target) SendFromStore(key store.Key) (err error) {
 	var eventData interface{}
-	eventData, err = h.store.Get(key)
+	eventData, err = h.store.Get(key.Name)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil
@@ -372,7 +372,7 @@ func (h *Target) SendFromStore(key string) (err error) {
 		return err
 	}
 	// Delete the event from store.
-	return h.store.Del(key)
+	return h.store.Del(key.Name)
 }
 
 // Send the log message 'entry' to the http target.

internal/logger/target/kafka/kafka.go

@@ -361,8 +361,8 @@ func (h *Target) Send(ctx context.Context, entry interface{}) error {
 }
 
 // SendFromStore - reads the log from store and sends it to kafka.
-func (h *Target) SendFromStore(key string) (err error) {
-	auditEntry, err := h.store.Get(key)
+func (h *Target) SendFromStore(key store.Key) (err error) {
+	auditEntry, err := h.store.Get(key.Name)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil
@@ -376,7 +376,7 @@ func (h *Target) SendFromStore(key string) (err error) {
 		return
 	}
 	// Delete the event from store.
-	return h.store.Del(key)
+	return h.store.Del(key.Name)
 }
 
 // Cancel - cancels the target

internal/store/batch.go

@@ -0,0 +1,117 @@
+// Copyright (c) 2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package store
+
+import (
+	"errors"
+	"fmt"
+	"sync"
+)
+
+// ErrBatchFull indicates that the batch is full
+var ErrBatchFull = errors.New("batch is full")
+
+type key interface {
+	string | int | int64
+}
+
+// Batch represents an ordered batch
+type Batch[K key, T any] struct {
+	keys  []K
+	items map[K]T
+	limit uint32
+
+	sync.Mutex
+}
+
+// Add adds the item to the batch
+func (b *Batch[K, T]) Add(key K, item T) error {
+	b.Lock()
+	defer b.Unlock()
+
+	if b.isFull() {
+		return ErrBatchFull
+	}
+
+	if _, ok := b.items[key]; !ok {
+		b.keys = append(b.keys, key)
+	}
+	b.items[key] = item
+
+	return nil
+}
+
+// GetAll fetches the items and resets the batch
+// Returned items are not referenced by the batch
+func (b *Batch[K, T]) GetAll() (orderedKeys []K, orderedItems []T, err error) {
+	b.Lock()
+	defer b.Unlock()
+
+	orderedKeys = append([]K(nil), b.keys...)
+	for _, key := range orderedKeys {
+		item, ok := b.items[key]
+		if !ok {
+			err = fmt.Errorf("item not found for the key: %v; should not happen;", key)
+			return
+		}
+		orderedItems = append(orderedItems, item)
+		delete(b.items, key)
+	}
+
+	b.keys = b.keys[:0]
+
+	return
+}
+
+// GetByKey will get the batch item by the provided key
+func (b *Batch[K, T]) GetByKey(key K) (T, bool) {
+	b.Lock()
+	defer b.Unlock()
+
+	item, ok := b.items[key]
+	return item, ok
+}
+
+// Len returns the no of items in the batch
+func (b *Batch[K, T]) Len() int {
+	b.Lock()
+	defer b.Unlock()
+
+	return len(b.keys)
+}
+
+// IsFull checks if the batch is full or not
+func (b *Batch[K, T]) IsFull() bool {
+	b.Lock()
+	defer b.Unlock()
+
+	return b.isFull()
+}
+
+func (b *Batch[K, T]) isFull() bool {
+	return len(b.items) >= int(b.limit)
+}
+
+// NewBatch creates a new batch
+func NewBatch[K key, T any](limit uint32) *Batch[K, T] {
+	return &Batch[K, T]{
+		keys:  make([]K, 0, limit),
+		items: make(map[K]T, limit),
+		limit: limit,
+	}
+}

internal/store/batch_test.go

@@ -0,0 +1,126 @@
+// Copyright (c) 2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package store
+
+import (
+	"errors"
+	"sync"
+	"testing"
+)
+
+func TestBatch(t *testing.T) {
+	var limit uint32 = 100
+	batch := NewBatch[int, int](limit)
+	for i := 0; i < int(limit); i++ {
+		if err := batch.Add(i, i); err != nil {
+			t.Fatalf("failed to add %v; %v", i, err)
+		}
+		if _, ok := batch.GetByKey(i); !ok {
+			t.Fatalf("failed to get the item by key %v after adding", i)
+		}
+	}
+	err := batch.Add(101, 101)
+	if err == nil || !errors.Is(err, ErrBatchFull) {
+		t.Fatalf("Expected err %v but got %v", ErrBatchFull, err)
+	}
+	if !batch.IsFull() {
+		t.Fatal("Expected batch.IsFull to be true but got false")
+	}
+	batchLen := batch.Len()
+	if batchLen != int(limit) {
+		t.Fatalf("expected batch length to be %v but got %v", limit, batchLen)
+	}
+	keys, items, err := batch.GetAll()
+	if err != nil {
+		t.Fatalf("unable to get the items from the batch; %v", err)
+	}
+	if len(items) != int(limit) {
+		t.Fatalf("Expected length of the batch items to be %v but got %v", limit, len(items))
+	}
+	if len(keys) != int(limit) {
+		t.Fatalf("Expected length of the batch keys to be %v but got %v", limit, len(items))
+	}
+	batchLen = batch.Len()
+	if batchLen != 0 {
+		t.Fatalf("expected batch to be empty but still left with %d items", batchLen)
+	}
+	// Add duplicate entries
+	for i := 0; i < 10; i++ {
+		if err := batch.Add(99, 99); err != nil {
+			t.Fatalf("failed to add duplicate item %v to batch after Get; %v", i, err)
+		}
+	}
+	if _, ok := batch.GetByKey(99); !ok {
+		t.Fatal("failed to get the duplicxate item by key '99' after adding")
+	}
+	keys, items, err = batch.GetAll()
+	if err != nil {
+		t.Fatalf("unable to get the items from the batch; %v", err)
+	}
+	if len(items) != 1 {
+		t.Fatalf("Expected length of the batch items to be 1 but got %v", len(items))
+	}
+	if len(keys) != 1 {
+		t.Fatalf("Expected length of the batch keys to be 1 but got %v", len(items))
+	}
+	// try adding again after Get.
+	for i := 0; i < int(limit); i++ {
+		if err := batch.Add(i, i); err != nil {
+			t.Fatalf("failed to add item %v to batch after Get; %v", i, err)
+		}
+		if _, ok := batch.GetByKey(i); !ok {
+			t.Fatalf("failed to get the item by key %v after adding", i)
+		}
+	}
+}
+
+func TestBatchWithConcurrency(t *testing.T) {
+	var limit uint32 = 100
+	batch := NewBatch[int, int](limit)
+
+	var wg sync.WaitGroup
+	for i := 0; i < int(limit); i++ {
+		wg.Add(1)
+		go func(item int) {
+			defer wg.Done()
+			if err := batch.Add(item, item); err != nil {
+				t.Errorf("failed to add item %v; %v", item, err)
+				return
+			}
+			if _, ok := batch.GetByKey(item); !ok {
+				t.Errorf("failed to get the item by key %v after adding", item)
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	keys, items, err := batch.GetAll()
+	if err != nil {
+		t.Fatalf("unable to get the items from the batch; %v", err)
+	}
+	if len(items) != int(limit) {
+		t.Fatalf("expected batch length %v but got %v", limit, len(items))
+	}
+	if len(keys) != int(limit) {
+		t.Fatalf("Expected length of the batch keys to be %v but got %v", limit, len(items))
+	}
+	batchLen := batch.Len()
+	if batchLen != 0 {
+		t.Fatalf("expected batch to be empty but still left with %d items", batchLen)
+	}
+}

internal/store/queuestore.go

@@ -167,6 +167,21 @@ func (store *QueueStore[_]) Del(key string) error {
 	return store.del(key)
 }
 
+// DelList - Deletes a list of entries from the store.
+// Returns an error even if one key fails to be deleted.
+func (store *QueueStore[_]) DelList(keys []string) error {
+	store.Lock()
+	defer store.Unlock()
+
+	for _, key := range keys {
+		if err := store.del(key); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // Len returns the entry count.
 func (store *QueueStore[_]) Len() int {
 	store.RLock()

internal/store/store.go

@@ -39,7 +39,7 @@ var ErrNotConnected = errors.New("not connected to target server/service")
 // Target - store target interface
 type Target interface {
 	Name() string
-	SendFromStore(key string) error
+	SendFromStore(key Key) error
 }
 
 // Store - Used to persist items.
@@ -49,16 +49,23 @@ type Store[I any] interface {
 	Len() int
 	List() ([]string, error)
 	Del(key string) error
+	DelList(key []string) error
 	Open() error
 	Extension() string
 }
 
+// Key denotes the key present in the store.
+type Key struct {
+	Name   string
+	IsLast bool
+}
+
 // replayItems - Reads the items from the store and replays.
-func replayItems[I any](store Store[I], doneCh <-chan struct{}, log logger, id string) <-chan string {
-	itemKeyCh := make(chan string)
+func replayItems[I any](store Store[I], doneCh <-chan struct{}, log logger, id string) <-chan Key {
+	keyCh := make(chan Key)
 
 	go func() {
-		defer close(itemKeyCh)
+		defer close(keyCh)
 
 		retryTicker := time.NewTicker(retryInterval)
 		defer retryTicker.Stop()
@@ -68,9 +75,10 @@ func replayItems[I any](store Store[I], doneCh <-chan struct{}, log logger, id s
 			if err != nil {
 				log(context.Background(), fmt.Errorf("store.List() failed with: %w", err), id)
 			} else {
-				for _, name := range names {
+				keyCount := len(names)
+				for i, name := range names {
 					select {
-					case itemKeyCh <- strings.TrimSuffix(name, store.Extension()):
+					case keyCh <- Key{strings.TrimSuffix(name, store.Extension()), keyCount == i+1}:
 					// Get next key.
 					case <-doneCh:
 						return
@@ -86,17 +94,17 @@ func replayItems[I any](store Store[I], doneCh <-chan struct{}, log logger, id s
 		}
 	}()
 
-	return itemKeyCh
+	return keyCh
 }
 
 // sendItems - Reads items from the store and re-plays.
-func sendItems(target Target, itemKeyCh <-chan string, doneCh <-chan struct{}, logger logger) {
+func sendItems(target Target, keyCh <-chan Key, doneCh <-chan struct{}, logger logger) {
 	retryTicker := time.NewTicker(retryInterval)
 	defer retryTicker.Stop()
 
-	send := func(itemKey string) bool {
+	send := func(key Key) bool {
 		for {
-			err := target.SendFromStore(itemKey)
+			err := target.SendFromStore(key)
 			if err == nil {
 				break
 			}
@@ -120,13 +128,13 @@ func sendItems(target Target, itemKeyCh <-chan string, doneCh <-chan struct{}, l
 
 	for {
 		select {
-		case itemKey, ok := <-itemKeyCh:
+		case key, ok := <-keyCh:
 			if !ok {
 				// closed channel.
 				return
 			}
 
-			if !send(itemKey) {
+			if !send(key) {
 				return
 			}
 		case <-doneCh:
@@ -139,8 +147,8 @@ func sendItems(target Target, itemKeyCh <-chan string, doneCh <-chan struct{}, l
 func StreamItems[I any](store Store[I], target Target, doneCh <-chan struct{}, logger logger) {
 	go func() {
 		// Replays the items from the store.
-		itemKeyCh := replayItems(store, doneCh, logger, target.Name())
+		keyCh := replayItems(store, doneCh, logger, target.Name())
 		// Send items from the store.
-		sendItems(target, itemKeyCh, doneCh, logger)
+		sendItems(target, keyCh, doneCh, logger)
 	}()
 }