Added new API errors for LDAP (#19415)

* change internal errors to named errors

* Change names

.github/workflows/depsreview.yaml

@@ -9,6 +9,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@v1
+        uses: actions/dependency-review-action@v4

.github/workflows/go-fips.yml

@@ -24,8 +24,8 @@ jobs:
         go-version: [1.21.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
 

.github/workflows/go-healing.yml

@@ -24,8 +24,8 @@ jobs:
         go-version: [1.21.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true

.github/workflows/go-lint.yml

@@ -24,8 +24,8 @@ jobs:
         go-version: [1.21.x]
         os: [ubuntu-latest, windows-latest]
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true

.github/workflows/go.yml

@@ -24,8 +24,8 @@ jobs:
         go-version: [1.21.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true

.github/workflows/helm-lint.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Helm
         uses: azure/setup-helm@v3

.github/workflows/iam-integrations.yaml

@@ -76,8 +76,8 @@ jobs:
             openid: "http://127.0.0.1:5556/dex"
 
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true

.github/workflows/mint.yml

@@ -25,10 +25,10 @@ jobs:
           sudo -S rm -rf ${GITHUB_WORKSPACE}
           mkdir ${GITHUB_WORKSPACE}
       - name: checkout-step
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: setup-go-step
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v5
         with:
           go-version: 1.21.x
 

.github/workflows/replication.yaml

@@ -25,8 +25,8 @@ jobs:
         go-version: [1.21.x]
 
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true

.github/workflows/root-disable.yml

@@ -25,8 +25,8 @@ jobs:
         os: [ubuntu-latest]
 
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true

.github/workflows/shfmt.yml

@@ -14,7 +14,7 @@ jobs:
     name: runner / shfmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: luizm/action-sh-checker@master
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/upgrade-ci-cd.yaml

@@ -25,8 +25,8 @@ jobs:
         os: [ubuntu-latest]
 
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true

.github/workflows/vulncheck.yml

@@ -17,11 +17,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.21.8
+          go-version: 1.21.9
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

.typos.toml

@@ -19,6 +19,19 @@ extend-ignore-re = [
 [default.extend-words]
 "encrypter" = "encrypter"
 "requestor" = "requestor"
+"KMS" = "KMS"
+"kms" = "kms"
+"Kms" = "Kms"
+"Dur" = "Dur"
+"EOF" = "EOF"
+"hd" = "hd"
+"ws" = "ws"
+"guid" = "guid"
+"lst" = "lst"
+"pn" = "pn"
+"Iy" = "Iy"
+"ro" = "ro"
+"thr" = "thr"
 
 [default.extend-identifiers]
 "bui" = "bui"

Dockerfile.hotfix

@@ -21,6 +21,11 @@ RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go
     curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
     chmod +x /go/bin/mc
 
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
+       curl -L -s -q https://github.com/moparisthebest/static-curl/releases/latest/download/curl-${TARGETARCH} -o /go/bin/curl; \
+       chmod +x /go/bin/curl; \
+    fi
+
 # Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
 RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
     minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
@@ -49,6 +54,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /go/bin/minio /usr/bin/minio
 COPY --from=build /go/bin/mc /usr/bin/mc
+COPY --from=build /go/bin/cur* /usr/bin/
 
 COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE

Dockerfile.release

@@ -21,6 +21,11 @@ RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go
     curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
     chmod +x /go/bin/mc
 
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
+       curl -L -s -q https://github.com/moparisthebest/static-curl/releases/latest/download/curl-${TARGETARCH} -o /go/bin/curl; \
+       chmod +x /go/bin/curl; \
+    fi
+
 # Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
 RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
     minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
@@ -49,6 +54,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /go/bin/minio /usr/bin/minio
 COPY --from=build /go/bin/mc /usr/bin/mc
+COPY --from=build /go/bin/cur* /usr/bin/
 
 COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE

Dockerfile.release.fips

@@ -16,6 +16,11 @@ RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archiv
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /go/bin/minio.minisig && \
     chmod +x /go/bin/minio
 
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
+       curl -L -s -q https://github.com/moparisthebest/static-curl/releases/latest/download/curl-${TARGETARCH} -o /go/bin/curl; \
+       chmod +x /go/bin/curl; \
+    fi
+
 # Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
 RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
 
@@ -41,6 +46,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /go/bin/minio /usr/bin/minio
+COPY --from=build /go/bin/cur* /usr/bin/
 
 COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE

Dockerfile.release.old_cpu

@@ -21,6 +21,11 @@ RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go
     curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
     chmod +x /go/bin/mc
 
+RUN if [ "$TARGETARCH" = "amd64" ]; then \
+       curl -L -s -q https://github.com/moparisthebest/static-curl/releases/latest/download/curl-${TARGETARCH} -o /go/bin/curl; \
+       chmod +x /go/bin/curl; \
+    fi
+
 # Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
 RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
     minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
@@ -49,6 +54,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /go/bin/minio /usr/bin/minio
 COPY --from=build /go/bin/mc /usr/bin/mc
+COPY --from=build /go/bin/cur* /usr/bin/
 
 COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE

Makefile

@@ -24,7 +24,7 @@ help: ## print this help
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
 	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR)
-	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@6ac204f0b4d48d17ab4fa442134c7fba13127a4e
+	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.10-0.20240227114326-6d6f813fff1b
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
 
 crosscompile: ## cross compile minio
@@ -107,6 +107,12 @@ test-site-replication-oidc: install-race ## verify automatic site replication
 test-site-replication-minio: install-race ## verify automatic site replication
 	@echo "Running tests for automatic site replication of IAM (with MinIO IDP)"
 	@(env bash $(PWD)/docs/site-replication/run-multi-site-minio-idp.sh)
+	@echo "Running tests for automatic site replication of SSE-C objects"
+	@(env bash $(PWD)/docs/site-replication/run-ssec-object-replication.sh)
+	@echo "Running tests for automatic site replication of SSE-C objects with SSE-KMS enabled for bucket"
+	@(env bash $(PWD)/docs/site-replication/run-sse-kms-object-replication.sh)
+	@echo "Running tests for automatic site replication of SSE-C objects with compression enabled for site"
+	@(env bash $(PWD)/docs/site-replication/run-ssec-object-replication-with-compression.sh)
 
 verify: ## verify minio various setups
 	@echo "Verifying build with race"
@@ -147,9 +153,9 @@ hotfix-vars:
 	$(eval VERSION := $(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD))
 
 hotfix: hotfix-vars clean install ## builds minio binary with hotfix tags
-	@wget -q -c https://github.com/minio/pkger/releases/download/v2.2.1/pkger_2.2.1_linux_amd64.deb
+	@wget -q -c https://github.com/minio/pkger/releases/download/v2.2.9/pkger_2.2.9_linux_amd64.deb
 	@wget -q -c https://raw.githubusercontent.com/minio/minio-service/v1.0.1/linux-systemd/distributed/minio.service
-	@sudo apt install ./pkger_2.2.1_linux_amd64.deb --yes
+	@sudo apt install ./pkger_2.2.9_linux_amd64.deb --yes
 	@mkdir -p minio-release/$(GOOS)-$(GOARCH)/archive
 	@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio
 	@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION)

buildscripts/minio-upgrade.sh

@@ -55,6 +55,11 @@ __init__() {
 
 	go install github.com/minio/mc@latest
 
+	## this is needed because github actions don't have
+	## docker-compose on all runners
+	go install github.com/docker/compose/v2/cmd@latest
+	mv -v /tmp/gopath/bin/cmd /tmp/gopath/bin/docker-compose
+
 	TAG=minio/minio:dev make docker
 
 	MINIO_VERSION=RELEASE.2019-12-19T22-52-26Z docker-compose \

cmd/admin-bucket-handlers.go

@@ -39,7 +39,6 @@ import (
 	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/policy"
 )
@@ -99,7 +98,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 	}
 
 	// Call site replication hook.
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta))
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta))
 
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
@@ -431,7 +430,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 			case bucketNotificationConfig:
 				config, err := globalBucketMetadataSys.GetNotificationConfig(bucket)
 				if err != nil {
-					logger.LogIf(ctx, err)
+					adminLogIf(ctx, err)
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
@@ -447,7 +446,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					if errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) {
 						continue
 					}
-					logger.LogIf(ctx, err)
+					adminLogIf(ctx, err)
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}

cmd/admin-handlers-config-kv.go

@@ -58,7 +58,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.ErrorKind)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
@@ -162,7 +162,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.ErrorKind)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
@@ -443,7 +443,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.ErrorKind)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}

cmd/admin-handlers-idp-config.go

@@ -31,7 +31,6 @@ import (
 	"github.com/minio/minio/internal/config"
 	cfgldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/ldap"
 	"github.com/minio/pkg/v2/policy"
@@ -60,7 +59,7 @@ func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.R
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.ErrorKind)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}

cmd/admin-handlers-idp-ldap.go

@@ -20,14 +20,12 @@ package cmd
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io"
 	"net/http"
 	"strings"
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/policy"
 )
@@ -105,6 +103,12 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 		return
 	}
 
+	// fail if ldap is not enabled
+	if !globalIAMSys.LDAPConfig.Enabled() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminLDAPNotEnabled), r.URL)
+		return
+	}
+
 	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
@@ -132,7 +136,7 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.ErrorKind)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
@@ -192,7 +196,8 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
 
 	// fail if ldap is not enabled
 	if !globalIAMSys.LDAPConfig.Enabled() {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errors.New("LDAP not enabled")), r.URL)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminLDAPNotEnabled), r.URL)
+		return
 	}
 
 	// Find the user for the request sender (as it may be sent via a service
@@ -232,12 +237,12 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
 		targetGroups = requestorGroups
 
 		// Deny if the target user is not LDAP
-		isLDAP, err := globalIAMSys.LDAPConfig.DoesUsernameExist(targetUser)
+		foundLDAPDN, err := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(targetUser)
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
-		if isLDAP == "" {
+		if foundLDAPDN == "" {
 			err := errors.New("Specified user does not exist on LDAP server")
 			APIErr := errorCodes.ToAPIErrWithErr(ErrAdminNoSuchUser, err)
 			writeErrorResponseJSON(ctx, w, APIErr, r.URL)
@@ -253,15 +258,23 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
 			opts.claims[k] = v
 		}
 	} else {
-		isDN := globalIAMSys.LDAPConfig.IsLDAPUserDN(targetUser)
+		// We still need to ensure that the target user is a valid LDAP user.
+		//
+		// The target user may be supplied as a (short) username or a DN.
+		// However, for now, we only support using the short username.
 
+		isDN := globalIAMSys.LDAPConfig.ParsesAsDN(targetUser)
 		opts.claims[ldapUserN] = targetUser // simple username
 		targetUser, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
 		if err != nil {
 			// if not found, check if DN
-			if strings.Contains(err.Error(), "not found") && isDN {
-				// warn user that DNs are not allowed
-				err = fmt.Errorf("Must use short username to add service account. %w", err)
+			if strings.Contains(err.Error(), "User DN not found for:") {
+				if isDN {
+					// warn user that DNs are not allowed
+					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminLDAPExpectedLoginName, err), r.URL)
+				} else {
+					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminNoSuchUser, err), r.URL)
+				}
 			}
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -300,7 +313,7 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
 	// Call hook for cluster-replication if the service account is not for a
 	// root user.
 	if newCred.ParentUser != globalActiveCred.AccessKey {
-		logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
 				Create: &madmin.SRSvcAccCreate{
@@ -373,11 +386,12 @@ func (a adminAPIHandlers) ListAccessKeysLDAP(w http.ResponseWriter, r *http.Requ
 		}
 	}
 
-	targetAccount, err := globalIAMSys.LDAPConfig.DoesUsernameExist(userDN)
+	targetAccount, err := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(userDN)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
-	} else if userDN == "" {
+	}
+	if targetAccount == "" {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
 		return
 	}

cmd/admin-handlers-pools.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -25,8 +26,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
+	"github.com/minio/pkg/v2/env"
 	"github.com/minio/pkg/v2/policy"
 )
 
@@ -106,21 +107,12 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 		poolIndices = append(poolIndices, idx)
 	}
 
-	if len(poolIndices) > 0 && !globalEndpoints[poolIndices[0]].Endpoints[0].IsLocal {
-		ep := globalEndpoints[poolIndices[0]].Endpoints[0]
-		for nodeIdx, proxyEp := range globalProxyEndpoints {
-			if proxyEp.Endpoint.Host == ep.Host {
-				if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
-					return
-				}
-			}
+	if len(poolIndices) == 0 || !proxyDecommissionRequest(ctx, globalEndpoints[poolIndices[0]].Endpoints[0], w, r) {
+		if err := z.Decommission(r.Context(), poolIndices...); err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
 		}
 	}
-
-	if err := z.Decommission(r.Context(), poolIndices...); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
 }
 
 func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Request) {
@@ -162,20 +154,12 @@ func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	if ep := globalEndpoints[idx].Endpoints[0]; !ep.IsLocal {
-		for nodeIdx, proxyEp := range globalProxyEndpoints {
-			if proxyEp.Endpoint.Host == ep.Host {
-				if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
-					return
-				}
-			}
+	if !proxyDecommissionRequest(ctx, globalEndpoints[idx].Endpoints[0], w, r) {
+		if err := pools.DecommissionCancel(ctx, idx); err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
 		}
 	}
-
-	if err := pools.DecommissionCancel(ctx, idx); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
 }
 
 func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
@@ -225,7 +209,7 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	logger.LogIf(r.Context(), json.NewEncoder(w).Encode(&status))
+	adminLogIf(r.Context(), json.NewEncoder(w).Encode(&status))
 }
 
 func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
@@ -258,7 +242,7 @@ func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
 		poolsStatus[idx] = status
 	}
 
-	logger.LogIf(r.Context(), json.NewEncoder(w).Encode(poolsStatus))
+	adminLogIf(r.Context(), json.NewEncoder(w).Encode(poolsStatus))
 }
 
 func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request) {
@@ -365,11 +349,11 @@ func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceNotStarted), r.URL)
 			return
 		}
-		logger.LogIf(ctx, fmt.Errorf("failed to fetch rebalance status: %w", err))
+		adminLogIf(ctx, fmt.Errorf("failed to fetch rebalance status: %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	logger.LogIf(r.Context(), json.NewEncoder(w).Encode(rs))
+	adminLogIf(r.Context(), json.NewEncoder(w).Encode(rs))
 }
 
 func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request) {
@@ -389,5 +373,20 @@ func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request)
 	// Cancel any ongoing rebalance operation
 	globalNotificationSys.StopRebalance(r.Context())
 	writeSuccessResponseHeadersOnly(w)
-	logger.LogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt))
+	adminLogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt))
+}
+
+func proxyDecommissionRequest(ctx context.Context, defaultEndPoint Endpoint, w http.ResponseWriter, r *http.Request) (proxy bool) {
+	host := env.Get("_MINIO_DECOM_ENDPOINT_HOST", defaultEndPoint.Host)
+	if host == "" {
+		return
+	}
+	for nodeIdx, proxyEp := range globalProxyEndpoints {
+		if proxyEp.Endpoint.Host == host && !proxyEp.IsLocal {
+			if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
+				return true
+			}
+		}
+	}
+	return
 }

cmd/admin-handlers-site-replication.go

@@ -32,7 +32,6 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v3"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/policy"
 )
@@ -55,7 +54,7 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 	opts := getSRAddOptions(r)
 	status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites, opts)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -93,7 +92,7 @@ func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := globalSiteReplicationSys.PeerJoinReq(ctx, joinArg); err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -140,7 +139,7 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
 		globalSiteReplicationSys.purgeDeletedBucket(ctx, objectAPI, bucket)
 	}
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -192,7 +191,7 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
 		err = globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo, item.UpdatedAt)
 	}
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -263,7 +262,7 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 		err = globalSiteReplicationSys.PeerBucketLCConfigHandler(ctx, item.Bucket, item.ExpiryLCConfig, item.UpdatedAt)
 	}
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -316,7 +315,6 @@ func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptio
 	if encryptionKey != "" {
 		data, err = madmin.DecryptData(encryptionKey, bytes.NewReader(data))
 		if err != nil {
-			logger.LogIf(ctx, err)
 			return SRError{
 				Cause: err,
 				Code:  ErrSiteReplicationInvalidRequest,
@@ -396,7 +394,7 @@ func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Req
 	opts := getSREditOptions(r)
 	status, err := globalSiteReplicationSys.EditPeerCluster(ctx, site, opts)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -433,7 +431,7 @@ func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := globalSiteReplicationSys.PeerEditReq(ctx, pi); err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -456,7 +454,7 @@ func (a adminAPIHandlers) SRStateEdit(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if err := globalSiteReplicationSys.PeerStateEditReq(ctx, state); err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -493,7 +491,7 @@ func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.R
 	}
 	status, err := globalSiteReplicationSys.RemovePeerCluster(ctx, objectAPI, rreq)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -524,7 +522,7 @@ func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := globalSiteReplicationSys.InternalRemoveReq(ctx, objectAPI, req); err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -586,7 +584,7 @@ func (a adminAPIHandlers) SiteReplicationDevNull(w http.ResponseWriter, r *http.
 			// If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec)
 			// would mean the network is not stable. Logging here will help in debugging network issues.
 			if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
-				logger.LogIf(ctx, err)
+				adminLogIf(ctx, err)
 			}
 		}
 		if err != nil {
@@ -609,5 +607,5 @@ func (a adminAPIHandlers) SiteReplicationNetPerf(w http.ResponseWriter, r *http.
 		duration = globalNetPerfMinDuration
 	}
 	result := siteNetperf(r.Context(), duration)
-	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(result))
+	adminLogIf(r.Context(), gob.NewEncoder(w).Encode(result))
 }

cmd/admin-handlers-users.go

@@ -35,9 +35,9 @@ import (
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/cachevalue"
 	"github.com/minio/minio/internal/config/dns"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/policy"
+	"github.com/puzpuzpuz/xsync/v3"
 )
 
 // RemoveUser - DELETE /minio/admin/v3/remove-user?accessKey=<access_key>
@@ -74,7 +74,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemIAMUser,
 		IAMUser: &madmin.SRIAMUser{
 			AccessKey:   accessKey,
@@ -278,7 +278,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemGroupInfo,
 		GroupInfo: &madmin.SRGroupInfo{
 			UpdateReq: updReq,
@@ -368,7 +368,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemGroupInfo,
 		GroupInfo: &madmin.SRGroupInfo{
 			UpdateReq: madmin.GroupAddRemove{
@@ -406,7 +406,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemIAMUser,
 		IAMUser: &madmin.SRIAMUser{
 			AccessKey:   accessKey,
@@ -495,14 +495,14 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	password := cred.SecretKey
 	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
 
 	var ureq madmin.AddOrUpdateUserReq
 	if err = json.Unmarshal(configBytes, &ureq); err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
@@ -513,7 +513,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemIAMUser,
 		IAMUser: &madmin.SRIAMUser{
 			AccessKey:   accessKey,
@@ -731,7 +731,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	// Call hook for cluster-replication if the service account is not for a
 	// root user.
 	if newCred.ParentUser != globalActiveCred.AccessKey {
-		logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
 				Create: &madmin.SRSvcAccCreate{
@@ -853,7 +853,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 
 	// Call site replication hook - non-root user accounts are replicated.
 	if svcAccount.ParentUser != globalActiveCred.AccessKey {
-		logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
 				Update: &madmin.SRSvcAccUpdate{
@@ -1115,7 +1115,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 
 	// Call site replication hook - non-root user accounts are replicated.
 	if svcAccount.ParentUser != "" && svcAccount.ParentUser != globalActiveCred.AccessKey {
-		logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
 				Delete: &madmin.SRSvcAccDelete{
@@ -1273,7 +1273,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	default:
 		policies, err := globalIAMSys.PolicyDBGet(accountName, cred.Groups...)
 		if err != nil {
-			logger.LogIf(ctx, err)
+			adminLogIf(ctx, err)
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
@@ -1425,7 +1425,7 @@ func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Requ
 	for name, p := range policies {
 		_, err = json.Marshal(p)
 		if err != nil {
-			logger.LogIf(ctx, err)
+			adminLogIf(ctx, err)
 			continue
 		}
 		newPolicies[name] = p
@@ -1455,7 +1455,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 	for name, p := range policies {
 		_, err = json.Marshal(p)
 		if err != nil {
-			logger.LogIf(ctx, err)
+			adminLogIf(ctx, err)
 			continue
 		}
 		newPolicies[name] = p
@@ -1485,7 +1485,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 
 	// Call cluster-replication policy creation hook to replicate policy deletion to
 	// other minio clusters.
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type:      madmin.SRIAMItemPolicy,
 		Name:      policyName,
 		UpdatedAt: UTCNow(),
@@ -1548,7 +1548,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 
 	// Call cluster-replication policy creation hook to replicate policy to
 	// other minio clusters.
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type:      madmin.SRIAMItemPolicy,
 		Name:      policyName,
 		Policy:    iamPolicyBytes,
@@ -1616,7 +1616,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 		return
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemPolicyMapping,
 		PolicyMapping: &madmin.SRPolicyMapping{
 			UserOrGroup: entityName,
@@ -1756,15 +1756,14 @@ func (a adminAPIHandlers) AttachDetachPolicyBuiltin(w http.ResponseWriter, r *ht
 }
 
 const (
-	allPoliciesFile            = "policies.json"
-	allUsersFile               = "users.json"
-	allGroupsFile              = "groups.json"
-	allSvcAcctsFile            = "svcaccts.json"
-	userPolicyMappingsFile     = "user_mappings.json"
-	groupPolicyMappingsFile    = "group_mappings.json"
-	stsUserPolicyMappingsFile  = "stsuser_mappings.json"
-	stsGroupPolicyMappingsFile = "stsgroup_mappings.json"
-	iamAssetsDir               = "iam-assets"
+	allPoliciesFile           = "policies.json"
+	allUsersFile              = "users.json"
+	allGroupsFile             = "groups.json"
+	allSvcAcctsFile           = "svcaccts.json"
+	userPolicyMappingsFile    = "user_mappings.json"
+	groupPolicyMappingsFile   = "group_mappings.json"
+	stsUserPolicyMappingsFile = "stsuser_mappings.json"
+	iamAssetsDir              = "iam-assets"
 )
 
 // ExportIAMHandler - exports all iam info as a zipped file
@@ -1790,17 +1789,17 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			sys:     nil,
 		})
 		if zerr != nil {
-			logger.LogIf(ctx, zerr)
+			adminLogIf(ctx, zerr)
 			return nil
 		}
 		header.Method = zip.Deflate
 		zwriter, zerr := zipWriter.CreateHeader(header)
 		if zerr != nil {
-			logger.LogIf(ctx, zerr)
+			adminLogIf(ctx, zerr)
 			return nil
 		}
 		if _, err := io.Copy(zwriter, r); err != nil {
-			logger.LogIf(ctx, err)
+			adminLogIf(ctx, err)
 		}
 		return nil
 	}
@@ -1813,7 +1812,6 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 		userPolicyMappingsFile,
 		groupPolicyMappingsFile,
 		stsUserPolicyMappingsFile,
-		stsGroupPolicyMappingsFile,
 	}
 	for _, f := range iamFiles {
 		iamFile := pathJoin(iamAssetsDir, f)
@@ -1821,7 +1819,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 		case allPoliciesFile:
 			allPolicies, err := globalIAMSys.ListPolicies(ctx, "")
 			if err != nil {
-				logger.LogIf(ctx, err)
+				adminLogIf(ctx, err)
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
 			}
@@ -1936,13 +1934,13 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 		case userPolicyMappingsFile:
-			userPolicyMap := make(map[string]MappedPolicy)
+			userPolicyMap := xsync.NewMapOf[string, MappedPolicy]()
 			err := globalIAMSys.store.loadMappedPolicies(ctx, regUser, false, userPolicyMap)
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
 			}
-			userPolData, err := json.Marshal(userPolicyMap)
+			userPolData, err := json.Marshal(mappedPoliciesToMap(userPolicyMap))
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -1953,13 +1951,13 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 		case groupPolicyMappingsFile:
-			groupPolicyMap := make(map[string]MappedPolicy)
+			groupPolicyMap := xsync.NewMapOf[string, MappedPolicy]()
 			err := globalIAMSys.store.loadMappedPolicies(ctx, regUser, true, groupPolicyMap)
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
 			}
-			grpPolData, err := json.Marshal(groupPolicyMap)
+			grpPolData, err := json.Marshal(mappedPoliciesToMap(groupPolicyMap))
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -1970,13 +1968,13 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 		case stsUserPolicyMappingsFile:
-			userPolicyMap := make(map[string]MappedPolicy)
+			userPolicyMap := xsync.NewMapOf[string, MappedPolicy]()
 			err := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, false, userPolicyMap)
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
 			}
-			userPolData, err := json.Marshal(userPolicyMap)
+			userPolData, err := json.Marshal(mappedPoliciesToMap(userPolicyMap))
 			if err != nil {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
@@ -1985,22 +1983,6 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
 			}
-		case stsGroupPolicyMappingsFile:
-			groupPolicyMap := make(map[string]MappedPolicy)
-			err := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, true, groupPolicyMap)
-			if err != nil {
-				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
-				return
-			}
-			grpPolData, err := json.Marshal(groupPolicyMap)
-			if err != nil {
-				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
-				return
-			}
-			if err = rawDataFn(bytes.NewReader(grpPolData), iamFile, len(grpPolData)); err != nil {
-				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
-				return
-			}
 		}
 	}
 }
@@ -2391,35 +2373,6 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 	}
-
-	// import sts group policy mappings
-	{
-		f, err := zr.Open(pathJoin(iamAssetsDir, stsGroupPolicyMappingsFile))
-		switch {
-		case errors.Is(err, os.ErrNotExist):
-		case err != nil:
-			writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, stsGroupPolicyMappingsFile, ""), r.URL)
-			return
-		default:
-			defer f.Close()
-			var grpPolicyMap map[string]MappedPolicy
-			data, err := io.ReadAll(f)
-			if err != nil {
-				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, stsGroupPolicyMappingsFile, ""), r.URL)
-				return
-			}
-			if err = json.Unmarshal(data, &grpPolicyMap); err != nil {
-				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, stsGroupPolicyMappingsFile, ""), r.URL)
-				return
-			}
-			for g, pm := range grpPolicyMap {
-				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, unknownIAMUserType, true); err != nil {
-					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsGroupPolicyMappingsFile, g), r.URL)
-					return
-				}
-			}
-		}
-	}
 }
 
 func addExpirationToCondValues(exp *time.Time, condValues map[string][]string) {

cmd/admin-handlers.go

@@ -130,7 +130,7 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 	// Download Binary Once
 	binC, bin, err := downloadBinary(u, mode)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -159,7 +159,7 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 				peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{
 					Host:           nerr.Host.String(),
 					CurrentVersion: Version,
-					UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
+					UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
 				}
 			}
 		}
@@ -176,7 +176,7 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 			peerResults[local] = madmin.ServerPeerUpdateStatus{
 				Host:           local,
 				CurrentVersion: Version,
-				UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
+				UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
 			}
 		}
 	} else {
@@ -212,7 +212,7 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 							Host:           nerr.Host.String(),
 							Err:            nerr.Err.Error(),
 							CurrentVersion: Version,
-							UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
+							UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
 						}
 					}
 				}
@@ -354,7 +354,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 	// Download Binary Once
 	binC, bin, err := downloadBinary(u, mode)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -368,7 +368,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 				StatusCode: http.StatusInternalServerError,
 			}
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+			adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
@@ -376,7 +376,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 
 	err = verifyBinary(u, sha256Sum, releaseInfo, mode, bytes.NewReader(bin))
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -389,7 +389,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 				StatusCode: http.StatusInternalServerError,
 			}
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+			adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
@@ -397,14 +397,14 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 
 	err = commitBinary()
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 	updateStatus := madmin.ServerUpdateStatus{
 		CurrentVersion: Version,
-		UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
+		UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
 	}
 
 	// Marshal API response
@@ -420,7 +420,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 	for _, nerr := range globalNotificationSys.SignalService(serviceRestart) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
+			adminLogIf(ctx, nerr.Err)
 		}
 	}
 
@@ -451,7 +451,7 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 	case madmin.ServiceActionUnfreeze:
 		serviceSig = serviceUnFreeze
 	default:
-		logger.LogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind)
+		adminLogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
 		return
 	}
@@ -473,7 +473,7 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 	for _, nerr := range globalNotificationSys.SignalService(serviceSig) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
+			adminLogIf(ctx, nerr.Err)
 		}
 	}
 
@@ -534,7 +534,7 @@ func (a adminAPIHandlers) ServiceV2Handler(w http.ResponseWriter, r *http.Reques
 	case madmin.ServiceActionUnfreeze:
 		serviceSig = serviceUnFreeze
 	default:
-		logger.LogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind)
+		adminLogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
 		return
 	}
@@ -1239,7 +1239,7 @@ func extractHealInitParams(vars map[string]string, qParams url.Values, r io.Read
 	if hip.clientToken == "" {
 		jerr := json.NewDecoder(r).Decode(&hip.hs)
 		if jerr != nil {
-			logger.LogIf(GlobalContext, jerr, logger.ErrorKind)
+			adminLogIf(GlobalContext, jerr, logger.ErrorKind)
 			err = ErrRequestBodyParse
 			return
 		}
@@ -1433,7 +1433,7 @@ func getAggregatedBackgroundHealState(ctx context.Context, o ObjectLayer) (madmi
 		var errCount int
 		for _, nerr := range nerrs {
 			if nerr.Err != nil {
-				logger.LogIf(ctx, nerr.Err)
+				adminLogIf(ctx, nerr.Err)
 				errCount++
 			}
 		}
@@ -1561,7 +1561,7 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
 		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 			// would mean the network is not stable. Logging here will help in debugging network issues.
 			if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
-				logger.LogIf(ctx, err)
+				adminLogIf(ctx, err)
 			}
 		}
 		totalRx += n
@@ -2280,8 +2280,6 @@ func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) ma
 	servers := globalNotificationSys.ServerInfo(metrics)
 	servers = append(servers, local)
 
-	assignPoolNumbers(servers)
-
 	var poolsInfo map[int]map[int]madmin.ErasureSetInfo
 	var backend madmin.ErasureBackend
 
@@ -2718,14 +2716,20 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 			for _, server := range infoMessage.Servers {
 				anonEndpoint := anonAddr(server.Endpoint)
 				servers = append(servers, madmin.ServerInfo{
-					State:      server.State,
-					Endpoint:   anonEndpoint,
-					Uptime:     server.Uptime,
-					Version:    server.Version,
-					CommitID:   server.CommitID,
-					Network:    anonymizeNetwork(server.Network),
-					Drives:     anonymizeDrives(server.Disks),
-					PoolNumber: server.PoolNumber,
+					State:    server.State,
+					Endpoint: anonEndpoint,
+					Uptime:   server.Uptime,
+					Version:  server.Version,
+					CommitID: server.CommitID,
+					Network:  anonymizeNetwork(server.Network),
+					Drives:   anonymizeDrives(server.Disks),
+					PoolNumber: func() int {
+						if len(server.PoolNumbers) == 1 {
+							return server.PoolNumbers[0]
+						}
+						return math.MaxInt // this indicates that its unset.
+					}(),
+					PoolNumbers: server.PoolNumbers,
 					MemStats: madmin.MemStats{
 						Alloc:      server.MemStats.Alloc,
 						TotalAlloc: server.MemStats.TotalAlloc,
@@ -2796,7 +2800,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 			w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID))
 		encodedErrorResponse := encodeResponse(errorResponse)
 		healthInfo.Error = string(encodedErrorResponse)
-		logger.LogIf(ctx, enc.Encode(healthInfo))
+		adminLogIf(ctx, enc.Encode(healthInfo))
 	}
 
 	deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics.
@@ -2915,22 +2919,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
-func assignPoolNumbers(servers []madmin.ServerProperties) {
-	for i := range servers {
-		for idx, ge := range globalEndpoints {
-			for _, endpoint := range ge.Endpoints {
-				if servers[i].Endpoint == endpoint.Host {
-					servers[i].PoolNumber = idx + 1
-				} else if host, err := xnet.ParseHost(servers[i].Endpoint); err == nil {
-					if host.Name == endpoint.Hostname() {
-						servers[i].PoolNumber = idx + 1
-					}
-				}
-			}
-		}
-	}
-}
-
 func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
 	lambdaMap := make(map[string][]madmin.TargetIDStatus)
 
@@ -3125,7 +3113,7 @@ func getClusterMetaInfo(ctx context.Context) []byte {
 	case ci := <-resultCh:
 		out, err := json.MarshalIndent(ci, "", "  ")
 		if err != nil {
-			logger.LogIf(ctx, err)
+			bugLogIf(ctx, err)
 			return nil
 		}
 		return out
@@ -3218,18 +3206,18 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 
 		clusterKey, err := bytesToPublicKey(getSubnetAdminPublicKey())
 		if err != nil {
-			logger.LogIf(ctx, stream.AddError(err.Error()))
+			bugLogIf(ctx, stream.AddError(err.Error()))
 			return
 		}
 		err = stream.AddKeyEncrypted(clusterKey)
 		if err != nil {
-			logger.LogIf(ctx, stream.AddError(err.Error()))
+			bugLogIf(ctx, stream.AddError(err.Error()))
 			return
 		}
 		if b := getClusterMetaInfo(ctx); len(b) > 0 {
 			w, err := stream.AddEncryptedStream("cluster.info", nil)
 			if err != nil {
-				logger.LogIf(ctx, err)
+				bugLogIf(ctx, err)
 				return
 			}
 			w.Write(b)
@@ -3238,12 +3226,12 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 
 		// Add new key for inspect data.
 		if err := stream.AddKeyEncrypted(publicKey); err != nil {
-			logger.LogIf(ctx, stream.AddError(err.Error()))
+			bugLogIf(ctx, stream.AddError(err.Error()))
 			return
 		}
 		encStream, err := stream.AddEncryptedStream("inspect.zip", nil)
 		if err != nil {
-			logger.LogIf(ctx, stream.AddError(err.Error()))
+			bugLogIf(ctx, stream.AddError(err.Error()))
 			return
 		}
 		defer encStream.Close()
@@ -3256,7 +3244,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		// MUST use crypto/rand
 		n, err := crand.Read(key[:])
 		if err != nil || n != len(key) {
-			logger.LogIf(ctx, err)
+			bugLogIf(ctx, err)
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
@@ -3270,7 +3258,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 
 		stream, err := sio.AES_256_GCM.Stream(key[:])
 		if err != nil {
-			logger.LogIf(ctx, err)
+			bugLogIf(ctx, err)
 			return
 		}
 		// Zero nonce, we only use each key once, and 32 bytes is plenty.
@@ -3284,7 +3272,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		defer inspectZipW.Close()
 
 		if b := getClusterMetaInfo(ctx); len(b) > 0 {
-			logger.LogIf(ctx, embedFileInZip(inspectZipW, "cluster.info", b, 0o600))
+			adminLogIf(ctx, embedFileInZip(inspectZipW, "cluster.info", b, 0o600))
 		}
 	}
 
@@ -3312,23 +3300,23 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 			sys:     nil,
 		})
 		if zerr != nil {
-			logger.LogIf(ctx, zerr)
+			bugLogIf(ctx, zerr)
 			return nil
 		}
 		header.Method = zip.Deflate
 		zwriter, zerr := inspectZipW.CreateHeader(header)
 		if zerr != nil {
-			logger.LogIf(ctx, zerr)
+			bugLogIf(ctx, zerr)
 			return nil
 		}
 		if _, err := io.Copy(zwriter, r); err != nil {
-			logger.LogIf(ctx, err)
+			adminLogIf(ctx, err)
 		}
 		return nil
 	}
 	err := o.GetRawData(ctx, volume, file, rawDataFn)
 	if !errors.Is(err, errFileNotFound) {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 	}
 
 	// save the format.json as part of inspect by default
@@ -3336,7 +3324,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		err = o.GetRawData(ctx, minioMetaBucket, formatConfigFile, rawDataFn)
 	}
 	if !errors.Is(err, errFileNotFound) {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 	}
 
 	// save args passed to inspect command
@@ -3348,7 +3336,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		sb.WriteString(pool.CmdLine)
 	}
 	sb.WriteString("\n")
-	logger.LogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes(), 0o600))
+	adminLogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes(), 0o600))
 
 	scheme := "https"
 	if !globalIsTLS {
@@ -3382,7 +3370,7 @@ function main() {
 }
 
 main "$@"`, scheme)
-	logger.LogIf(ctx, embedFileInZip(inspectZipW, "start-minio.sh", scrb.Bytes(), 0o755))
+	adminLogIf(ctx, embedFileInZip(inspectZipW, "start-minio.sh", scrb.Bytes(), 0o755))
 }
 
 func getSubnetAdminPublicKey() []byte {

cmd/admin-heal-ops.go

@@ -329,21 +329,25 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence, objAPI ObjectLay
 	// Add heal state and start sequence
 	ahs.healSeqMap[hpath] = h
 
-	// Launch top-level background heal go-routine
-	go h.healSequenceStart(objAPI)
-
 	clientToken := h.clientToken
 	if globalIsDistErasure {
 		clientToken = fmt.Sprintf("%s:%d", h.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
 	}
 
+	if h.clientToken == bgHealingUUID {
+		// For background heal do nothing, do not spawn an unnecessary goroutine.
+	} else {
+		// Launch top-level background heal go-routine
+		go h.healSequenceStart(objAPI)
+	}
+
 	b, err := json.Marshal(madmin.HealStartSuccess{
 		ClientToken:   clientToken,
 		ClientAddress: h.clientAddress,
 		StartTime:     h.startTime,
 	})
 	if err != nil {
-		logger.LogIf(h.ctx, err)
+		bugLogIf(h.ctx, err)
 		return nil, toAdminAPIErr(h.ctx, err), ""
 	}
 	return b, noError, ""
@@ -390,7 +394,7 @@ func (ahs *allHealState) PopHealStatusJSON(hpath string,
 	if err != nil {
 		h.currentStatus.Items = nil
 
-		logger.LogIf(h.ctx, err)
+		bugLogIf(h.ctx, err)
 		return nil, ErrInternalError
 	}
 
@@ -537,9 +541,9 @@ func (h *healSequence) getHealedItemsMap() map[madmin.HealItemType]int64 {
 	return retMap
 }
 
-// gethealFailedItemsMap - returns map of all items where heal failed against
+// getHealFailedItemsMap - returns map of all items where heal failed against
 // drive endpoint and status
-func (h *healSequence) gethealFailedItemsMap() map[string]int64 {
+func (h *healSequence) getHealFailedItemsMap() map[string]int64 {
 	h.mutex.RLock()
 	defer h.mutex.RUnlock()
 
@@ -552,6 +556,32 @@ func (h *healSequence) gethealFailedItemsMap() map[string]int64 {
 	return retMap
 }
 
+func (h *healSequence) countFailed(res madmin.HealResultItem) {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
+
+	for _, d := range res.After.Drives {
+		// For failed items we report the endpoint and drive state
+		// This will help users take corrective actions for drives
+		h.healFailedItemsMap[d.Endpoint+","+d.State]++
+	}
+
+	h.lastHealActivity = UTCNow()
+}
+
+func (h *healSequence) countHeals(healType madmin.HealItemType, healed bool) {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
+
+	if !healed {
+		h.scannedItemsMap[healType]++
+	} else {
+		h.healedItemsMap[healType]++
+	}
+
+	h.lastHealActivity = UTCNow()
+}
+
 // isQuitting - determines if the heal sequence is quitting (due to an
 // external signal)
 func (h *healSequence) isQuitting() bool {
@@ -704,10 +734,7 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		task.opts.ScanMode = madmin.HealNormalScan
 	}
 
-	h.mutex.Lock()
-	h.scannedItemsMap[healType]++
-	h.lastHealActivity = UTCNow()
-	h.mutex.Unlock()
+	h.countHeals(healType, false)
 
 	if source.noWait {
 		select {
@@ -744,32 +771,11 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 				return nil
 			}
 
-			h.mutex.Lock()
-			defer h.mutex.Unlock()
-
-			// Progress is not reported in case of background heal processing.
-			// Instead we increment relevant counter based on the heal result
-			// for prometheus reporting.
-			if res.err != nil {
-				for _, d := range res.result.After.Drives {
-					// For failed items we report the endpoint and drive state
-					// This will help users take corrective actions for drives
-					h.healFailedItemsMap[d.Endpoint+","+d.State]++
-				}
-			} else {
-				// Only object type reported for successful healing
-				h.healedItemsMap[res.result.Type]++
-			}
-
 			// Report caller of any failure
 			return res.err
 		}
 		res.result.Type = healType
 		if res.err != nil {
-			// Only report object error
-			if healType != madmin.HealItemObject {
-				return res.err
-			}
 			res.result.Detail = res.err.Error()
 		}
 		return h.pushHealResultItem(res.result)

cmd/admin-router.go

@@ -60,7 +60,7 @@ const (
 	noObjLayerFlag
 )
 
-// Has checks if the the given flag is enabled in `h`.
+// Has checks if the given flag is enabled in `h`.
 func (h hFlag) Has(flag hFlag) bool {
 	// Use bitwise-AND and check if the result is non-zero.
 	return h&flag != 0

cmd/admin-server-info.go

@@ -19,17 +19,18 @@ package cmd
 
 import (
 	"context"
+	"math"
 	"net/http"
 	"os"
 	"runtime"
 	"runtime/debug"
+	"sort"
 	"strings"
 	"time"
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
 )
 
 // getLocalServerProperty - returns madmin.ServerProperties for only the
@@ -42,9 +43,13 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 	if globalIsDistErasure {
 		addr = globalLocalNodeName
 	}
+	poolNumbers := make(map[int]struct{})
 	network := make(map[string]string)
 	for _, ep := range endpointServerPools {
 		for _, endpoint := range ep.Endpoints {
+			if endpoint.IsLocal {
+				poolNumbers[endpoint.PoolIdx+1] = struct{}{}
+			}
 			nodeName := endpoint.Host
 			if nodeName == "" {
 				nodeName = addr
@@ -61,7 +66,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 				} else {
 					network[nodeName] = string(madmin.ItemOffline)
 					// log once the error
-					logger.LogOnceIf(context.Background(), err, nodeName)
+					peersLogOnceIf(context.Background(), err, nodeName)
 				}
 			}
 		}
@@ -112,6 +117,17 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		MinioEnvVars: make(map[string]string, 10),
 	}
 
+	for poolNumber := range poolNumbers {
+		props.PoolNumbers = append(props.PoolNumbers, poolNumber)
+	}
+	sort.Ints(props.PoolNumbers)
+	props.PoolNumber = func() int {
+		if len(props.PoolNumbers) == 1 {
+			return props.PoolNumbers[0]
+		}
+		return math.MaxInt // this indicates that its unset.
+	}()
+
 	sensitive := map[string]struct{}{
 		config.EnvAccessKey:         {},
 		config.EnvSecretKey:         {},

cmd/api-errors.go

@@ -278,6 +278,7 @@ const (
 	ErrMalformedJSON
 	ErrAdminNoSuchUser
 	ErrAdminNoSuchUserLDAPWarn
+	ErrAdminLDAPExpectedLoginName
 	ErrAdminNoSuchGroup
 	ErrAdminGroupNotEmpty
 	ErrAdminGroupDisabled
@@ -300,6 +301,7 @@ const (
 	ErrAdminConfigIDPCfgNameDoesNotExist
 	ErrInsecureClientRequest
 	ErrObjectTampered
+	ErrAdminLDAPNotEnabled
 
 	// Site-Replication errors
 	ErrSiteReplicationInvalidRequest
@@ -2079,7 +2081,16 @@ var errorCodes = errorCodeMap{
 		Description:    "Invalid attribute name specified.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	// Add your error structure here.
+	ErrAdminLDAPNotEnabled: {
+		Code:           "XMinioLDAPNotEnabled",
+		Description:    "LDAP is not enabled. LDAP must be enabled to make LDAP requests.",
+		HTTPStatusCode: http.StatusNotImplemented,
+	},
+	ErrAdminLDAPExpectedLoginName: {
+		Code:           "XMinioLDAPExpectedLoginName",
+		Description:    "Expected LDAP short username but was given full DN.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 }
 
 // toAPIErrorCode - Converts embedded errors. Convenience
@@ -2519,7 +2530,7 @@ func toAPIError(ctx context.Context, err error) APIError {
 		// Make sure to log the errors which we cannot translate
 		// to a meaningful S3 API errors. This is added to aid in
 		// debugging unexpected/unhandled errors.
-		logger.LogIf(ctx, err)
+		internalLogIf(ctx, err)
 	}
 
 	return apiErr

cmd/api-headers.go

@@ -30,7 +30,6 @@ import (
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 	xxml "github.com/minio/xxml"
 )
 
@@ -50,7 +49,7 @@ func setEventStreamHeaders(w http.ResponseWriter) {
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
 	// Set the "Server" http header.
-	w.Header().Set(xhttp.ServerInfo, "MinIO")
+	w.Header().Set(xhttp.ServerInfo, MinioStoreName)
 
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.
@@ -68,7 +67,7 @@ func encodeResponse(response interface{}) []byte {
 	var buf bytes.Buffer
 	buf.WriteString(xml.Header)
 	if err := xml.NewEncoder(&buf).Encode(response); err != nil {
-		logger.LogIf(GlobalContext, err)
+		bugLogIf(GlobalContext, err)
 		return nil
 	}
 	return buf.Bytes()
@@ -86,7 +85,7 @@ func encodeResponseList(response interface{}) []byte {
 	var buf bytes.Buffer
 	buf.WriteString(xxml.Header)
 	if err := xxml.NewEncoder(&buf).Encode(response); err != nil {
-		logger.LogIf(GlobalContext, err)
+		bugLogIf(GlobalContext, err)
 		return nil
 	}
 	return buf.Bytes()

cmd/api-response.go

@@ -891,7 +891,7 @@ func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType
 	}
 	// Similar check to http.checkWriteHeaderCode
 	if statusCode < 100 || statusCode > 999 {
-		logger.LogIf(context.Background(), fmt.Errorf("invalid WriteHeader code %v", statusCode))
+		bugLogIf(context.Background(), fmt.Errorf("invalid WriteHeader code %v", statusCode))
 		statusCode = http.StatusInternalServerError
 	}
 	setCommonHeaders(w)
@@ -961,7 +961,7 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 
 	// Similar check to http.checkWriteHeaderCode
 	if err.HTTPStatusCode < 100 || err.HTTPStatusCode > 999 {
-		logger.LogIf(ctx, fmt.Errorf("invalid WriteHeader code %v from %v", err.HTTPStatusCode, err.Code))
+		bugLogIf(ctx, fmt.Errorf("invalid WriteHeader code %v from %v", err.HTTPStatusCode, err.Code))
 		err.HTTPStatusCode = http.StatusInternalServerError
 	}
 

cmd/auth-handler.go

@@ -126,7 +126,7 @@ func getRequestAuthType(r *http.Request) (at authType) {
 		var err error
 		r.Form, err = url.ParseQuery(r.URL.RawQuery)
 		if err != nil {
-			logger.LogIf(r.Context(), err)
+			authNLogIf(r.Context(), err)
 			return authTypeUnknown
 		}
 	}
@@ -257,7 +257,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 		if err != nil {
 			// Base64 decoding fails, we should log to indicate
 			// something is malforming the request sent by client.
-			logger.LogIf(GlobalContext, err, logger.ErrorKind)
+			authNLogIf(GlobalContext, err, logger.ErrorKind)
 			return nil, errAuthentication
 		}
 		claims.MapClaims[sessionPolicyNameExtracted] = string(spBytes)
@@ -298,15 +298,15 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 	if cred.IsTemp() && cred.IsExpired() {
 		return nil, toAPIErrorCode(r.Context(), errInvalidAccessKeyID)
 	}
-
 	secret := globalActiveCred.SecretKey
-	var err error
 	if globalSiteReplicationSys.isEnabled() && cred.AccessKey != siteReplicatorSvcAcc {
-		if cred.ParentUser != globalActiveCred.AccessKey {
-			secret, err = getTokenSigningKey()
-			if err != nil {
-				return nil, toAPIErrorCode(r.Context(), err)
-			}
+		nsecret, err := getTokenSigningKey()
+		if err != nil {
+			return nil, toAPIErrorCode(r.Context(), err)
+		}
+		// sign root's temporary accounts also with site replicator creds
+		if cred.ParentUser != globalActiveCred.AccessKey || cred.IsTemp() {
+			secret = nsecret
 		}
 	}
 	if cred.IsServiceAccount() {
@@ -353,7 +353,7 @@ func checkRequestAuthTypeWithVID(ctx context.Context, r *http.Request, action po
 
 func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
 	if logger.GetReqInfo(ctx) == nil {
-		logger.LogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.ErrorKind)
+		bugLogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.ErrorKind)
 		return ErrAccessDenied
 	}
 
@@ -392,7 +392,7 @@ func authenticateRequest(ctx context.Context, r *http.Request, action policy.Act
 		// To extract region from XML in request body, get copy of request body.
 		payload, err := io.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
 		if err != nil {
-			logger.LogIf(ctx, err, logger.ErrorKind)
+			authZLogIf(ctx, err, logger.ErrorKind)
 			return ErrMalformedXML
 		}
 

cmd/background-heal-ops.go

@@ -25,7 +25,6 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go/v3"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
 )
 
@@ -101,16 +100,17 @@ func waitForLowHTTPReq() {
 }
 
 func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
+	bgSeq := newBgHealSequence()
 	// Run the background healer
 	for i := 0; i < globalBackgroundHealRoutine.workers; i++ {
-		go globalBackgroundHealRoutine.AddWorker(ctx, objAPI)
+		go globalBackgroundHealRoutine.AddWorker(ctx, objAPI, bgSeq)
 	}
 
-	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence(), objAPI)
+	globalBackgroundHealState.LaunchNewHealSequence(bgSeq, objAPI)
 }
 
 // Wait for heal requests and process them
-func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer) {
+func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer, bgSeq *healSequence) {
 	for {
 		select {
 		case task, ok := <-h.tasks:
@@ -133,6 +133,15 @@ func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer) {
 				}
 			}
 
+			if bgSeq != nil {
+				// We increment relevant counter based on the heal result for prometheus reporting.
+				if err != nil {
+					bgSeq.countFailed(res)
+				} else {
+					bgSeq.countHeals(res.Type, false)
+				}
+			}
+
 			if task.respCh != nil {
 				task.respCh <- healResult{result: res, err: err}
 			}
@@ -148,7 +157,7 @@ func newHealRoutine() *healRoutine {
 
 	if envHealWorkers := env.Get("_MINIO_HEAL_WORKERS", ""); envHealWorkers != "" {
 		if numHealers, err := strconv.Atoi(envHealWorkers); err != nil {
-			logger.LogIf(context.Background(), fmt.Errorf("invalid _MINIO_HEAL_WORKERS value: %w", err))
+			bugLogIf(context.Background(), fmt.Errorf("invalid _MINIO_HEAL_WORKERS value: %w", err))
 		} else {
 			workers = numHealers
 		}

cmd/background-newdisks-heal-ops.go

@@ -33,7 +33,6 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
 )
 
@@ -409,11 +408,11 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		if errors.Is(err, errFileNotFound) {
 			return nil
 		}
-		logger.LogIf(ctx, fmt.Errorf("Unable to load healing tracker on '%s': %w, re-initializing..", disk, err))
+		healingLogIf(ctx, fmt.Errorf("Unable to load healing tracker on '%s': %w, re-initializing..", disk, err))
 		tracker = initHealingTracker(disk, mustGetUUID())
 	}
 
-	logger.Event(ctx, "Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint)
+	healingLogEvent(ctx, "Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint)
 
 	buckets, _ := z.ListBuckets(ctx, BucketOptions{})
 	// Buckets data are dispersed in multiple pools/sets, make
@@ -452,7 +451,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		return err
 	}
 
-	logger.Event(ctx, "Healing of drive '%s' is finished (healed: %d, skipped: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
+	healingLogEvent(ctx, "Healing of drive '%s' is finished (healed: %d, skipped: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
 
 	if len(tracker.QueuedBuckets) > 0 {
 		return fmt.Errorf("not all buckets were healed: %v", tracker.QueuedBuckets)
@@ -464,7 +463,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	if tracker.HealID == "" { // HealID was empty only before Feb 2023
-		logger.LogIf(ctx, tracker.delete(ctx))
+		bugLogIf(ctx, tracker.delete(ctx))
 		return nil
 	}
 
@@ -482,7 +481,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		t, err := loadHealingTracker(ctx, disk)
 		if err != nil {
 			if !errors.Is(err, errFileNotFound) {
-				logger.LogIf(ctx, err)
+				healingLogIf(ctx, err)
 			}
 			continue
 		}
@@ -517,7 +516,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
 			// Reformat disks immediately
 			_, err := z.HealFormat(context.Background(), false)
 			if err != nil && !errors.Is(err, errNoHealRequired) {
-				logger.LogIf(ctx, err)
+				healingLogIf(ctx, err)
 				// Reset for next interval.
 				diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
 				continue

cmd/batch-expire.go

@@ -33,7 +33,6 @@ import (
 	"github.com/minio/minio/internal/bucket/versioning"
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
 	"github.com/minio/pkg/v2/wildcard"
 	"github.com/minio/pkg/v2/workers"
@@ -156,7 +155,7 @@ func (ef BatchJobExpireFilter) Matches(obj ObjectInfo, now time.Time) bool {
 		}
 	default:
 		// we should never come here, Validate should have caught this.
-		logger.LogOnceIf(context.Background(), fmt.Errorf("invalid filter type: %s", ef.Type), ef.Type)
+		batchLogOnceIf(context.Background(), fmt.Errorf("invalid filter type: %s", ef.Type), ef.Type)
 		return false
 	}
 
@@ -321,7 +320,7 @@ func (r BatchJobExpire) Notify(ctx context.Context, body io.Reader) error {
 		req.Header.Set("Authorization", r.NotificationCfg.Token)
 	}
 
-	clnt := http.Client{Transport: getRemoteInstanceTransport}
+	clnt := http.Client{Transport: getRemoteInstanceTransport()}
 	resp, err := clnt.Do(req)
 	if err != nil {
 		return err
@@ -427,12 +426,13 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
 					default:
 					}
 					stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
-					_, err := api.DeleteObject(ctx, exp.Bucket, exp.Name, ObjectOptions{
-						DeletePrefix: true,
+					_, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{
+						DeletePrefix:       true,
+						DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
 					})
 					if err != nil {
 						stopFn(exp, err)
-						logger.LogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", toExpire[i].Bucket, toExpire[i].Name, toExpire[i].VersionID, err, attempts))
+						batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", toExpire[i].Bucket, toExpire[i].Name, toExpire[i].VersionID, err, attempts))
 					} else {
 						stopFn(exp, err)
 						success = true
@@ -470,7 +470,7 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
 				for i, err := range errs {
 					if err != nil {
 						stopFn(toDelCopy[i], err)
-						logger.LogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID, err, attempts))
+						batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID, err, attempts))
 						failed++
 						if attempts == retryAttempts { // all retry attempts failed, record failure
 							if oi, ok := oiCache.Get(toDelCopy[i]); ok {
@@ -556,16 +556,16 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 			select {
 			case <-saveTicker.C:
 				// persist in-memory state to disk after every 10secs.
-				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 
 			case <-ctx.Done():
 				// persist in-memory state immediately before exiting due to context cancellation.
-				logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+				batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 				return
 
 			case <-saverQuitCh:
 				// persist in-memory state immediately to disk.
-				logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+				batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 				return
 			}
 		}
@@ -669,7 +669,7 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 	// Notify expire jobs final status to the configured endpoint
 	buf, _ := json.Marshal(ri)
 	if err := r.Notify(context.Background(), bytes.NewReader(buf)); err != nil {
-		logger.LogIf(context.Background(), fmt.Errorf("unable to notify %v", err))
+		batchLogIf(context.Background(), fmt.Errorf("unable to notify %v", err))
 	}
 
 	return nil

cmd/batch-handlers.go

@@ -48,7 +48,6 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/ioutil"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 	"github.com/minio/pkg/v2/env"
 	"github.com/minio/pkg/v2/policy"
@@ -63,7 +62,6 @@ type BatchJobRequest struct {
 	ID        string               `yaml:"-" json:"name"`
 	User      string               `yaml:"-" json:"user"`
 	Started   time.Time            `yaml:"-" json:"started"`
-	Location  string               `yaml:"-" json:"location"`
 	Replicate *BatchJobReplicateV1 `yaml:"replicate" json:"replicate"`
 	KeyRotate *BatchJobKeyRotateV1 `yaml:"keyrotate" json:"keyrotate"`
 	Expire    *BatchJobExpire      `yaml:"expire" json:"expire"`
@@ -93,7 +91,7 @@ func notifyEndpoint(ctx context.Context, ri *batchJobInfo, endpoint, token strin
 	}
 	req.Header.Set("Content-Type", "application/json")
 
-	clnt := http.Client{Transport: getRemoteInstanceTransport}
+	clnt := http.Client{Transport: getRemoteInstanceTransport()}
 	resp, err := clnt.Do(req)
 	if err != nil {
 		return err
@@ -207,7 +205,7 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a
 				if aerr == nil {
 					return
 				}
-				logger.LogIf(ctx,
+				batchLogIf(ctx,
 					fmt.Errorf("trying %s: Unable to cleanup failed multipart replication %s on remote %s/%s: %w - this may consume space on remote cluster",
 						humanize.Ordinal(attempts), res.UploadID, tgtBucket, tgtObject, aerr))
 				attempts++
@@ -278,7 +276,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	}
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
-	isTags := len(r.Flags.Filter.Tags) != 0
+	hasTags := len(r.Flags.Filter.Tags) != 0
 	isMetadata := len(r.Flags.Filter.Metadata) != 0
 	isStorageClassOnly := len(r.Flags.Filter.Metadata) == 1 && strings.EqualFold(r.Flags.Filter.Metadata[0].Key, xhttp.AmzStorageClass)
 
@@ -303,7 +301,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			return true
 		}
 
-		if isTags {
+		if hasTags {
 			// Only parse object tags if tags filter is specified.
 			tagMap := map[string]string{}
 			tagStr := oi.UserTags
@@ -352,7 +350,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	c, err := miniogo.New(u.Host, &miniogo.Options{
 		Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
 		Secure:       u.Scheme == "https",
-		Transport:    getRemoteInstanceTransport,
+		Transport:    getRemoteInstanceTransport(),
 		BucketLookup: lookupStyle(r.Source.Path),
 	})
 	if err != nil {
@@ -403,19 +401,19 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 					} else {
 						if !isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) &&
 							!isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) {
-							logger.LogIf(ctx, err)
+							batchLogIf(ctx, err)
 						}
 						continue
 					}
 				}
-				if isTags {
+				if hasTags {
 					tags, err := c.GetObjectTagging(ctx, r.Source.Bucket, obj.Key, minio.GetObjectTaggingOptions{})
 					if err == nil {
 						oi.UserTags = tags.String()
 					} else {
 						if !isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) &&
 							!isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) {
-							logger.LogIf(ctx, err)
+							batchLogIf(ctx, err)
 						}
 						continue
 					}
@@ -444,7 +442,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 						return
 					}
 					stopFn(oi, err)
-					logger.LogIf(ctx, err)
+					batchLogIf(ctx, err)
 					success = false
 				} else {
 					stopFn(oi, nil)
@@ -452,7 +450,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 				ri.trackCurrentBucketObject(r.Target.Bucket, oi, success)
 				globalBatchJobsMetrics.save(job.ID, ri)
 				// persist in-memory state to disk after every 10secs.
-				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 
 				if wait := globalBatchConfig.ReplicationWait(); wait > 0 {
 					time.Sleep(wait)
@@ -467,10 +465,10 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 
 		globalBatchJobsMetrics.save(job.ID, ri)
 		// persist in-memory state to disk.
-		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+		batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 
 		if err := r.Notify(ctx, ri); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
+			batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
 		}
 
 		cancel()
@@ -554,7 +552,7 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
 					VersionID: entry.VersionID,
 				})
 			if err != nil {
-				logger.LogIf(ctx, err)
+				batchLogIf(ctx, err)
 				continue
 			}
 
@@ -573,7 +571,7 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
 
 			opts, err := batchReplicationOpts(ctx, "", gr.ObjInfo)
 			if err != nil {
-				logger.LogIf(ctx, err)
+				batchLogIf(ctx, err)
 				continue
 			}
 
@@ -710,38 +708,52 @@ type batchJobInfo struct {
 }
 
 const (
-	batchReplName      = "batch-replicate.bin"
-	batchReplFormat    = 1
-	batchReplVersionV1 = 1
-	batchReplVersion   = batchReplVersionV1
-	batchJobName       = "job.bin"
-	batchJobPrefix     = "batch-jobs"
+	batchReplName         = "batch-replicate.bin"
+	batchReplFormat       = 1
+	batchReplVersionV1    = 1
+	batchReplVersion      = batchReplVersionV1
+	batchJobName          = "job.bin"
+	batchJobPrefix        = "batch-jobs"
+	batchJobReportsPrefix = batchJobPrefix + "/reports"
 
 	batchReplJobAPIVersion        = "v1"
 	batchReplJobDefaultRetries    = 3
 	batchReplJobDefaultRetryDelay = 250 * time.Millisecond
 )
 
-func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+func getJobReportPath(job BatchJobRequest) string {
 	var fileName string
-	var format, version uint16
 	switch {
 	case job.Replicate != nil:
 		fileName = batchReplName
+	case job.KeyRotate != nil:
+		fileName = batchKeyRotationName
+	case job.Expire != nil:
+		fileName = batchExpireName
+	}
+	return pathJoin(batchJobReportsPrefix, job.ID, fileName)
+}
+
+func getJobPath(job BatchJobRequest) string {
+	return pathJoin(batchJobPrefix, job.ID)
+}
+
+func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
+	var format, version uint16
+	switch {
+	case job.Replicate != nil:
 		version = batchReplVersionV1
 		format = batchReplFormat
 	case job.KeyRotate != nil:
-		fileName = batchKeyRotationName
 		version = batchKeyRotateVersionV1
 		format = batchKeyRotationFormat
 	case job.Expire != nil:
-		fileName = batchExpireName
 		version = batchExpireVersionV1
 		format = batchExpireFormat
 	default:
 		return errors.New("no supported batch job request specified")
 	}
-	data, err := readConfig(ctx, api, pathJoin(job.Location, fileName))
+	data, err := readConfig(ctx, api, getJobReportPath(job))
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) || isErrObjectNotFound(err) {
 			ri.Version = int(version)
@@ -852,8 +864,8 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 	now := UTCNow()
 	ri.mu.Lock()
 	var (
-		format, version  uint16
-		jobTyp, fileName string
+		format, version uint16
+		jobTyp          string
 	)
 
 	if now.Sub(ri.LastUpdate) >= duration {
@@ -862,19 +874,16 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 			format = batchReplFormat
 			version = batchReplVersion
 			jobTyp = string(job.Type())
-			fileName = batchReplName
 			ri.Version = batchReplVersionV1
 		case madmin.BatchJobKeyRotate:
 			format = batchKeyRotationFormat
 			version = batchKeyRotateVersion
 			jobTyp = string(job.Type())
-			fileName = batchKeyRotationName
 			ri.Version = batchKeyRotateVersionV1
 		case madmin.BatchJobExpire:
 			format = batchExpireFormat
 			version = batchExpireVersion
 			jobTyp = string(job.Type())
-			fileName = batchExpireName
 			ri.Version = batchExpireVersionV1
 		default:
 			return errInvalidArgument
@@ -895,7 +904,7 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 		if err != nil {
 			return err
 		}
-		return saveConfig(ctx, api, pathJoin(job.Location, fileName), buf)
+		return saveConfig(ctx, api, getJobReportPath(job), buf)
 	}
 	ri.mu.Unlock()
 	return nil
@@ -1038,7 +1047,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	c, err := miniogo.NewCore(u.Host, &miniogo.Options{
 		Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
 		Secure:       u.Scheme == "https",
-		Transport:    getRemoteInstanceTransport,
+		Transport:    getRemoteInstanceTransport(),
 		BucketLookup: lookupStyle(r.Target.Path),
 	})
 	if err != nil {
@@ -1054,36 +1063,38 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 	if !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
 		go func() {
-			defer xioutil.SafeClose(slowCh)
-
 			// Snowball currently needs the high level minio-go Client, not the Core one
 			cl, err := miniogo.New(u.Host, &miniogo.Options{
 				Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
 				Secure:       u.Scheme == "https",
-				Transport:    getRemoteInstanceTransport,
+				Transport:    getRemoteInstanceTransport(),
 				BucketLookup: lookupStyle(r.Target.Path),
 			})
 			if err != nil {
-				logger.LogIf(ctx, err)
+				batchLogIf(ctx, err)
 				return
 			}
 
 			// Already validated before arriving here
 			smallerThan, _ := humanize.ParseBytes(*r.Source.Snowball.SmallerThan)
 
-			var (
-				obj   = ObjectInfo{}
-				batch = make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
-				valid = true
-			)
-
-			for valid {
-				obj, valid = <-walkCh
-
-				if !valid {
-					goto write
+			batch := make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
+			writeFn := func(batch []ObjectInfo) {
+				if len(batch) > 0 {
+					if err := r.writeAsArchive(ctx, api, cl, batch); err != nil {
+						batchLogIf(ctx, err)
+						for _, b := range batch {
+							slowCh <- b
+						}
+					} else {
+						ri.trackCurrentBucketBatch(r.Source.Bucket, batch)
+						globalBatchJobsMetrics.save(job.ID, ri)
+						// persist in-memory state to disk after every 10secs.
+						batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+					}
 				}
-
+			}
+			for obj := range walkCh {
 				if obj.DeleteMarker || !obj.VersionPurgeStatus.Empty() || obj.Size >= int64(smallerThan) {
 					slowCh <- obj
 					continue
@@ -1094,23 +1105,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				if len(batch) < *r.Source.Snowball.Batch {
 					continue
 				}
-
-			write:
-				if len(batch) > 0 {
-					if err := r.writeAsArchive(ctx, api, cl, batch); err != nil {
-						logger.LogIf(ctx, err)
-						for _, b := range batch {
-							slowCh <- b
-						}
-					} else {
-						ri.trackCurrentBucketBatch(r.Source.Bucket, batch)
-						globalBatchJobsMetrics.save(job.ID, ri)
-						// persist in-memory state to disk after every 10secs.
-						logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
-					}
-					batch = batch[:0]
-				}
+				writeFn(batch)
+				batch = batch[:0]
 			}
+			writeFn(batch)
+			xioutil.SafeClose(slowCh)
 		}()
 	} else {
 		slowCh = walkCh
@@ -1179,7 +1178,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 						return
 					}
 					stopFn(result, err)
-					logger.LogIf(ctx, err)
+					batchLogIf(ctx, err)
 					success = false
 				} else {
 					stopFn(result, nil)
@@ -1187,7 +1186,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				ri.trackCurrentBucketObject(r.Source.Bucket, result, success)
 				globalBatchJobsMetrics.save(job.ID, ri)
 				// persist in-memory state to disk after every 10secs.
-				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 
 				if wait := globalBatchConfig.ReplicationWait(); wait > 0 {
 					time.Sleep(wait)
@@ -1202,10 +1201,10 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 		globalBatchJobsMetrics.save(job.ID, ri)
 		// persist in-memory state to disk.
-		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+		batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 
 		if err := r.Notify(ctx, ri); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
+			batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
 		}
 
 		cancel()
@@ -1354,7 +1353,7 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 	c, err := miniogo.NewCore(u.Host, &miniogo.Options{
 		Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
 		Secure:       u.Scheme == "https",
-		Transport:    getRemoteInstanceTransport,
+		Transport:    getRemoteInstanceTransport(),
 		BucketLookup: lookupStyle(pathStyle),
 	})
 	if err != nil {
@@ -1417,15 +1416,8 @@ func (j BatchJobRequest) Validate(ctx context.Context, o ObjectLayer) error {
 }
 
 func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) {
-	switch {
-	case j.Replicate != nil:
-		deleteConfig(ctx, api, pathJoin(j.Location, batchReplName))
-	case j.KeyRotate != nil:
-		deleteConfig(ctx, api, pathJoin(j.Location, batchKeyRotationName))
-	case j.Expire != nil:
-		deleteConfig(ctx, api, pathJoin(j.Location, batchExpireName))
-	}
-	deleteConfig(ctx, api, j.Location)
+	deleteConfig(ctx, api, getJobReportPath(j))
+	deleteConfig(ctx, api, getJobPath(j))
 }
 
 func (j *BatchJobRequest) save(ctx context.Context, api ObjectLayer) error {
@@ -1437,13 +1429,12 @@ func (j *BatchJobRequest) save(ctx context.Context, api ObjectLayer) error {
 		return err
 	}
 
-	j.Location = pathJoin(batchJobPrefix, j.ID)
 	job, err := j.MarshalMsg(nil)
 	if err != nil {
 		return err
 	}
 
-	return saveConfig(ctx, api, j.Location, job)
+	return saveConfig(ctx, api, getJobPath(*j), job)
 }
 
 func (j *BatchJobRequest) load(ctx context.Context, api ObjectLayer, name string) error {
@@ -1508,7 +1499,7 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
 		req := &BatchJobRequest{}
 		if err := req.load(ctx, objectAPI, result.Name); err != nil {
 			if !errors.Is(err, errNoSuchJob) {
-				logger.LogIf(ctx, err)
+				batchLogIf(ctx, err)
 			}
 			continue
 		}
@@ -1524,7 +1515,7 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
 		}
 	}
 
-	logger.LogIf(ctx, json.NewEncoder(w).Encode(&listResult))
+	batchLogIf(ctx, json.NewEncoder(w).Encode(&listResult))
 }
 
 var errNoSuchJob = errors.New("no such job")
@@ -1547,7 +1538,7 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
 	req := &BatchJobRequest{}
 	if err := req.load(ctx, objectAPI, pathJoin(batchJobPrefix, jobID)); err != nil {
 		if !errors.Is(err, errNoSuchJob) {
-			logger.LogIf(ctx, err)
+			batchLogIf(ctx, err)
 		}
 
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1556,7 +1547,7 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
 
 	buf, err := yaml.Marshal(req)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		batchLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1671,8 +1662,7 @@ func (a adminAPIHandlers) CancelBatchJob(w http.ResponseWriter, r *http.Request)
 	}
 
 	j := BatchJobRequest{
-		ID:       jobID,
-		Location: pathJoin(batchJobPrefix, jobID),
+		ID: jobID,
 	}
 
 	j.delete(ctx, objectAPI)
@@ -1716,7 +1706,7 @@ func (j *BatchJobPool) resume() {
 	ctx, cancel := context.WithCancel(j.ctx)
 	defer cancel()
 	if err := j.objLayer.Walk(ctx, minioMetaBucket, batchJobPrefix, results, WalkOptions{}); err != nil {
-		logger.LogIf(j.ctx, err)
+		batchLogIf(j.ctx, err)
 		return
 	}
 	for result := range results {
@@ -1726,7 +1716,7 @@ func (j *BatchJobPool) resume() {
 		}
 		req := &BatchJobRequest{}
 		if err := req.load(ctx, j.objLayer, result.Name); err != nil {
-			logger.LogIf(ctx, err)
+			batchLogIf(ctx, err)
 			continue
 		}
 		_, nodeIdx := parseRequestToken(req.ID)
@@ -1735,7 +1725,7 @@ func (j *BatchJobPool) resume() {
 			continue
 		}
 		if err := j.queueJob(req); err != nil {
-			logger.LogIf(ctx, err)
+			batchLogIf(ctx, err)
 			continue
 		}
 	}
@@ -1759,7 +1749,7 @@ func (j *BatchJobPool) AddWorker() {
 				if job.Replicate.RemoteToLocal() {
 					if err := job.Replicate.StartFromSource(job.ctx, j.objLayer, *job); err != nil {
 						if !isErrBucketNotFound(err) {
-							logger.LogIf(j.ctx, err)
+							batchLogIf(j.ctx, err)
 							j.canceler(job.ID, false)
 							continue
 						}
@@ -1768,7 +1758,7 @@ func (j *BatchJobPool) AddWorker() {
 				} else {
 					if err := job.Replicate.Start(job.ctx, j.objLayer, *job); err != nil {
 						if !isErrBucketNotFound(err) {
-							logger.LogIf(j.ctx, err)
+							batchLogIf(j.ctx, err)
 							j.canceler(job.ID, false)
 							continue
 						}
@@ -1778,14 +1768,14 @@ func (j *BatchJobPool) AddWorker() {
 			case job.KeyRotate != nil:
 				if err := job.KeyRotate.Start(job.ctx, j.objLayer, *job); err != nil {
 					if !isErrBucketNotFound(err) {
-						logger.LogIf(j.ctx, err)
+						batchLogIf(j.ctx, err)
 						continue
 					}
 				}
 			case job.Expire != nil:
 				if err := job.Expire.Start(job.ctx, j.objLayer, *job); err != nil {
 					if !isErrBucketNotFound(err) {
-						logger.LogIf(j.ctx, err)
+						batchLogIf(j.ctx, err)
 						continue
 					}
 				}

cmd/batch-handlers_gen.go

@@ -42,12 +42,6 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Started")
 				return
 			}
-		case "Location":
-			z.Location, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "Location")
-				return
-			}
 		case "Replicate":
 			if dc.IsNil() {
 				err = dc.ReadNil()
@@ -115,9 +109,9 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 7
+	// map header, size 6
 	// write "ID"
-	err = en.Append(0x87, 0xa2, 0x49, 0x44)
+	err = en.Append(0x86, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -146,16 +140,6 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Started")
 		return
 	}
-	// write "Location"
-	err = en.Append(0xa8, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Location)
-	if err != nil {
-		err = msgp.WrapError(err, "Location")
-		return
-	}
 	// write "Replicate"
 	err = en.Append(0xa9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65)
 	if err != nil {
@@ -213,9 +197,9 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 7
+	// map header, size 6
 	// string "ID"
-	o = append(o, 0x87, 0xa2, 0x49, 0x44)
+	o = append(o, 0x86, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "User"
 	o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x72)
@@ -223,9 +207,6 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Started"
 	o = append(o, 0xa7, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64)
 	o = msgp.AppendTime(o, z.Started)
-	// string "Location"
-	o = append(o, 0xa8, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e)
-	o = msgp.AppendString(o, z.Location)
 	// string "Replicate"
 	o = append(o, 0xa9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65)
 	if z.Replicate == nil {
@@ -298,12 +279,6 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Started")
 				return
 			}
-		case "Location":
-			z.Location, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Location")
-				return
-			}
 		case "Replicate":
 			if msgp.IsNil(bts) {
 				bts, err = msgp.ReadNilBytes(bts)
@@ -369,7 +344,7 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobRequest) Msgsize() (s int) {
-	s = 1 + 3 + msgp.StringPrefixSize + len(z.ID) + 5 + msgp.StringPrefixSize + len(z.User) + 8 + msgp.TimeSize + 9 + msgp.StringPrefixSize + len(z.Location) + 10
+	s = 1 + 3 + msgp.StringPrefixSize + len(z.ID) + 5 + msgp.StringPrefixSize + len(z.User) + 8 + msgp.TimeSize + 10
 	if z.Replicate == nil {
 		s += msgp.NilSize
 	} else {

cmd/batch-rotate.go

@@ -33,7 +33,6 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
 	"github.com/minio/pkg/v2/workers"
 )
@@ -274,7 +273,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
-	skip := func(info FileInfo) (ok bool) {
+	selectObj := func(info FileInfo) (ok bool) {
 		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
 			// skip all objects that are newer than specified older duration
 			return false
@@ -360,7 +359,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	results := make(chan ObjectInfo, 100)
 	if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{
 		Marker: lastObject,
-		Filter: skip,
+		Filter: selectObj,
 	}); err != nil {
 		cancel()
 		// Do not need to retry if we can't list objects on source.
@@ -383,7 +382,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				success := true
 				if err := r.KeyRotate(ctx, api, result); err != nil {
 					stopFn(result, err)
-					logger.LogIf(ctx, err)
+					batchLogIf(ctx, err)
 					success = false
 				} else {
 					stopFn(result, nil)
@@ -392,7 +391,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				ri.RetryAttempts = attempts
 				globalBatchJobsMetrics.save(job.ID, ri)
 				// persist in-memory state to disk after every 10secs.
-				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 				if success {
 					break
 				}
@@ -412,10 +411,10 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	ri.Failed = ri.ObjectsFailed > 0
 	globalBatchJobsMetrics.save(job.ID, ri)
 	// persist in-memory state to disk.
-	logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+	batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 
 	if err := r.Notify(ctx, ri); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
+		batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
 	}
 
 	cancel()

cmd/bootstrap-peer-server.go

@@ -198,7 +198,7 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 				if err != nil {
 					bootstrapTraceMsg(fmt.Sprintf("clnt.Verify: %v, endpoint: %s", err, clnt))
 					if !isNetworkError(err) {
-						logger.LogOnceIf(context.Background(), fmt.Errorf("%s has incorrect configuration: %w", clnt, err), "incorrect_"+clnt.String())
+						bootLogOnceIf(context.Background(), fmt.Errorf("%s has incorrect configuration: %w", clnt, err), "incorrect_"+clnt.String())
 						incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt, err))
 					} else {
 						offlineEndpoints = append(offlineEndpoints, fmt.Errorf("%s is unreachable: %w", clnt, err))

cmd/bucket-encryption-handlers.go

@@ -114,7 +114,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeSSEConfig,
 		Bucket:    bucket,
 		SSEConfig: &cfgStr,
@@ -203,7 +203,7 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 	}
 
 	// Call site replication hook.
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeSSEConfig,
 		Bucket:    bucket,
 		SSEConfig: nil,

cmd/bucket-handlers.go

@@ -98,7 +98,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	// Get buckets in the DNS
 	dnsBuckets, err := globalDNSConfig.List()
 	if err != nil && !IsErrIgnored(err, dns.ErrNoEntriesFound, dns.ErrNotImplemented, dns.ErrDomainMissing) {
-		logger.LogIf(GlobalContext, err)
+		dnsLogIf(GlobalContext, err)
 		return
 	}
 
@@ -160,13 +160,13 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	ctx := GlobalContext
 	for _, err := range g.Wait() {
 		if err != nil {
-			logger.LogIf(ctx, err)
+			dnsLogIf(ctx, err)
 			return
 		}
 	}
 
 	for _, bucket := range bucketsInConflict.ToSlice() {
-		logger.LogIf(ctx, fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket by a different tenant. This local bucket will be ignored. Bucket names are globally unique in federated deployments. Use path style requests on following addresses '%v' to access this bucket", bucket, globalDomainIPs.ToSlice()))
+		dnsLogIf(ctx, fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket by a different tenant. This local bucket will be ignored. Bucket names are globally unique in federated deployments. Use path style requests on following addresses '%v' to access this bucket", bucket, globalDomainIPs.ToSlice()))
 	}
 
 	var wg sync.WaitGroup
@@ -187,7 +187,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 			// We go to here, so we know the bucket no longer exists,
 			// but is registered in DNS to this server
 			if err := globalDNSConfig.Delete(bucket); err != nil {
-				logger.LogIf(GlobalContext, fmt.Errorf("Failed to remove DNS entry for %s due to %w",
+				dnsLogIf(GlobalContext, fmt.Errorf("Failed to remove DNS entry for %s due to %w",
 					bucket, err))
 			}
 		}(bucket)
@@ -790,7 +790,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 
 	// check if client is attempting to create more buckets, complain about it.
 	if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
-		logger.LogIf(ctx, fmt.Errorf("Please avoid creating more buckets %d beyond recommended %d", currBuckets+1, maxBuckets))
+		internalLogIf(ctx, fmt.Errorf("Please avoid creating more buckets %d beyond recommended %d", currBuckets+1, maxBuckets), logger.WarningKind)
 	}
 
 	opts := MakeBucketOptions{
@@ -871,7 +871,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 
 	// Call site replication hook
-	logger.LogIf(ctx, globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts))
+	replLogIf(ctx, globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts))
 
 	// Make sure to add Location information here only for bucket
 	w.Header().Set(xhttp.Location, pathJoin(SlashSeparator, bucket))
@@ -1218,11 +1218,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			return
 		}
 
-		if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) {
-			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL)
-			return
-		}
-
 		var (
 			reader io.Reader
 			keyID  string
@@ -1698,7 +1693,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	if globalDNSConfig != nil {
 		if err := globalDNSConfig.Delete(bucket); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually, bucket on MinIO no longer exists", err))
+			dnsLogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually, bucket on MinIO no longer exists", err))
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
@@ -1708,7 +1703,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	globalReplicationPool.deleteResyncMetadata(ctx, bucket)
 
 	// Call site replication hook.
-	logger.LogIf(ctx, globalSiteReplicationSys.DeleteBucketHook(ctx, bucket, forceDelete))
+	replLogIf(ctx, globalSiteReplicationSys.DeleteBucketHook(ctx, bucket, forceDelete))
 
 	// Write success response.
 	writeSuccessNoContent(w)
@@ -1781,7 +1776,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:             madmin.SRBucketMetaTypeObjectLockConfig,
 		Bucket:           bucket,
 		ObjectLockConfig: &cfgStr,
@@ -1885,7 +1880,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeTags,
 		Bucket:    bucket,
 		Tags:      &cfgStr,
@@ -1961,7 +1956,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeTags,
 		Bucket:    bucket,
 		UpdatedAt: updatedAt,

cmd/bucket-lifecycle.go

@@ -233,6 +233,10 @@ func (es *expiryState) enqueueByDays(oi ObjectInfo, event lifecycle.Event, src l
 // enqueueByNewerNoncurrent enqueues object versions expired by
 // NewerNoncurrentVersions limit for expiry.
 func (es *expiryState) enqueueByNewerNoncurrent(bucket string, versions []ObjectToDelete, lcEvent lifecycle.Event) {
+	if len(versions) == 0 {
+		return
+	}
+
 	task := newerNoncurrentTask{bucket: bucket, versions: versions, event: lcEvent}
 	wrkr := es.getWorkerCh(task.OpHash())
 	if wrkr == nil {
@@ -332,7 +336,7 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
 			case newerNoncurrentTask:
 				deleteObjectVersions(es.ctx, es.objAPI, v.bucket, v.versions, v.event)
 			case jentry:
-				logger.LogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName))
+				transitionLogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName))
 			case freeVersionTask:
 				oi := v.ObjectInfo
 				traceFn := globalLifecycleSys.trace(oi)
@@ -351,7 +355,7 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
 				// Remove the remote object
 				err := deleteObjectFromRemoteTier(es.ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
 				if ignoreNotFoundErr(err) != nil {
-					logger.LogIf(es.ctx, err)
+					transitionLogIf(es.ctx, err)
 					return
 				}
 
@@ -364,10 +368,10 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
 					auditLogLifecycle(es.ctx, oi, ILMFreeVersionDelete, nil, traceFn)
 				}
 				if ignoreNotFoundErr(err) != nil {
-					logger.LogIf(es.ctx, err)
+					transitionLogIf(es.ctx, err)
 				}
 			default:
-				logger.LogIf(es.ctx, fmt.Errorf("Invalid work type - %v", v))
+				bugLogIf(es.ctx, fmt.Errorf("Invalid work type - %v", v))
 			}
 		}
 	}
@@ -482,7 +486,7 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 			if err := transitionObject(t.ctx, objectAPI, task.objInfo, newLifecycleAuditEvent(task.src, task.event)); err != nil {
 				if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) && !xnet.IsNetworkOrHostDown(err, false) {
 					if !strings.Contains(err.Error(), "use of closed network connection") {
-						logger.LogIf(t.ctx, fmt.Errorf("Transition to %s failed for %s/%s version:%s with %w",
+						transitionLogIf(t.ctx, fmt.Errorf("Transition to %s failed for %s/%s version:%s with %w",
 							task.event.StorageClass, task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
 					}
 				}
@@ -610,7 +614,7 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 		// remote object
 		opts.SkipFreeVersion = true
 	} else {
-		logger.LogIf(ctx, err)
+		transitionLogIf(ctx, err)
 	}
 
 	// Now, delete object from hot-tier namespace
@@ -875,7 +879,7 @@ func postRestoreOpts(ctx context.Context, r *http.Request, bucket, object string
 	if vid != "" && vid != nullVersionID {
 		_, err := uuid.Parse(vid)
 		if err != nil {
-			logger.LogIf(ctx, err)
+			s3LogIf(ctx, err)
 			return opts, InvalidVersionID{
 				Bucket:    bucket,
 				Object:    object,

cmd/bucket-metadata-sys.go

@@ -500,7 +500,7 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 	errs := g.Wait()
 	for _, err := range errs {
 		if err != nil {
-			logger.LogIf(ctx, err)
+			internalLogIf(ctx, err, logger.WarningKind)
 		}
 	}
 
@@ -542,7 +542,7 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, fa
 		case <-t.C:
 			buckets, err := sys.objAPI.ListBuckets(ctx, BucketOptions{})
 			if err != nil {
-				logger.LogIf(ctx, err)
+				internalLogIf(ctx, err, logger.WarningKind)
 				break
 			}
 
@@ -560,7 +560,7 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, fa
 
 				meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[i].Name)
 				if err != nil {
-					logger.LogIf(ctx, err)
+					internalLogIf(ctx, err, logger.WarningKind)
 					wait() // wait to proceed to next entry.
 					continue
 				}

cmd/bucket-metadata.go

@@ -145,7 +145,7 @@ func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
 // If an error is returned the returned metadata will be default initialized.
 func readBucketMetadata(ctx context.Context, api ObjectLayer, name string) (BucketMetadata, error) {
 	if name == "" {
-		logger.LogIf(ctx, errors.New("bucket name cannot be empty"))
+		internalLogIf(ctx, errors.New("bucket name cannot be empty"), logger.WarningKind)
 		return BucketMetadata{}, errInvalidArgument
 	}
 	b := newBucketMetadata(name)
@@ -400,7 +400,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 	for legacyFile := range configs {
 		configFile := path.Join(bucketMetaPrefix, b.Name, legacyFile)
 		if err := deleteConfig(ctx, objectAPI, configFile); err != nil && !errors.Is(err, errConfigNotFound) {
-			logger.LogIf(ctx, err)
+			internalLogIf(ctx, err, logger.WarningKind)
 		}
 	}
 

cmd/bucket-object-lock.go

@@ -66,7 +66,7 @@ func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locke
 	if ret.Mode.Valid() && (ret.Mode == objectlock.RetCompliance || ret.Mode == objectlock.RetGovernance) {
 		t, err := objectlock.UTCNowNTP()
 		if err != nil {
-			logger.LogIf(ctx, err)
+			internalLogIf(ctx, err, logger.WarningKind)
 			return true
 		}
 		if ret.RetainUntilDate.After(t) {
@@ -114,7 +114,7 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 			// duration of the retention period.
 			t, err := objectlock.UTCNowNTP()
 			if err != nil {
-				logger.LogIf(ctx, err)
+				internalLogIf(ctx, err, logger.WarningKind)
 				return ObjectLocked{}
 			}
 
@@ -140,7 +140,7 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 			if !byPassSet {
 				t, err := objectlock.UTCNowNTP()
 				if err != nil {
-					logger.LogIf(ctx, err)
+					internalLogIf(ctx, err, logger.WarningKind)
 					return ObjectLocked{}
 				}
 
@@ -170,7 +170,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
 
 	t, err := objectlock.UTCNowNTP()
 	if err != nil {
-		logger.LogIf(ctx, err)
+		internalLogIf(ctx, err, logger.WarningKind)
 		return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
 	}
 
@@ -277,7 +277,7 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
 			r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
 			t, err := objectlock.UTCNowNTP()
 			if err != nil {
-				logger.LogIf(ctx, err)
+				internalLogIf(ctx, err, logger.WarningKind)
 				return mode, retainDate, legalHold, ErrObjectLocked
 			}
 			if r.Mode == objectlock.RetCompliance && r.RetainUntilDate.After(t) {
@@ -324,7 +324,7 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
 
 		t, err := objectlock.UTCNowNTP()
 		if err != nil {
-			logger.LogIf(ctx, err)
+			internalLogIf(ctx, err, logger.WarningKind)
 			return mode, retainDate, legalHold, ErrObjectLocked
 		}
 

cmd/bucket-policy-handlers.go

@@ -113,7 +113,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Call site replication hook.
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypePolicy,
 		Bucket:    bucket,
 		Policy:    bucketPolicyBytes,
@@ -157,7 +157,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	}
 
 	// Call site replication hook.
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypePolicy,
 		Bucket:    bucket,
 		UpdatedAt: updatedAt,

cmd/bucket-policy.go

@@ -53,7 +53,7 @@ func (sys *PolicySys) IsAllowed(args policy.BucketPolicyArgs) bool {
 
 	// Log unhandled errors.
 	if _, ok := err.(BucketPolicyNotFound); !ok {
-		logger.LogIf(GlobalContext, err)
+		internalLogIf(GlobalContext, err, logger.WarningKind)
 	}
 
 	// As policy is not available for given bucket name, returns IsOwner i.e.

cmd/bucket-quota.go

@@ -64,9 +64,9 @@ func (sys *BucketQuotaSys) GetBucketUsageInfo(bucket string) (BucketUsageInfo, e
 	timedout := OperationTimedOut{}
 	if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &timedout) {
 		if len(dui.BucketsUsage) > 0 {
-			logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err), "bucket-usage-cache-"+bucket)
+			internalLogOnceIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err), "bucket-usage-cache-"+bucket)
 		} else {
-			logger.LogOnceIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"), "bucket-usage-empty-"+bucket)
+			internalLogOnceIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"), "bucket-usage-empty-"+bucket)
 		}
 	}
 
@@ -87,7 +87,7 @@ func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota,
 	}
 	if !quotaCfg.IsValid() {
 		if quotaCfg.Type == "fifo" {
-			logger.LogIf(GlobalContext, errors.New("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore. Please clear your quota configs using 'mc admin bucket quota alias/bucket --clear' and use 'mc ilm add' for expiration of objects"))
+			internalLogIf(GlobalContext, errors.New("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore. Please clear your quota configs using 'mc admin bucket quota alias/bucket --clear' and use 'mc ilm add' for expiration of objects"), logger.WarningKind)
 			return quotaCfg, fmt.Errorf("invalid quota type 'fifo'")
 		}
 		return quotaCfg, fmt.Errorf("Invalid quota config %#v", quotaCfg)

cmd/bucket-replication-metrics_gen.go

@@ -584,6 +584,7 @@ func (z *InQueueStats) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z InQueueStats) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 0
+	_ = z
 	err = en.Append(0x80)
 	if err != nil {
 		return
@@ -595,6 +596,7 @@ func (z InQueueStats) EncodeMsg(en *msgp.Writer) (err error) {
 func (z InQueueStats) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 0
+	_ = z
 	o = append(o, 0x80)
 	return
 }

cmd/bucket-replication-utils_gen.go

@@ -749,6 +749,7 @@ func (z *ReplicateDecision) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z ReplicateDecision) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 0
+	_ = z
 	err = en.Append(0x80)
 	if err != nil {
 		return
@@ -760,6 +761,7 @@ func (z ReplicateDecision) EncodeMsg(en *msgp.Writer) (err error) {
 func (z ReplicateDecision) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 0
+	_ = z
 	o = append(o, 0x80)
 	return
 }
@@ -1388,6 +1390,7 @@ func (z *ResyncDecision) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z ResyncDecision) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 0
+	_ = z
 	err = en.Append(0x80)
 	if err != nil {
 		return
@@ -1399,6 +1402,7 @@ func (z ResyncDecision) EncodeMsg(en *msgp.Writer) (err error) {
 func (z ResyncDecision) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 0
+	_ = z
 	o = append(o, 0x80)
 	return
 }

cmd/bucket-replication.go

@@ -51,6 +51,8 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/tinylib/msgp/msgp"
 	"github.com/zeebo/xxh3"
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
 )
 
 const (
@@ -76,11 +78,6 @@ const (
 	ReplicationWorkerMultiplier = 1.5
 )
 
-func isReplicationEnabled(ctx context.Context, bucketName string) bool {
-	rc, _ := getReplicationConfig(ctx, bucketName)
-	return rc != nil
-}
-
 // gets replication config associated to a given bucket name.
 func getReplicationConfig(ctx context.Context, bucketName string) (rc *replication.Config, err error) {
 	rCfg, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucketName)
@@ -426,7 +423,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 
 	rcfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil || rcfg == nil {
-		logger.LogOnceIf(ctx, fmt.Errorf("unable to obtain replication config for bucket: %s: err: %s", bucket, err), bucket)
+		replLogOnceIf(ctx, fmt.Errorf("unable to obtain replication config for bucket: %s: err: %s", bucket, err), bucket)
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object: ObjectInfo{
@@ -443,7 +440,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	}
 	dsc, err := parseReplicateDecision(ctx, bucket, dobj.ReplicationState.ReplicateDecisionStr)
 	if err != nil {
-		logger.LogOnceIf(ctx, fmt.Errorf("unable to parse replication decision parameters for bucket: %s, err: %s, decision: %s",
+		replLogOnceIf(ctx, fmt.Errorf("unable to parse replication decision parameters for bucket: %s, err: %s, decision: %s",
 			bucket, err, dobj.ReplicationState.ReplicateDecisionStr), dobj.ReplicationState.ReplicateDecisionStr)
 		sendEvent(eventArgs{
 			BucketName: bucket,
@@ -497,7 +494,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		tgtClnt := globalBucketTargetSys.GetRemoteTargetClient(bucket, tgtEntry.Arn)
 		if tgtClnt == nil {
 			// Skip stale targets if any and log them to be missing at least once.
-			logger.LogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtEntry.Arn), tgtEntry.Arn)
+			replLogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtEntry.Arn), tgtEntry.Arn)
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
@@ -609,7 +606,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		return
 	}
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete-"+tgt.ARN)
+		replLogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete-"+tgt.ARN)
 		sendEvent(eventArgs{
 			BucketName: dobj.Bucket,
 			Object: ObjectInfo{
@@ -684,7 +681,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		} else {
 			rinfo.VersionPurgeStatus = Failed
 		}
-		logger.LogIf(ctx, fmt.Errorf("unable to replicate delete marker to %s: %s/%s(%s): %w", tgt.EndpointURL(), tgt.Bucket, dobj.ObjectName, versionID, rmErr))
+		replLogIf(ctx, fmt.Errorf("unable to replicate delete marker to %s: %s/%s(%s): %w", tgt.EndpointURL(), tgt.Bucket, dobj.ObjectName, versionID, rmErr))
 		if rmErr != nil && minio.IsNetworkOrHostDown(rmErr, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 			globalBucketTargetSys.markOffline(tgt.EndpointURL())
 		}
@@ -764,13 +761,20 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
 func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, err error) {
 	meta := make(map[string]string)
 	for k, v := range objInfo.UserDefined {
-		if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
-			continue
+		// In case of SSE-C objects copy the allowed internal headers as well
+		if !crypto.SSEC.IsEncrypted(objInfo.UserDefined) || !slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
+			if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
+				continue
+			}
+			if isStandardHeader(k) {
+				continue
+			}
 		}
-		if isStandardHeader(k) {
-			continue
+		if slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
+			meta[validSSEReplicationHeaders[k]] = v
+		} else {
+			meta[k] = v
 		}
-		meta[k] = v
 	}
 
 	if sc == "" && (objInfo.StorageClass == storageclass.STANDARD || objInfo.StorageClass == storageclass.RRS) {
@@ -990,7 +994,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
-		logger.LogOnceIf(ctx, err, "get-replication-config-"+bucket)
+		replLogOnceIf(ctx, err, "get-replication-config-"+bucket)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1029,7 +1033,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 	for _, tgtArn := range tgtArns {
 		tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, tgtArn)
 		if tgt == nil {
-			logger.LogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtArn), tgtArn)
+			replLogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtArn), tgtArn)
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
@@ -1151,7 +1155,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	}
 
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s retry:%d", bucket, tgt.ARN, ri.RetryCount), "replication-target-offline"+tgt.ARN)
+		replLogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s retry:%d", bucket, tgt.ARN, ri.RetryCount), "replication-target-offline"+tgt.ARN)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1166,9 +1170,10 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
 
 	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{
-		VersionID:        ri.VersionID,
-		Versioned:        versioned,
-		VersionSuspended: versionSuspended,
+		VersionID:          ri.VersionID,
+		Versioned:          versioned,
+		VersionSuspended:   versionSuspended,
+		ReplicationRequest: true,
 	})
 	if err != nil {
 		if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
@@ -1180,7 +1185,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 				UserAgent:  "Internal: [Replication]",
 				Host:       globalLocalNodeName,
 			})
-			logger.LogOnceIf(ctx, fmt.Errorf("unable to read source object %s/%s(%s): %w", bucket, object, objInfo.VersionID, err), object+":"+objInfo.VersionID)
+			replLogOnceIf(ctx, fmt.Errorf("unable to read source object %s/%s(%s): %w", bucket, object, objInfo.VersionID, err), object+":"+objInfo.VersionID)
 		}
 		return
 	}
@@ -1193,7 +1198,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 
 	size, err := objInfo.GetActualSize()
 	if err != nil {
-		logger.LogIf(ctx, err)
+		replLogIf(ctx, err)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1205,7 +1210,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	}
 
 	if tgt.Bucket == "" {
-		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty for target %s", objInfo.Name, objInfo.VersionID, tgt.EndpointURL()))
+		replLogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty for target %s", objInfo.Name, objInfo.VersionID, tgt.EndpointURL()))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1231,7 +1236,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 
 	putOpts, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err))
+		replLogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1266,14 +1271,14 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			r, objInfo, putOpts); rinfo.Err != nil {
 			if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
 				rinfo.ReplicationStatus = replication.Failed
-				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s (target: %s)", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err, tgt.EndpointURL()))
+				replLogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s (target: %s)", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err, tgt.EndpointURL()))
 			}
 		}
 	} else {
 		if _, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); rinfo.Err != nil {
 			if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
 				rinfo.ReplicationStatus = replication.Failed
-				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s (target: %s)", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err, tgt.EndpointURL()))
+				replLogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s (target: %s)", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err, tgt.EndpointURL()))
 			}
 		}
 	}
@@ -1308,7 +1313,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	}
 
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s retry:%d", bucket, tgt.ARN, ri.RetryCount), "replication-target-offline-heal"+tgt.ARN)
+		replLogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s retry:%d", bucket, tgt.ARN, ri.RetryCount), "replication-target-offline-heal"+tgt.ARN)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1322,11 +1327,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object)
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
 
-	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{
-		VersionID:        ri.VersionID,
-		Versioned:        versioned,
-		VersionSuspended: versionSuspended,
-	})
+	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{},
+		ObjectOptions{
+			VersionID:          ri.VersionID,
+			Versioned:          versioned,
+			VersionSuspended:   versionSuspended,
+			ReplicationRequest: true,
+		})
 	if err != nil {
 		if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
 			objInfo := ri.ToObjectInfo()
@@ -1337,13 +1344,14 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				UserAgent:  "Internal: [Replication]",
 				Host:       globalLocalNodeName,
 			})
-			logger.LogIf(ctx, fmt.Errorf("unable to replicate to target %s for %s/%s(%s): %w", tgt.EndpointURL(), bucket, object, objInfo.VersionID, err))
+			replLogIf(ctx, fmt.Errorf("unable to replicate to target %s for %s/%s(%s): %w", tgt.EndpointURL(), bucket, object, objInfo.VersionID, err))
 		}
 		return
 	}
 	defer gr.Close()
 
 	objInfo := gr.ObjInfo
+
 	// make sure we have the latest metadata for metrics calculation
 	rinfo.PrevReplicationStatus = objInfo.TargetReplicationStatus(tgt.ARN)
 
@@ -1356,7 +1364,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 
 	size, err := objInfo.GetActualSize()
 	if err != nil {
-		logger.LogIf(ctx, err)
+		replLogIf(ctx, err)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1367,8 +1375,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		return
 	}
 
+	// Set the encrypted size for SSE-C objects
+	if crypto.SSEC.IsEncrypted(objInfo.UserDefined) {
+		size = objInfo.Size
+	}
+
 	if tgt.Bucket == "" {
-		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s) to %s, target bucket is missing", objInfo.Name, objInfo.VersionID, tgt.EndpointURL()))
+		replLogIf(ctx, fmt.Errorf("unable to replicate object %s(%s) to %s, target bucket is missing", objInfo.Name, objInfo.VersionID, tgt.EndpointURL()))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1398,7 +1411,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		if rAction == replicateNone {
 			if ri.OpType == replication.ExistingObjectReplicationType &&
 				objInfo.ModTime.Unix() > oi.LastModified.Unix() && objInfo.VersionID == nullVersionID {
-				logger.LogIf(ctx, fmt.Errorf("unable to replicate %s/%s (null). Newer version exists on target %s", bucket, object, tgt.EndpointURL()))
+				replLogIf(ctx, fmt.Errorf("unable to replicate %s/%s (null). Newer version exists on target %s", bucket, object, tgt.EndpointURL()))
 				sendEvent(eventArgs{
 					EventName:  event.ObjectReplicationNotTracked,
 					BucketName: bucket,
@@ -1438,7 +1451,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			rAction = replicateAll
 		default:
 			rinfo.Err = cerr
-			logger.LogIf(ctx, fmt.Errorf("unable to replicate %s/%s (%s). Target (%s) returned %s error on HEAD",
+			replLogIf(ctx, fmt.Errorf("unable to replicate %s/%s (%s). Target (%s) returned %s error on HEAD",
 				bucket, object, objInfo.VersionID, tgt.EndpointURL(), cerr))
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
@@ -1488,13 +1501,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		}
 		if _, rinfo.Err = c.CopyObject(ctx, tgt.Bucket, object, tgt.Bucket, object, getCopyObjMetadata(objInfo, tgt.StorageClass), srcOpts, dstOpts); rinfo.Err != nil {
 			rinfo.ReplicationStatus = replication.Failed
-			logger.LogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
+			replLogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
 		}
 	} else {
 		var putOpts minio.PutObjectOptions
 		putOpts, err = putReplicationOpts(ctx, tgt.StorageClass, objInfo)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
+			replLogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
@@ -1528,7 +1541,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				r, objInfo, putOpts); rinfo.Err != nil {
 				if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
 					rinfo.ReplicationStatus = replication.Failed
-					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
+					replLogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
 				} else {
 					rinfo.ReplicationStatus = replication.Completed
 				}
@@ -1537,7 +1550,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			if _, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); rinfo.Err != nil {
 				if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
 					rinfo.ReplicationStatus = replication.Failed
-					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
+					replLogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
 				} else {
 					rinfo.ReplicationStatus = replication.Completed
 				}
@@ -1563,6 +1576,9 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 		if err == nil {
 			break
 		}
+		if minio.ToErrorResponse(err).Code == "PreconditionFailed" {
+			return err
+		}
 		attempts++
 		time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
 	}
@@ -1577,14 +1593,10 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 			for attempts <= 3 {
 				actx, acancel := context.WithTimeout(ctx, time.Minute)
 				aerr := c.AbortMultipartUpload(actx, bucket, object, uploadID)
+				acancel()
 				if aerr == nil {
-					acancel()
 					return
 				}
-				acancel()
-				logger.LogIf(actx,
-					fmt.Errorf("trying %s: Unable to cleanup failed multipart replication %s on remote %s/%s: %w - this may consume space on remote cluster",
-						humanize.Ordinal(attempts), uploadID, bucket, object, aerr))
 				attempts++
 				time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
 			}
@@ -1596,21 +1608,35 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 		pInfo minio.ObjectPart
 	)
 
+	var objectSize int64
 	for _, partInfo := range objInfo.Parts {
-		hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize)
+		if crypto.SSEC.IsEncrypted(objInfo.UserDefined) {
+			hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.Size), partInfo.Size, "", "", partInfo.ActualSize)
+		} else {
+			hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize)
+		}
 		if err != nil {
 			return err
 		}
 
+		cHeader := http.Header{}
+		cHeader.Add(xhttp.MinIOSourceReplicationRequest, "true")
 		popts := minio.PutObjectPartOptions{
-			SSE: opts.ServerSideEncryption,
+			SSE:          opts.ServerSideEncryption,
+			CustomHeader: cHeader,
 		}
 
-		pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, popts)
+		if crypto.SSEC.IsEncrypted(objInfo.UserDefined) {
+			objectSize += partInfo.Size
+			pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.Size, popts)
+		} else {
+			objectSize += partInfo.ActualSize
+			pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, popts)
+		}
 		if err != nil {
 			return err
 		}
-		if pInfo.Size != partInfo.ActualSize {
+		if !crypto.SSEC.IsEncrypted(objInfo.UserDefined) && pInfo.Size != partInfo.ActualSize {
 			return fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, partInfo.ActualSize)
 		}
 		uploadedParts = append(uploadedParts, minio.CompletePart{
@@ -1618,9 +1644,12 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 			ETag:       pInfo.ETag,
 		})
 	}
+
+	// really big value but its okay on heavily loaded systems. This is just tail end timeout.
 	cctx, ccancel := context.WithTimeout(ctx, 10*time.Minute)
 	defer ccancel()
 	_, err = c.CompleteMultipartUpload(cctx, bucket, object, uploadID, uploadedParts, minio.PutObjectOptions{
+		UserMetadata: map[string]string{validSSEReplicationHeaders[ReservedMetadataPrefix+"Actual-Object-Size"]: objInfo.UserDefined[ReservedMetadataPrefix+"actual-size"]},
 		Internal: minio.AdvancedPutOptions{
 			SourceMTime: objInfo.ModTime,
 			// always set this to distinguish between `mc mirror` replication and serverside
@@ -1835,7 +1864,7 @@ func (p *ReplicationPool) AddMRFWorker() {
 				globalReplicationStats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 
 			default:
-				logger.LogOnceIf(p.ctx, fmt.Errorf("unknown mrf replication type: %T", oi), "unknown-mrf-replicate-type")
+				bugLogIf(p.ctx, fmt.Errorf("unknown mrf replication type: %T", oi), "unknown-mrf-replicate-type")
 			}
 		case <-p.mrfWorkerKillCh:
 			return
@@ -1879,7 +1908,7 @@ func (p *ReplicationPool) AddWorker(input <-chan ReplicationWorkerOperation, opT
 					atomic.AddInt32(opTracker, -1)
 				}
 			default:
-				logger.LogOnceIf(p.ctx, fmt.Errorf("unknown replication type: %T", oi), "unknown-replicate-type")
+				bugLogIf(p.ctx, fmt.Errorf("unknown replication type: %T", oi), "unknown-replicate-type")
 			}
 		}
 	}
@@ -1918,7 +1947,7 @@ func (p *ReplicationPool) AddLargeWorker(input <-chan ReplicationWorkerOperation
 			case DeletedObjectReplicationInfo:
 				replicateDelete(p.ctx, v, p.objLayer)
 			default:
-				logger.LogOnceIf(p.ctx, fmt.Errorf("unknown replication type: %T", oi), "unknown-replicate-type")
+				bugLogIf(p.ctx, fmt.Errorf("unknown replication type: %T", oi), "unknown-replicate-type")
 			}
 		}
 	}
@@ -2065,9 +2094,9 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		p.mu.RUnlock()
 		switch prio {
 		case "fast":
-			logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic"), string(replicationSubsystem))
+			replLogOnceIf(GlobalContext, fmt.Errorf("Unable to keep up with incoming traffic"), string(replicationSubsystem), logger.WarningKind)
 		case "slow":
-			logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic - we recommend increasing replication priority with `mc admin config set api replication_priority=auto`"), string(replicationSubsystem))
+			replLogOnceIf(GlobalContext, fmt.Errorf("Unable to keep up with incoming traffic - we recommend increasing replication priority with `mc admin config set api replication_priority=auto`"), string(replicationSubsystem), logger.WarningKind)
 		default:
 			maxWorkers = min(maxWorkers, WorkerMaxLimit)
 			if p.ActiveWorkers() < maxWorkers {
@@ -2122,9 +2151,9 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 		p.mu.RUnlock()
 		switch prio {
 		case "fast":
-			logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming deletes"), string(replicationSubsystem))
+			replLogOnceIf(GlobalContext, fmt.Errorf("Unable to keep up with incoming deletes"), string(replicationSubsystem), logger.WarningKind)
 		case "slow":
-			logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming deletes - we recommend increasing replication priority with `mc admin config set api replication_priority=auto`"), string(replicationSubsystem))
+			replLogOnceIf(GlobalContext, fmt.Errorf("Unable to keep up with incoming deletes - we recommend increasing replication priority with `mc admin config set api replication_priority=auto`"), string(replicationSubsystem), logger.WarningKind)
 		default:
 			maxWorkers = min(maxWorkers, WorkerMaxLimit)
 			if p.ActiveWorkers() < maxWorkers {
@@ -2257,7 +2286,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
 		if rs != nil {
 			h, err := rs.ToHeader()
 			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("invalid range header for %s/%s(%s) - %w", bucket, object, opts.VersionID, err))
+				replLogIf(ctx, fmt.Errorf("invalid range header for %s/%s(%s) - %w", bucket, object, opts.VersionID, err))
 				continue
 			}
 			gopts.Set(xhttp.Range, h)
@@ -2625,7 +2654,7 @@ func (s *replicationResyncer) PersistToDisk(ctx context.Context, objectAPI Objec
 				}
 				if updt {
 					if err := saveResyncStatus(ctx, bucket, brs, objectAPI); err != nil {
-						logger.LogIf(ctx, fmt.Errorf("could not save resync metadata to drive for %s - %w", bucket, err))
+						replLogIf(ctx, fmt.Errorf("could not save resync metadata to drive for %s - %w", bucket, err))
 					} else {
 						lastResyncStatusSave[bucket] = brs.LastUpdate
 					}
@@ -2713,12 +2742,12 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	objInfoCh := make(chan ObjectInfo)
 	cfg, err := getReplicationConfig(ctx, opts.bucket)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("replication resync of %s for arn %s failed with %w", opts.bucket, opts.arn, err))
+		replLogIf(ctx, fmt.Errorf("replication resync of %s for arn %s failed with %w", opts.bucket, opts.arn, err))
 		return
 	}
 	tgts, err := globalBucketTargetSys.ListBucketTargets(ctx, opts.bucket)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("replication resync of %s for arn %s failed  %w", opts.bucket, opts.arn, err))
+		replLogIf(ctx, fmt.Errorf("replication resync of %s for arn %s failed  %w", opts.bucket, opts.arn, err))
 		return
 	}
 	rcfg := replicationConfig{
@@ -2731,12 +2760,12 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 			TargetArn: opts.arn,
 		})
 	if len(tgtArns) != 1 {
-		logger.LogIf(ctx, fmt.Errorf("replication resync failed for %s - arn specified %s is missing in the replication config", opts.bucket, opts.arn))
+		replLogIf(ctx, fmt.Errorf("replication resync failed for %s - arn specified %s is missing in the replication config", opts.bucket, opts.arn))
 		return
 	}
 	tgt := globalBucketTargetSys.GetRemoteTargetClient(opts.bucket, opts.arn)
 	if tgt == nil {
-		logger.LogIf(ctx, fmt.Errorf("replication resync failed for %s - target could not be created for arn %s", opts.bucket, opts.arn))
+		replLogIf(ctx, fmt.Errorf("replication resync failed for %s - target could not be created for arn %s", opts.bucket, opts.arn))
 		return
 	}
 	// mark resync status as resync started
@@ -2747,7 +2776,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	// Walk through all object versions - Walk() is always in ascending order needed to ensure
 	// delete marker replicated to target after object version is first created.
 	if err := objectAPI.Walk(ctx, opts.bucket, "", objInfoCh, WalkOptions{}); err != nil {
-		logger.LogIf(ctx, err)
+		replLogIf(ctx, err)
 		return
 	}
 
@@ -3022,7 +3051,7 @@ func (p *ReplicationPool) loadResync(ctx context.Context, buckets []BucketInfo,
 		meta, err := loadBucketResyncMetadata(ctx, bucket, objAPI)
 		if err != nil {
 			if !errors.Is(err, errVolumeNotFound) {
-				logger.LogIf(ctx, err)
+				replLogIf(ctx, err)
 			}
 			continue
 		}
@@ -3109,18 +3138,18 @@ func saveResyncStatus(ctx context.Context, bucket string, brs BucketReplicationR
 func getReplicationDiff(ctx context.Context, objAPI ObjectLayer, bucket string, opts madmin.ReplDiffOpts) (chan madmin.DiffInfo, error) {
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		replLogIf(ctx, err)
 		return nil, err
 	}
 	tgts, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		replLogIf(ctx, err)
 		return nil, err
 	}
 
 	objInfoCh := make(chan ObjectInfo, 10)
 	if err := objAPI.Walk(ctx, bucket, opts.Prefix, objInfoCh, WalkOptions{}); err != nil {
-		logger.LogIf(ctx, err)
+		replLogIf(ctx, err)
 		return nil, err
 	}
 	rcfg := replicationConfig{
@@ -3504,7 +3533,7 @@ func (p *ReplicationPool) processMRF() {
 				continue
 			}
 			if err := p.queueMRFHeal(); err != nil && !osIsNotExist(err) {
-				logger.LogIf(p.ctx, err)
+				replLogIf(p.ctx, err)
 			}
 			pTimer.Reset(mrfQueueInterval)
 		case <-p.ctx.Done():

cmd/bucket-targets.go

@@ -20,7 +20,6 @@ package cmd
 import (
 	"context"
 	"errors"
-	"fmt"
 	"net/url"
 	"sync"
 	"time"
@@ -32,7 +31,6 @@ import (
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
 )
 
 const (
@@ -131,7 +129,7 @@ func (sys *BucketTargetSys) initHC(ep *url.URL) {
 func newHCClient() *madmin.AnonymousClient {
 	clnt, e := madmin.NewAnonymousClientNoEndpoint()
 	if e != nil {
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to initialize health check client"), string(replicationSubsystem))
+		bugLogIf(GlobalContext, errors.New("Unable to initialize health check client"))
 		return nil
 	}
 	clnt.SetCustomTransport(globalRemoteTargetTransport)
@@ -624,7 +622,7 @@ func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
 	for _, tgt := range cfg.Targets {
 		tgtClient, err := sys.getRemoteTargetClient(&tgt)
 		if err != nil {
-			logger.LogIf(GlobalContext, err)
+			replLogIf(GlobalContext, err)
 			continue
 		}
 		sys.arnRemotesMap[tgt.Arn] = arnTarget{Client: tgtClient}

cmd/bucket-versioning-handler.go

@@ -108,7 +108,7 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:       madmin.SRBucketMetaTypeVersionConfig,
 		Bucket:     bucket,
 		Versioning: &cfgStr,

cmd/build-constants.go

@@ -17,6 +17,8 @@
 
 package cmd
 
+import "runtime"
+
 // DO NOT EDIT THIS FILE DIRECTLY. These are build-time constants
 // set through ‘buildscripts/gen-ldflags.go’.
 var (
@@ -40,4 +42,28 @@ var (
 
 	// CopyrightYear - dynamic value of the copyright end year
 	CopyrightYear = "0000"
+
+	// MinioReleaseTagTimeLayout - release tag time layout.
+	MinioReleaseTagTimeLayout = "2006-01-02T15-04-05Z"
+
+	// MinioOSARCH - OS and ARCH.
+	minioOSARCH = runtime.GOOS + "-" + runtime.GOARCH
+
+	// MinioReleaseBaseURL - release url without os and arch.
+	MinioReleaseBaseURL = "https://dl.min.io/server/minio/release/"
+
+	// MinioReleaseURL - release URL.
+	MinioReleaseURL = MinioReleaseBaseURL + minioOSARCH + SlashSeparator
+
+	// MinioStoreName - MinIO store name.
+	MinioStoreName = "MinIO"
+
+	// MinioUAName - MinIO user agent name.
+	MinioUAName = "MinIO"
+
+	// MinioBannerName - MinIO banner name for startup message.
+	MinioBannerName = "MinIO Object Storage Server"
+
+	// MinioLicense - MinIO server license.
+	MinioLicense = "GNU AGPLv3 <https://www.gnu.org/licenses/agpl-3.0.html>"
 )

cmd/callhome.go

@@ -29,7 +29,6 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go/v3"
-	"github.com/minio/minio/internal/logger"
 )
 
 var callhomeLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
@@ -112,7 +111,7 @@ func performCallhome(ctx context.Context) {
 	deadline := 10 * time.Second // Default deadline is 10secs for callhome
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
-		logger.LogIf(ctx, errors.New("Callhome: object layer not ready"))
+		internalLogIf(ctx, errors.New("Callhome: object layer not ready"))
 		return
 	}
 
@@ -145,7 +144,7 @@ func performCallhome(ctx context.Context) {
 				// Received all data. Send to SUBNET and return
 				err := sendHealthInfo(ctx, healthInfo)
 				if err != nil {
-					logger.LogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err))
+					internalLogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err))
 				}
 				return
 			}
@@ -180,12 +179,12 @@ func createHealthJSONGzip(ctx context.Context, healthInfo madmin.HealthInfo) []b
 
 	enc := json.NewEncoder(gzWriter)
 	if e := enc.Encode(header); e != nil {
-		logger.LogIf(ctx, fmt.Errorf("Could not encode health info header: %w", e))
+		internalLogIf(ctx, fmt.Errorf("Could not encode health info header: %w", e))
 		return nil
 	}
 
 	if e := enc.Encode(healthInfo); e != nil {
-		logger.LogIf(ctx, fmt.Errorf("Could not encode health info: %w", e))
+		internalLogIf(ctx, fmt.Errorf("Could not encode health info: %w", e))
 		return nil
 	}
 

cmd/common-main.go

@@ -66,9 +66,11 @@ import (
 )
 
 // serverDebugLog will enable debug printing
-var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
-
-var currentReleaseTime time.Time
+var (
+	serverDebugLog     = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
+	currentReleaseTime time.Time
+	orchestrated       = IsKubernetes() || IsDocker()
+)
 
 func init() {
 	if runtime.GOOS == "windows" {
@@ -361,6 +363,14 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 		ctxt.ConsoleAddr = ctx.String("console-address")
 	}
 
+	if cxml := ctx.String("crossdomain-xml"); cxml != "" {
+		buf, err := os.ReadFile(cxml)
+		if err != nil {
+			return err
+		}
+		ctxt.CrossDomainXML = string(buf)
+	}
+
 	// Check "no-compat" flag from command line argument.
 	ctxt.StrictS3Compat = !(ctx.IsSet("no-compat") || ctx.GlobalIsSet("no-compat"))
 
@@ -486,7 +496,11 @@ func runDNSCache(ctx *cli.Context) {
 	dnsTTL := ctx.Duration("dns-cache-ttl")
 	// Check if we have configured a custom DNS cache TTL.
 	if dnsTTL <= 0 {
-		dnsTTL = 10 * time.Minute
+		if orchestrated {
+			dnsTTL = 30 * time.Second
+		} else {
+			dnsTTL = 10 * time.Minute
+		}
 	}
 
 	// Call to refresh will refresh names in cache.
@@ -749,12 +763,7 @@ func serverHandleEnvVars() {
 		for _, endpoint := range minioEndpoints {
 			if net.ParseIP(endpoint) == nil {
 				// Checking if the IP is a DNS entry.
-				lookupHost := globalDNSCache.LookupHost
-				if IsKubernetes() || IsDocker() {
-					lookupHost = net.DefaultResolver.LookupHost
-				}
-
-				addrs, err := lookupHost(GlobalContext, endpoint)
+				addrs, err := globalDNSCache.LookupHost(GlobalContext, endpoint)
 				if err != nil {
 					logger.FatalIf(err, "Unable to initialize MinIO server with [%s] invalid entry found in MINIO_PUBLIC_IPS", endpoint)
 				}
@@ -1035,7 +1044,7 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 		}
 		if err = manager.AddCertificate(certFile, keyFile); err != nil {
 			err = fmt.Errorf("Unable to load TLS certificate '%s,%s': %w", certFile, keyFile, err)
-			logger.LogIf(GlobalContext, err, logger.ErrorKind)
+			bootLogIf(GlobalContext, err, logger.ErrorKind)
 		}
 	}
 	secureConn = true

cmd/config-current.go

@@ -479,7 +479,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 	dnsURL, dnsUser, dnsPass, err := env.LookupEnv(config.EnvDNSWebhook)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
+		configLogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
 	}
 	if err == nil && dnsURL != "" {
 		bootstrapTraceMsg("initialize remote bucket DNS store")
@@ -487,27 +487,27 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 			dns.Authentication(dnsUser, dnsPass),
 			dns.RootCAs(globalRootCAs))
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
 		}
 	}
 
 	etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+		configLogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 	}
 
 	if etcdCfg.Enabled {
 		bootstrapTraceMsg("initialize etcd store")
 		globalEtcdClient, err = etcd.New(etcdCfg)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 		}
 
 		if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
 			if globalDNSConfig != nil {
 				// if global DNS is already configured, indicate with a warning, in case
 				// users are confused.
-				logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, etcd is not used for DNS store", globalDNSConfig))
+				configLogIf(ctx, fmt.Errorf("DNS store is already configured with %s, etcd is not used for DNS store", globalDNSConfig))
 			} else {
 				globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
 					dns.DomainNames(globalDomainNames),
@@ -516,7 +516,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 					dns.CoreDNSPath(etcdCfg.CoreDNSPath),
 				)
 				if err != nil {
-					logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
+					configLogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
 						globalDomainNames, err))
 				}
 			}
@@ -532,7 +532,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 	globalSite, err = config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default])
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err))
+		configLogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err))
 	}
 
 	globalAutoEncryption = crypto.LookupAutoEncryption() // Enable auto-encryption if enabled
@@ -545,19 +545,19 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	bootstrapTraceMsg("initialize the event notification targets")
 	globalNotifyTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
+		configLogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
 
 	bootstrapTraceMsg("initialize the lambda targets")
 	globalLambdaTargetList, err = lambda.FetchEnabledTargets(GlobalContext, s, transport)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize lambda target(s): %w", err))
+		configLogIf(ctx, fmt.Errorf("Unable to initialize lambda target(s): %w", err))
 	}
 
 	bootstrapTraceMsg("applying the dynamic configuration")
 	// Apply dynamic config values
 	if err := applyDynamicConfig(ctx, objAPI, s); err != nil {
-		logger.LogIf(ctx, err)
+		configLogIf(ctx, err)
 	}
 }
 
@@ -571,16 +571,12 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 	case config.APISubSys:
 		apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
+			configLogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
 		}
 
 		globalAPIConfig.init(apiConfig, setDriveCounts)
 		autoGenerateRootCredentials() // Generate the KMS root credentials here since we don't know whether API root access is disabled until now.
-
-		// Initialize remote instance transport once.
-		getRemoteInstanceTransportOnce.Do(func() {
-			getRemoteInstanceTransport = NewHTTPTransportWithTimeout(apiConfig.RemoteTransportDeadline)
-		})
+		setRemoteInstanceTransport(NewHTTPTransportWithTimeout(apiConfig.RemoteTransportDeadline))
 	case config.CompressionSubSys:
 		cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
 		if err != nil {
@@ -611,64 +607,64 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		scannerCycle.Store(scannerCfg.Cycle)
 		scannerExcessObjectVersions.Store(scannerCfg.ExcessVersions)
 		scannerExcessFolders.Store(scannerCfg.ExcessFolders)
-		logger.LogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait))
+		configLogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait))
 	case config.LoggerWebhookSubSys:
 		loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.LoggerWebhookSubSys)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to load logger webhook config: %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to load logger webhook config: %w", err))
 		}
 		userAgent := getUserAgent(getMinioMode())
 		for n, l := range loggerCfg.HTTP {
 			if l.Enabled {
-				l.LogOnce = logger.LogOnceConsoleIf
+				l.LogOnceIf = configLogOnceConsoleIf
 				l.UserAgent = userAgent
 				l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			}
 			loggerCfg.HTTP[n] = l
 		}
-		if errs := logger.UpdateSystemTargets(ctx, loggerCfg); len(errs) > 0 {
-			logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %v", errs))
+		if errs := logger.UpdateHTTPWebhooks(ctx, loggerCfg.HTTP); len(errs) > 0 {
+			configLogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %v", errs))
 		}
 	case config.AuditWebhookSubSys:
 		loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.AuditWebhookSubSys)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to load audit webhook config: %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to load audit webhook config: %w", err))
 		}
 		userAgent := getUserAgent(getMinioMode())
 		for n, l := range loggerCfg.AuditWebhook {
 			if l.Enabled {
-				l.LogOnce = logger.LogOnceConsoleIf
+				l.LogOnceIf = configLogOnceConsoleIf
 				l.UserAgent = userAgent
 				l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			}
 			loggerCfg.AuditWebhook[n] = l
 		}
 
-		if errs := logger.UpdateAuditWebhookTargets(ctx, loggerCfg); len(errs) > 0 {
-			logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %v", errs))
+		if errs := logger.UpdateAuditWebhooks(ctx, loggerCfg.AuditWebhook); len(errs) > 0 {
+			configLogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %v", errs))
 		}
 	case config.AuditKafkaSubSys:
 		loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.AuditKafkaSubSys)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to load audit kafka config: %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to load audit kafka config: %w", err))
 		}
 		for n, l := range loggerCfg.AuditKafka {
 			if l.Enabled {
 				if l.TLS.Enable {
 					l.TLS.RootCAs = globalRootCAs
 				}
-				l.LogOnce = logger.LogOnceIf
+				l.LogOnce = configLogOnceIf
 				loggerCfg.AuditKafka[n] = l
 			}
 		}
 		if errs := logger.UpdateAuditKafkaTargets(ctx, loggerCfg); len(errs) > 0 {
-			logger.LogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %v", errs))
+			configLogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %v", errs))
 		}
 	case config.StorageClassSubSys:
 		for i, setDriveCount := range setDriveCounts {
 			sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
 			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
+				configLogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
 				break
 			}
 			// if we validated all setDriveCounts and it was successful
@@ -680,7 +676,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 	case config.SubnetSubSys:
 		subnetConfig, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
 		} else {
 			globalSubnetConfig.Update(subnetConfig, globalIsCICD)
 			globalSubnetConfig.ApplyEnv() // update environment settings for Console UI
@@ -688,7 +684,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 	case config.CallhomeSubSys:
 		callhomeCfg, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default])
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to load callhome config: %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to load callhome config: %w", err))
 		} else {
 			enable := callhomeCfg.Enable && !globalCallhomeConfig.Enabled()
 			globalCallhomeConfig.Update(callhomeCfg)
@@ -698,17 +694,17 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		}
 	case config.DriveSubSys:
 		if driveConfig, err := drive.LookupConfig(s[config.DriveSubSys][config.Default]); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to load drive config: %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to load drive config: %w", err))
 		} else {
 			err := globalDriveConfig.Update(driveConfig)
 			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to update drive config: %v", err))
+				configLogIf(ctx, fmt.Errorf("Unable to update drive config: %v", err))
 			}
 		}
 	case config.CacheSubSys:
 		cacheCfg, err := cache.LookupConfig(s[config.CacheSubSys][config.Default], globalRemoteTargetTransport)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to load cache config: %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to load cache config: %w", err))
 		} else {
 			globalCacheConfig.Update(cacheCfg)
 		}
@@ -753,7 +749,7 @@ func autoGenerateRootCredentials() {
 	if manager, ok := GlobalKMS.(kms.KeyManager); ok {
 		stat, err := GlobalKMS.Stat(GlobalContext)
 		if err != nil {
-			logger.LogIf(GlobalContext, err, "Unable to generate root credentials using KMS")
+			kmsLogIf(GlobalContext, err, "Unable to generate root credentials using KMS")
 			return
 		}
 

cmd/crossdomain-xml-handler.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -32,10 +32,14 @@ const crossDomainXMLEntity = "/crossdomain.xml"
 // policy file that grants access to the source domain, allowing the client to continue the transaction.
 func setCrossDomainPolicyMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		cxml := crossDomainXML
+		if globalServerCtxt.CrossDomainXML != "" {
+			cxml = globalServerCtxt.CrossDomainXML
+		}
 		// Look for 'crossdomain.xml' in the incoming request.
 		if r.URL.Path == crossDomainXMLEntity {
 			// Write the standard cross domain policy xml.
-			w.Write([]byte(crossDomainXML))
+			w.Write([]byte(cxml))
 			// Request completed, no need to serve to other handlers.
 			return
 		}

cmd/data-scanner.go

@@ -41,7 +41,6 @@ import (
 	"github.com/minio/minio/internal/config/heal"
 	"github.com/minio/minio/internal/event"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 	uatomic "go.uber.org/atomic"
 )
@@ -122,13 +121,13 @@ func readBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer) backgroundH
 	buf, err := readConfig(ctx, objAPI, backgroundHealInfoPath)
 	if err != nil {
 		if !errors.Is(err, errConfigNotFound) {
-			logger.LogOnceIf(ctx, err, backgroundHealInfoPath)
+			internalLogOnceIf(ctx, err, backgroundHealInfoPath)
 		}
 		return backgroundHealInfo{}
 	}
 	var info backgroundHealInfo
 	if err = json.Unmarshal(buf, &info); err != nil {
-		logger.LogOnceIf(ctx, err, backgroundHealInfoPath)
+		bugLogIf(ctx, err, backgroundHealInfoPath)
 	}
 	return info
 }
@@ -140,13 +139,13 @@ func saveBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer, info backgr
 
 	b, err := json.Marshal(info)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		bugLogIf(ctx, err)
 		return
 	}
 	// Get last healing information
 	err = saveConfig(ctx, objAPI, backgroundHealInfoPath, b)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		internalLogIf(ctx, err)
 	}
 }
 
@@ -167,7 +166,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 		cycleInfo.next = binary.LittleEndian.Uint64(buf[:8])
 		buf = buf[8:]
 		_, err := cycleInfo.UnmarshalMsg(buf)
-		logger.LogIf(ctx, err)
+		bugLogIf(ctx, err)
 	}
 
 	scannerTimer := time.NewTimer(scannerCycle.Load())
@@ -204,7 +203,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
 			err := objAPI.NSScanner(ctx, results, uint32(cycleInfo.current), scanMode)
-			logger.LogOnceIf(ctx, err, "ns-scanner")
+			scannerLogIf(ctx, err)
 			res := map[string]string{"cycle": strconv.FormatUint(cycleInfo.current, 10)}
 			if err != nil {
 				res["error"] = err.Error()
@@ -224,7 +223,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 				binary.LittleEndian.PutUint64(tmp, cycleInfo.next)
 				tmp, _ = cycleInfo.MarshalMsg(tmp)
 				err = saveConfig(ctx, objAPI, dataUsageBloomNamePath, tmp)
-				logger.LogOnceIf(ctx, err, dataUsageBloomNamePath)
+				scannerLogIf(ctx, err, dataUsageBloomNamePath)
 			}
 		}
 	}
@@ -752,7 +751,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 							versionID: "",
 						}, madmin.HealItemObject)
 						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-							logger.LogOnceIf(ctx, err, entry.name)
+							scannerLogIf(ctx, err)
 						}
 						foundObjs = foundObjs || err == nil
 						return
@@ -769,7 +768,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						}, madmin.HealItemObject)
 						stopFn(int(ver.Size))
 						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-							logger.LogOnceIf(ctx, err, fiv.Name)
+							scannerLogIf(ctx, err, fiv.Name)
 						}
 						if err == nil {
 							successVersions++
@@ -945,7 +944,7 @@ func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi Object
 func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi ObjectInfo) (action lifecycle.Action, size int64) {
 	size, err := oi.GetActualSize()
 	if i.debug {
-		logger.LogIf(ctx, err)
+		scannerLogIf(ctx, err)
 	}
 	if i.lifeCycle == nil {
 		return action, size
@@ -1051,7 +1050,9 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 		})
 	}
 
-	expState.enqueueByNewerNoncurrent(i.bucket, toDel, event)
+	if len(toDel) > 0 {
+		expState.enqueueByNewerNoncurrent(i.bucket, toDel, event)
+	}
 	return objectInfos, nil
 }
 
@@ -1121,7 +1122,7 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 				err := o.CheckAbandonedParts(ctx, i.bucket, i.objectPath(), madmin.HealOpts{Remove: healDeleteDangling})
 				done()
 				if err != nil {
-					logger.LogOnceIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err), i.objectPath())
+					healingLogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err), i.objectPath())
 				}
 			}
 		}
@@ -1197,7 +1198,7 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return false
 		}
-		logger.LogOnceIf(ctx, err, obj.Name)
+		ilmLogIf(ctx, err)
 		return false
 	}
 	// Notification already sent in *expireTransitionedObject*, just return 'true' here.
@@ -1219,6 +1220,8 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 
 	if lcEvent.Action.DeleteAll() {
 		opts.DeletePrefix = true
+		// use prefix delete on exact object (this is an optimization to avoid fan-out calls)
+		opts.DeletePrefixObject = true
 	}
 	var (
 		dobj ObjectInfo
@@ -1238,13 +1241,13 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		}
 	}()
 
-	dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, obj.Name, opts)
+	dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts)
 	if err != nil {
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return false
 		}
 		// Assume it is still there.
-		logger.LogOnceIf(ctx, err, "non-transition-expiry")
+		ilmLogOnceIf(ctx, err, "non-transition-expiry")
 		return false
 	}
 	if dobj.Name == "" {
@@ -1259,7 +1262,9 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	if obj.DeleteMarker {
 		eventName = event.ObjectRemovedDeleteMarkerCreated
 	}
-
+	if lcEvent.Action.DeleteAll() {
+		eventName = event.ObjectRemovedDeleteAllVersions
+	}
 	// Notify object deleted event.
 	sendEvent(eventArgs{
 		EventName:  eventName,

cmd/data-usage-cache.go

@@ -37,7 +37,6 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/hash"
-	"github.com/minio/minio/internal/logger"
 	"github.com/tinylib/msgp/msgp"
 	"github.com/valyala/bytebufferpool"
 )
@@ -191,6 +190,22 @@ type replicationAllStatsV1 struct {
 	ReplicaCount uint64 `msg:"ReplicaCount,omitempty"`
 }
 
+// empty returns true if the replicationAllStats is empty (contains no entries).
+func (r *replicationAllStats) empty() bool {
+	if r == nil {
+		return true
+	}
+	if r.ReplicaSize != 0 || r.ReplicaCount != 0 {
+		return false
+	}
+	for _, v := range r.Targets {
+		if !v.Empty() {
+			return false
+		}
+	}
+	return true
+}
+
 // clone creates a deep-copy clone.
 func (r *replicationAllStats) clone() *replicationAllStats {
 	if r == nil {
@@ -523,20 +538,18 @@ func (d *dataUsageCache) searchParent(h dataUsageHash) *dataUsageHash {
 	want := h.Key()
 	if idx := strings.LastIndexByte(want, '/'); idx >= 0 {
 		if v := d.find(want[:idx]); v != nil {
-			for child := range v.Children {
-				if child == want {
-					found := hashPath(want[:idx])
-					return &found
-				}
+			_, ok := v.Children[want]
+			if ok {
+				found := hashPath(want[:idx])
+				return &found
 			}
 		}
 	}
 	for k, v := range d.Cache {
-		for child := range v.Children {
-			if child == want {
-				found := dataUsageHash(k)
-				return &found
-			}
+		_, ok := v.Children[want]
+		if ok {
+			found := dataUsageHash(k)
+			return &found
 		}
 	}
 	return nil
@@ -621,7 +634,7 @@ func (d *dataUsageCache) copyWithChildren(src *dataUsageCache, hash dataUsageHas
 	d.Cache[hash.Key()] = e
 	for ch := range e.Children {
 		if ch == hash.Key() {
-			logger.LogIf(GlobalContext, errors.New("dataUsageCache.copyWithChildren: Circular reference"))
+			scannerLogIf(GlobalContext, errors.New("dataUsageCache.copyWithChildren: Circular reference"))
 			return
 		}
 		d.copyWithChildren(src, dataUsageHash(ch), &hash)
@@ -901,6 +914,9 @@ func (d *dataUsageCache) sizeRecursive(path string) *dataUsageEntry {
 		return root
 	}
 	flat := d.flatten(*root)
+	if flat.ReplicationStats.empty() {
+		flat.ReplicationStats = nil
+	}
 	return &flat
 }
 
@@ -1024,7 +1040,7 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string)
 	}
 
 	if retries == 5 {
-		logger.LogOnceIf(ctx, fmt.Errorf("maximum retry reached to load the data usage cache `%s`", name), "retry-loading-data-usage-cache")
+		scannerLogOnceIf(ctx, fmt.Errorf("maximum retry reached to load the data usage cache `%s`", name), "retry-loading-data-usage-cache")
 	}
 
 	return nil

cmd/data-usage-cache_gen.go

@@ -1762,7 +1762,7 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
-	// omitempty: check for empty values
+	// check for omitted fields
 	zb0001Len := uint32(10)
 	var zb0001Mask uint16 /* 10 bits */
 	_ = zb0001Mask
@@ -1866,7 +1866,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
-	if (zb0001Mask & 0x80) == 0 { // if not empty
+	if (zb0001Mask & 0x80) == 0 { // if not omitted
 		// write "rs"
 		err = en.Append(0xa2, 0x72, 0x73)
 		if err != nil {
@@ -1885,7 +1885,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x100) == 0 { // if not empty
+	if (zb0001Mask & 0x100) == 0 { // if not omitted
 		// write "ats"
 		err = en.Append(0xa3, 0x61, 0x74, 0x73)
 		if err != nil {
@@ -1920,7 +1920,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// omitempty: check for empty values
+	// check for omitted fields
 	zb0001Len := uint32(10)
 	var zb0001Mask uint16 /* 10 bits */
 	_ = zb0001Mask
@@ -1968,7 +1968,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	for za0002 := range z.ObjVersions {
 		o = msgp.AppendUint64(o, z.ObjVersions[za0002])
 	}
-	if (zb0001Mask & 0x80) == 0 { // if not empty
+	if (zb0001Mask & 0x80) == 0 { // if not omitted
 		// string "rs"
 		o = append(o, 0xa2, 0x72, 0x73)
 		if z.ReplicationStats == nil {
@@ -1981,7 +1981,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x100) == 0 { // if not empty
+	if (zb0001Mask & 0x100) == 0 { // if not omitted
 		// string "ats"
 		o = append(o, 0xa3, 0x61, 0x74, 0x73)
 		if z.AllTierStats == nil {
@@ -3243,7 +3243,7 @@ func (z *replicationAllStats) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
-	// omitempty: check for empty values
+	// check for omitted fields
 	zb0001Len := uint32(3)
 	var zb0001Mask uint8 /* 3 bits */
 	_ = zb0001Mask
@@ -3267,7 +3267,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 	if zb0001Len == 0 {
 		return
 	}
-	if (zb0001Mask & 0x1) == 0 { // if not empty
+	if (zb0001Mask & 0x1) == 0 { // if not omitted
 		// write "t"
 		err = en.Append(0xa1, 0x74)
 		if err != nil {
@@ -3291,7 +3291,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x2) == 0 { // if not empty
+	if (zb0001Mask & 0x2) == 0 { // if not omitted
 		// write "r"
 		err = en.Append(0xa1, 0x72)
 		if err != nil {
@@ -3303,7 +3303,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
-	if (zb0001Mask & 0x4) == 0 { // if not empty
+	if (zb0001Mask & 0x4) == 0 { // if not omitted
 		// write "rc"
 		err = en.Append(0xa2, 0x72, 0x63)
 		if err != nil {
@@ -3321,7 +3321,7 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// omitempty: check for empty values
+	// check for omitted fields
 	zb0001Len := uint32(3)
 	var zb0001Mask uint8 /* 3 bits */
 	_ = zb0001Mask
@@ -3342,7 +3342,7 @@ func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) {
 	if zb0001Len == 0 {
 		return
 	}
-	if (zb0001Mask & 0x1) == 0 { // if not empty
+	if (zb0001Mask & 0x1) == 0 { // if not omitted
 		// string "t"
 		o = append(o, 0xa1, 0x74)
 		o = msgp.AppendMapHeader(o, uint32(len(z.Targets)))
@@ -3355,12 +3355,12 @@ func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x2) == 0 { // if not empty
+	if (zb0001Mask & 0x2) == 0 { // if not omitted
 		// string "r"
 		o = append(o, 0xa1, 0x72)
 		o = msgp.AppendUint64(o, z.ReplicaSize)
 	}
-	if (zb0001Mask & 0x4) == 0 { // if not empty
+	if (zb0001Mask & 0x4) == 0 { // if not omitted
 		// string "rc"
 		o = append(o, 0xa2, 0x72, 0x63)
 		o = msgp.AppendUint64(o, z.ReplicaCount)
@@ -3478,6 +3478,8 @@ func (z *replicationAllStatsV1) DecodeMsg(dc *msgp.Reader) (err error) {
 			delete(z.Targets, key)
 		}
 	}
+	var field []byte
+	_ = field
 	for zb0002 > 0 {
 		zb0002--
 		var za0001 string
@@ -3588,6 +3590,8 @@ func (z *replicationAllStatsV1) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			delete(z.Targets, key)
 		}
 	}
+	var field []byte
+	_ = field
 	for zb0002 > 0 {
 		var za0001 string
 		var za0002 replicationStats

cmd/data-usage-utils.go

@@ -140,7 +140,7 @@ func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
 	return infos
 }
 
-func (dui DataUsageInfo) tierMetrics() (metrics []Metric) {
+func (dui DataUsageInfo) tierMetrics() (metrics []MetricV2) {
 	if dui.TierStats == nil {
 		return nil
 	}
@@ -148,17 +148,17 @@ func (dui DataUsageInfo) tierMetrics() (metrics []Metric) {
 	//     minio_cluster_ilm_transitioned_objects{tier="S3TIER-1"}=1
 	//     minio_cluster_ilm_transitioned_versions{tier="S3TIER-1"}=3
 	for tier, st := range dui.TierStats.Tiers {
-		metrics = append(metrics, Metric{
+		metrics = append(metrics, MetricV2{
 			Description:    getClusterTransitionedBytesMD(),
 			Value:          float64(st.TotalSize),
 			VariableLabels: map[string]string{"tier": tier},
 		})
-		metrics = append(metrics, Metric{
+		metrics = append(metrics, MetricV2{
 			Description:    getClusterTransitionedObjectsMD(),
 			Value:          float64(st.NumObjects),
 			VariableLabels: map[string]string{"tier": tier},
 		})
-		metrics = append(metrics, Metric{
+		metrics = append(metrics, MetricV2{
 			Description:    getClusterTransitionedVersionsMD(),
 			Value:          float64(st.NumVersions),
 			VariableLabels: map[string]string{"tier": tier},

cmd/data-usage.go

@@ -25,7 +25,6 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/internal/cachevalue"
-	"github.com/minio/minio/internal/logger"
 )
 
 const (
@@ -49,7 +48,7 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
 		json := jsoniter.ConfigCompatibleWithStandardLibrary
 		dataUsageJSON, err := json.Marshal(dataUsageInfo)
 		if err != nil {
-			logger.LogIf(ctx, err)
+			scannerLogIf(ctx, err)
 			continue
 		}
 		if attempts > 10 {
@@ -57,7 +56,7 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
 			attempts = 1
 		}
 		if err = saveConfig(ctx, objAPI, dataUsageObjNamePath, dataUsageJSON); err != nil {
-			logger.LogOnceIf(ctx, err, dataUsageObjNamePath)
+			scannerLogOnceIf(ctx, err, dataUsageObjNamePath)
 		}
 		attempts++
 	}

cmd/data-usage_test.go

@@ -364,6 +364,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 			}
 			if e == nil {
 				t.Fatal("got nil result")
+				return
 			}
 			if e.Size != int64(w.size) {
 				t.Error("got size", e.Size, "want", w.size)

cmd/encryption-v1.go

@@ -1089,7 +1089,7 @@ func (o *ObjectInfo) decryptPartsChecksums() {
 	if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted {
 		decrypted, err := o.metadataDecrypter()("object-checksum", data)
 		if err != nil {
-			logger.LogIf(GlobalContext, err)
+			encLogIf(GlobalContext, err)
 			return
 		}
 		data = decrypted
@@ -1151,7 +1151,7 @@ func (o *ObjectInfo) decryptChecksums(part int) map[string]string {
 	if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted {
 		decrypted, err := o.metadataDecrypter()("object-checksum", data)
 		if err != nil {
-			logger.LogIf(GlobalContext, err)
+			encLogIf(GlobalContext, err)
 			return nil
 		}
 		data = decrypted

cmd/endpoint.go

@@ -514,7 +514,7 @@ func (l EndpointServerPools) hostsSorted() []*xnet.Host {
 		}
 		host, err := xnet.ParseHost(hostStr)
 		if err != nil {
-			logger.LogIf(GlobalContext, err)
+			internalLogIf(GlobalContext, err)
 			continue
 		}
 		hosts[i] = host
@@ -590,8 +590,6 @@ func hostResolveToLocalhost(endpoint Endpoint) bool {
 
 // UpdateIsLocal - resolves the host and discovers the local host.
 func (endpoints Endpoints) UpdateIsLocal() error {
-	orchestrated := IsDocker() || IsKubernetes()
-
 	var epsResolved int
 	var foundLocal bool
 	resolvedList := make([]bool, len(endpoints))
@@ -647,7 +645,7 @@ func (endpoints Endpoints) UpdateIsLocal() error {
 							))
 						ctx := logger.SetReqInfo(GlobalContext,
 							reqInfo)
-						logger.LogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
+						bootLogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
 							endpoints[i].Hostname()), endpoints[i].Hostname(), logger.ErrorKind)
 					}
 
@@ -677,7 +675,7 @@ func (endpoints Endpoints) UpdateIsLocal() error {
 							))
 						ctx := logger.SetReqInfo(GlobalContext,
 							reqInfo)
-						logger.LogOnceIf(ctx, err, endpoints[i].Hostname(), logger.ErrorKind)
+						bootLogOnceIf(ctx, err, endpoints[i].Hostname(), logger.ErrorKind)
 					}
 				} else {
 					resolvedList[i] = true
@@ -775,8 +773,6 @@ type PoolEndpointList []Endpoints
 
 // UpdateIsLocal - resolves all hosts and discovers which are local
 func (p PoolEndpointList) UpdateIsLocal() error {
-	orchestrated := IsDocker() || IsKubernetes()
-
 	var epsResolved int
 	var epCount int
 
@@ -841,7 +837,7 @@ func (p PoolEndpointList) UpdateIsLocal() error {
 								))
 							ctx := logger.SetReqInfo(GlobalContext,
 								reqInfo)
-							logger.LogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
+							bootLogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
 								endpoint.Hostname()), endpoint.Hostname(), logger.ErrorKind)
 						}
 						continue
@@ -870,7 +866,7 @@ func (p PoolEndpointList) UpdateIsLocal() error {
 								))
 							ctx := logger.SetReqInfo(GlobalContext,
 								reqInfo)
-							logger.LogOnceIf(ctx, fmt.Errorf("Unable to resolve DNS for %s: %w", endpoint, err), endpoint.Hostname(), logger.ErrorKind)
+							bootLogOnceIf(ctx, fmt.Errorf("Unable to resolve DNS for %s: %w", endpoint, err), endpoint.Hostname(), logger.ErrorKind)
 						}
 					} else {
 						resolvedList[endpoint] = true
@@ -1034,7 +1030,6 @@ func CreatePoolEndpoints(serverAddr string, poolsLayout ...poolDisksLayout) ([]E
 			}
 		}
 
-		orchestrated := IsKubernetes() || IsDocker()
 		reverseProxy := (env.Get("_MINIO_REVERSE_PROXY", "") != "") && ((env.Get("MINIO_CI_CD", "") != "") || (env.Get("CI", "") != ""))
 		// If not orchestrated
 		// and not setup in reverse proxy

cmd/erasure-common.go

@@ -25,7 +25,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/sync/errgroup"
 )
 
@@ -163,7 +162,7 @@ func readMultipleFiles(ctx context.Context, disks []StorageAPI, req ReadMultiple
 			continue
 		}
 		if !IsErr(err, ignoredErrs...) {
-			logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
+			storageLogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
 				disks[index], req.Bucket, req.Prefix, err),
 				disks[index].String())
 		}

cmd/erasure-healing.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/grid"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/sync/errgroup"
 )
@@ -210,6 +211,24 @@ func (fi FileInfo) DataMov() bool {
 	return ok
 }
 
+func auditHealObject(ctx context.Context, bucket, object, versionID string, err error) {
+	if len(logger.AuditTargets()) == 0 {
+		return
+	}
+
+	opts := AuditLogOptions{
+		Event:     "HealObject",
+		Bucket:    bucket,
+		Object:    object,
+		VersionID: versionID,
+	}
+	if err != nil {
+		opts.Error = err.Error()
+	}
+
+	auditLogInternal(ctx, opts)
+}
+
 // Heals an object by re-writing corrupt/missing erasure blocks.
 func (er *erasureObjects) healObject(ctx context.Context, bucket string, object string, versionID string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) {
 	dryRun := opts.DryRun
@@ -218,6 +237,8 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	storageDisks := er.getDisks()
 	storageEndpoints := er.getEndpoints()
 
+	defer auditHealObject(ctx, bucket, object, versionID, err)
+
 	if globalTrace.NumSubscribers(madmin.TraceHealing) > 0 {
 		startTime := time.Now()
 		defer func() {
@@ -429,7 +450,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(availableDisks) {
 		err := fmt.Errorf("unexpected file distribution (%v) from available disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
 			latestMeta.Erasure.Distribution, availableDisks, bucket, object, versionID)
-		logger.LogOnceIf(ctx, err, "heal-object-available-disks")
+		healingLogOnceIf(ctx, err, "heal-object-available-disks")
 		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
 			bucket, object, versionID), err
 	}
@@ -439,7 +460,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(outDatedDisks) {
 		err := fmt.Errorf("unexpected file distribution (%v) from outdated disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
 			latestMeta.Erasure.Distribution, outDatedDisks, bucket, object, versionID)
-		logger.LogOnceIf(ctx, err, "heal-object-outdated-disks")
+		healingLogOnceIf(ctx, err, "heal-object-outdated-disks")
 		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
 			bucket, object, versionID), err
 	}
@@ -449,7 +470,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(partsMetadata) {
 		err := fmt.Errorf("unexpected file distribution (%v) from metadata entries (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
 			latestMeta.Erasure.Distribution, len(partsMetadata), bucket, object, versionID)
-		logger.LogOnceIf(ctx, err, "heal-object-metadata-entries")
+		healingLogOnceIf(ctx, err, "heal-object-metadata-entries")
 		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
 			bucket, object, versionID), err
 	}
@@ -519,7 +540,10 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 				}
 				partPath := pathJoin(tmpID, dstDataDir, fmt.Sprintf("part.%d", partNumber))
 				if len(inlineBuffers) > 0 {
-					inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, erasure.ShardFileSize(latestMeta.Size)+32))
+					buf := grid.GetByteBufferCap(int(erasure.ShardFileSize(latestMeta.Size)) + 64)
+					inlineBuffers[i] = bytes.NewBuffer(buf[:0])
+					defer grid.PutByteBuffer(buf)
+
 					writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
 				} else {
 					writers[i] = newBitrotWriter(disk, bucket, minioMetaTmpBucket, partPath,

cmd/erasure-healing_test.go

@@ -634,7 +634,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	defer removeRoots(fsDirs)
 
 	// Everything is fine, should return nil
-	objLayer, disks, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...))
+	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -650,7 +650,7 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
 
-	disks = objLayer.(*erasureServerPools).serverPools[0].erasureDisks[0]
+	disks := objLayer.(*erasureServerPools).serverPools[0].erasureDisks[0]
 	orgDisks := append([]StorageAPI{}, disks...)
 
 	// Enable versioning.

cmd/erasure-metadata-utils.go

@@ -22,7 +22,6 @@ import (
 	"errors"
 	"hash/crc32"
 
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/sync/errgroup"
 )
 
@@ -284,7 +283,7 @@ func shuffleDisks(disks []StorageAPI, distribution []int) (shuffledDisks []Stora
 // the corresponding error in errs slice is not nil
 func evalDisks(disks []StorageAPI, errs []error) []StorageAPI {
 	if len(errs) != len(disks) {
-		logger.LogIf(GlobalContext, errors.New("unexpected drives/errors slice length"))
+		bugLogIf(GlobalContext, errors.New("unexpected drives/errors slice length"))
 		return nil
 	}
 	newDisks := make([]StorageAPI, len(disks))

cmd/erasure-metadata.go

@@ -30,7 +30,6 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/sync/errgroup"
 )
 
@@ -268,7 +267,7 @@ func (fi FileInfo) ObjectToPartOffset(ctx context.Context, offset int64) (partIn
 		// Continue to towards the next part.
 		partOffset -= part.Size
 	}
-	logger.LogIf(ctx, InvalidRange{})
+	internalLogIf(ctx, InvalidRange{})
 	// Offset beyond the size of the object return InvalidRange.
 	return 0, 0, InvalidRange{}
 }
@@ -502,14 +501,6 @@ func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []
 		return -1, -1, errErasureReadQuorum
 	}
 
-	if parityBlocks == 0 {
-		// For delete markers do not use 'defaultParityCount' as it is not expected to be the case.
-		// Use maximum allowed read quorum instead, writeQuorum+1 is returned for compatibility sake
-		// but there are no callers that shall be using this.
-		readQuorum := len(partsMetaData) / 2
-		return readQuorum, readQuorum + 1, nil
-	}
-
 	dataBlocks := len(partsMetaData) - parityBlocks
 
 	writeQuorum := dataBlocks

cmd/erasure-multipart.go

@@ -381,7 +381,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		rctx := lkctx.Context()
 		obj, err := er.getObjectInfo(rctx, bucket, object, opts)
 		lk.RUnlock(lkctx)
-		if err != nil && !isErrVersionNotFound(err) {
+		if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
 			return nil, err
 		}
 		if opts.CheckPrecondFn(obj) {
@@ -401,36 +401,33 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		parityDrives = er.defaultParityCount
 	}
 
-	// If we have offline disks upgrade the number of erasure codes for this object.
-	parityOrig := parityDrives
+	if globalStorageClass.AvailabilityOptimized() {
+		// If we have offline disks upgrade the number of erasure codes for this object.
+		parityOrig := parityDrives
 
-	var offlineDrives int
-	for _, disk := range onlineDisks {
-		if disk == nil {
-			parityDrives++
-			offlineDrives++
-			continue
+		var offlineDrives int
+		for _, disk := range onlineDisks {
+			if disk == nil || !disk.IsOnline() {
+				parityDrives++
+				offlineDrives++
+				continue
+			}
 		}
-		if !disk.IsOnline() {
-			parityDrives++
-			offlineDrives++
-			continue
+
+		if offlineDrives >= (len(onlineDisks)+1)/2 {
+			// if offline drives are more than 50% of the drives
+			// we have no quorum, we shouldn't proceed just
+			// fail at that point.
+			return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
 		}
-	}
 
-	if offlineDrives >= (len(onlineDisks)+1)/2 {
-		// if offline drives are more than 50% of the drives
-		// we have no quorum, we shouldn't proceed just
-		// fail at that point.
-		return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
-	}
+		if parityDrives >= len(onlineDisks)/2 {
+			parityDrives = len(onlineDisks) / 2
+		}
 
-	if parityDrives >= len(onlineDisks)/2 {
-		parityDrives = len(onlineDisks) / 2
-	}
-
-	if parityOrig != parityDrives {
-		userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
+		if parityOrig != parityDrives {
+			userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
+		}
 	}
 
 	dataDrives := len(onlineDisks) - parityDrives
@@ -593,7 +590,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	data := r.Reader
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < -1 {
-		logger.LogIf(ctx, errInvalidArgument, logger.ErrorKind)
+		bugLogIf(ctx, errInvalidArgument, logger.ErrorKind)
 		return pi, toObjectErr(errInvalidArgument)
 	}
 
@@ -1029,7 +1026,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	if len(partInfoFiles) != len(parts) {
 		// Should only happen through internal error
 		err := fmt.Errorf("unexpected part result count: %d, want %d", len(partInfoFiles), len(parts))
-		logger.LogIf(ctx, err)
+		bugLogIf(ctx, err)
 		return oi, toObjectErr(err, bucket, object)
 	}
 
@@ -1099,7 +1096,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		_, err := pfi.UnmarshalMsg(part.Data)
 		if err != nil {
 			// Maybe crash or similar.
-			logger.LogIf(ctx, err)
+			bugLogIf(ctx, err)
 			return oi, InvalidPart{
 				PartNumber: partID,
 			}
@@ -1108,7 +1105,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		partI := pfi.Parts[0]
 		partNumber := partI.Number
 		if partID != partNumber {
-			logger.LogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", partID, partID, partI.Number))
+			internalLogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", partID, partID, partI.Number))
 			return oi, InvalidPart{
 				PartNumber: partID,
 			}
@@ -1258,7 +1255,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	}
 
 	// Save the consolidated actual size.
-	fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)
+	if opts.ReplicationRequest {
+		fi.Metadata[ReservedMetadataPrefix+"actual-size"] = opts.UserDefined["X-Minio-Internal-Actual-Object-Size"]
+	} else {
+		fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)
+	}
 
 	if opts.DataMovement {
 		fi.SetDataMov()

cmd/erasure-object.go

@@ -31,6 +31,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/klauspost/readahead"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/tags"
@@ -40,6 +41,7 @@ import (
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/grid"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
@@ -244,6 +246,11 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		}, toObjectErr(errMethodNotAllowed, bucket, object)
 	}
 
+	// Set NoDecryption for SSE-C objects and if replication request
+	if crypto.SSEC.IsEncrypted(objInfo.UserDefined) && opts.ReplicationRequest {
+		opts.NoDecryption = true
+	}
+
 	if objInfo.IsRemote() {
 		gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, h, objInfo, opts)
 		if err != nil {
@@ -921,7 +928,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	if !fi.Deleted && len(fi.Erasure.Distribution) != len(onlineDisks) {
 		err := fmt.Errorf("unexpected file distribution (%v) from online disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
 			fi.Erasure.Distribution, onlineDisks, bucket, object, opts.VersionID)
-		logger.LogOnceIf(ctx, err, "get-object-file-info-manually-modified")
+		storageLogOnceIf(ctx, err, "get-object-file-info-manually-modified")
 		return fi, nil, nil, toObjectErr(err, bucket, object, opts.VersionID)
 	}
 
@@ -1100,7 +1107,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < -1 {
-		logger.LogIf(ctx, errInvalidArgument, logger.ErrorKind)
+		bugLogIf(ctx, errInvalidArgument, logger.ErrorKind)
 		return ObjectInfo{}, toObjectErr(errInvalidArgument)
 	}
 
@@ -1141,7 +1148,6 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 		buffer = buffer[:fi.Erasure.BlockSize]
 	}
 
-	shardFileSize := erasure.ShardFileSize(data.Size())
 	writers := make([]io.Writer, len(onlineDisks))
 	inlineBuffers := make([]*bytes.Buffer, len(onlineDisks))
 	for i, disk := range onlineDisks {
@@ -1149,7 +1155,9 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 			continue
 		}
 		if disk.IsOnline() {
-			inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, shardFileSize))
+			buf := grid.GetByteBufferCap(int(erasure.ShardFileSize(data.Size())) + 64)
+			inlineBuffers[i] = bytes.NewBuffer(buf[:0])
+			defer grid.PutByteBuffer(buf)
 			writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 	}
@@ -1289,7 +1297,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	// Validate input data size and it can never be less than -1.
 	if data.Size() < -1 {
-		logger.LogIf(ctx, errInvalidArgument, logger.ErrorKind)
+		bugLogIf(ctx, errInvalidArgument, logger.ErrorKind)
 		return ObjectInfo{}, toObjectErr(errInvalidArgument)
 	}
 
@@ -1297,25 +1305,21 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	storageDisks := er.getDisks()
 
-	parityDrives := len(storageDisks) / 2
-	if !opts.MaxParity {
-		// Get parity and data drive count based on storage class metadata
-		parityDrives = globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass])
-		if parityDrives < 0 {
-			parityDrives = er.defaultParityCount
-		}
-
+	// Get parity and data drive count based on storage class metadata
+	parityDrives := globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass])
+	if parityDrives < 0 {
+		parityDrives = er.defaultParityCount
+	}
+	if opts.MaxParity {
+		parityDrives = len(storageDisks) / 2
+	}
+	if !opts.MaxParity && globalStorageClass.AvailabilityOptimized() {
 		// If we have offline disks upgrade the number of erasure codes for this object.
 		parityOrig := parityDrives
 
 		var offlineDrives int
 		for _, disk := range storageDisks {
-			if disk == nil {
-				parityDrives++
-				offlineDrives++
-				continue
-			}
-			if !disk.IsOnline() {
+			if disk == nil || !disk.IsOnline() {
 				parityDrives++
 				offlineDrives++
 				continue
@@ -1400,20 +1404,25 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	defer er.deleteAll(context.Background(), minioMetaTmpBucket, tempObj)
 
 	shardFileSize := erasure.ShardFileSize(data.Size())
+	inlineBlock := globalStorageClass.InlineBlock()
+	if inlineBlock <= 0 {
+		inlineBlock = 128 * humanize.KiByte
+	}
+
 	writers := make([]io.Writer, len(onlineDisks))
 	var inlineBuffers []*bytes.Buffer
 	if shardFileSize >= 0 {
-		if !opts.Versioned && shardFileSize < smallFileThreshold {
+		if !opts.Versioned && shardFileSize < inlineBlock {
 			inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
-		} else if shardFileSize < smallFileThreshold/8 {
+		} else if shardFileSize < inlineBlock/8 {
 			inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
 		}
 	} else {
 		// If compressed, use actual size to determine.
 		if sz := erasure.ShardFileSize(data.ActualSize()); sz > 0 {
-			if !opts.Versioned && sz < smallFileThreshold {
+			if !opts.Versioned && sz < inlineBlock {
 				inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
-			} else if sz < smallFileThreshold/8 {
+			} else if sz < inlineBlock/8 {
 				inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
 			}
 		}
@@ -1428,11 +1437,9 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		}
 
 		if len(inlineBuffers) > 0 {
-			sz := shardFileSize
-			if sz < 0 {
-				sz = data.ActualSize()
-			}
-			inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, sz))
+			buf := grid.GetByteBufferCap(int(shardFileSize) + 64)
+			inlineBuffers[i] = bytes.NewBuffer(buf[:0])
+			defer grid.PutByteBuffer(buf)
 			writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
 			continue
 		}
@@ -1441,7 +1448,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	toEncode := io.Reader(data)
-	if data.Size() > bigFileThreshold {
+	if data.Size() >= bigFileThreshold {
 		// We use 2 buffers, so we always have a full buffer of input.
 		bufA := globalBytePoolCap.Get()
 		bufB := globalBytePoolCap.Get()
@@ -1452,7 +1459,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			toEncode = ra
 			defer ra.Close()
 		}
-		logger.LogIf(ctx, err)
+		bugLogIf(ctx, err)
 	}
 	n, erasureErr := erasure.Encode(ctx, toEncode, writers, buffer, writeQuorum)
 	closeBitrotWriters(writers)
@@ -1791,20 +1798,12 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string) error {
 	disks := er.getDisks()
 	g := errgroup.WithNErrs(len(disks))
-	dirPrefix := encodeDirObject(prefix)
 	for index := range disks {
 		index := index
 		g.Go(func() error {
 			if disks[index] == nil {
 				return nil
 			}
-			// Deletes
-			// - The prefix and its children
-			// - The prefix__XLDIR__
-			defer disks[index].Delete(ctx, bucket, dirPrefix, DeleteOptions{
-				Recursive: true,
-				Immediate: true,
-			})
 			return disks[index].Delete(ctx, bucket, prefix, DeleteOptions{
 				Recursive: true,
 				Immediate: true,
@@ -1827,13 +1826,6 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		auditObjectErasureSet(ctx, object, &er)
 	}
 
-	if opts.DeletePrefix {
-		if globalCacheConfig.Enabled() {
-			return ObjectInfo{}, toObjectErr(errMethodNotAllowed, bucket, object)
-		}
-		return ObjectInfo{}, toObjectErr(er.deletePrefix(ctx, bucket, object), bucket, object)
-	}
-
 	var lc *lifecycle.Lifecycle
 	var rcfg lock.Retention
 	var replcfg *replication.Config
@@ -1860,12 +1852,63 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		}
 	}
 
+	if opts.DeletePrefix {
+		if opts.Expiration.Expire {
+			// Expire all versions expiration must still verify the state() on disk
+			// via a getObjectInfo() call as follows, any read quorum issues we
+			// must not proceed further for safety reasons. attempt a MRF heal
+			// while we see such quorum errors.
+			goi, _, gerr := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
+			if gerr != nil && goi.Name == "" {
+				if _, ok := gerr.(InsufficientReadQuorum); ok {
+					// Add an MRF heal for next time.
+					er.addPartial(bucket, object, opts.VersionID)
+
+					return objInfo, InsufficientWriteQuorum{}
+				}
+				return objInfo, gerr
+			}
+
+			// Add protection and re-verify the ILM rules for qualification
+			// based on the latest objectInfo and see if the object still
+			// qualifies for deletion.
+			if gerr == nil {
+				evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi)
+				var isErr bool
+				switch evt.Action {
+				case lifecycle.NoneAction:
+					isErr = true
+				case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+					isErr = true
+				}
+				if isErr {
+					if goi.VersionID != "" {
+						return goi, VersionNotFound{
+							Bucket:    bucket,
+							Object:    object,
+							VersionID: goi.VersionID,
+						}
+					}
+					return goi, ObjectNotFound{
+						Bucket: bucket,
+						Object: object,
+					}
+				}
+			}
+		} // Delete marker and any latest that qualifies shall be expired permanently.
+
+		return ObjectInfo{}, toObjectErr(er.deletePrefix(ctx, bucket, object), bucket, object)
+	}
+
 	storageDisks := er.getDisks()
 	versionFound := true
 	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
 	goi, _, gerr := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
 	if gerr != nil && goi.Name == "" {
 		if _, ok := gerr.(InsufficientReadQuorum); ok {
+			// Add an MRF heal for next time.
+			er.addPartial(bucket, object, opts.VersionID)
+
 			return objInfo, InsufficientWriteQuorum{}
 		}
 		// For delete marker replication, versionID being replicated will not exist on disk
@@ -1875,6 +1918,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 			return objInfo, gerr
 		}
 	}
+
 	if opts.EvalMetadataFn != nil {
 		dsc, err := opts.EvalMetadataFn(&goi, err)
 		if err != nil {
@@ -2345,7 +2389,7 @@ func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, obje
 	}, ObjectOptions{
 		VersionID: oi.VersionID,
 	}); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err))
+		storageLogIf(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err))
 		return err
 	}
 	return nil

cmd/erasure-server-pool-decom.go

@@ -72,9 +72,6 @@ func (pd *PoolDecommissionInfo) Clone() *PoolDecommissionInfo {
 	if pd == nil {
 		return nil
 	}
-	if pd.StartTime.IsZero() {
-		return nil
-	}
 	return &PoolDecommissionInfo{
 		StartTime:               pd.StartTime,
 		StartSize:               pd.StartSize,
@@ -97,7 +94,7 @@ func (pd *PoolDecommissionInfo) Clone() *PoolDecommissionInfo {
 
 // bucketPop should be called when a bucket is done decommissioning.
 // Adds the bucket to the list of decommissioned buckets and updates resume numbers.
-func (pd *PoolDecommissionInfo) bucketPop(bucket string) {
+func (pd *PoolDecommissionInfo) bucketPop(bucket string) bool {
 	pd.DecommissionedBuckets = append(pd.DecommissionedBuckets, bucket)
 	for i, b := range pd.QueuedBuckets {
 		if b == bucket {
@@ -109,9 +106,10 @@ func (pd *PoolDecommissionInfo) bucketPop(bucket string) {
 				pd.Prefix = "" // empty this out for the next bucket
 				pd.Object = "" // empty this out for next object
 			}
-			return
+			return true
 		}
 	}
+	return false
 }
 
 func (pd *PoolDecommissionInfo) isBucketDecommissioned(bucket string) bool {
@@ -222,12 +220,12 @@ func (p poolMeta) isBucketDecommissioned(idx int, bucket string) bool {
 	return p.Pools[idx].Decommission.isBucketDecommissioned(bucket)
 }
 
-func (p *poolMeta) BucketDone(idx int, bucket decomBucketInfo) {
+func (p *poolMeta) BucketDone(idx int, bucket decomBucketInfo) bool {
 	if p.Pools[idx].Decommission == nil {
 		// Decommission not in progress.
-		return
+		return false
 	}
-	p.Pools[idx].Decommission.bucketPop(bucket.String())
+	return p.Pools[idx].Decommission.bucketPop(bucket.String())
 }
 
 func (p poolMeta) ResumeBucketObject(idx int) (bucket, object string) {
@@ -469,7 +467,7 @@ func (p poolMeta) save(ctx context.Context, pools []*erasureSets) error {
 	for i, eset := range pools {
 		if err = saveConfig(ctx, eset, poolMetaName, buf); err != nil {
 			if !errors.Is(err, context.Canceled) {
-				logger.LogIf(ctx, fmt.Errorf("saving pool.bin for pool index %d failed with: %v", i, err))
+				storageLogIf(ctx, fmt.Errorf("saving pool.bin for pool index %d failed with: %v", i, err))
 			}
 			return err
 		}
@@ -544,11 +542,11 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 						return
 					}
 					if configRetriableErrors(err) {
-						logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pools %v: %w: retrying..", pools, err))
+						decomLogIf(ctx, fmt.Errorf("Unable to resume decommission of pools %v: %w: retrying..", pools, err))
 						time.Sleep(time.Second + time.Duration(r.Float64()*float64(5*time.Second)))
 						continue
 					}
-					logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pools, err))
+					decomLogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pools, err))
 					return
 				}
 			}
@@ -743,7 +741,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 	const envDecomWorkers = "_MINIO_DECOMMISSION_WORKERS"
 	workerSize, err := env.GetInt(envDecomWorkers, len(pool.sets))
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("invalid workers value err: %v, defaulting to %d", err, len(pool.sets)))
+		decomLogIf(ctx, fmt.Errorf("invalid workers value err: %v, defaulting to %d", err, len(pool.sets)))
 		workerSize = len(pool.sets)
 	}
 
@@ -854,7 +852,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					}
 					stopFn(err)
 					if err != nil {
-						logger.LogIf(ctx, err)
+						decomLogIf(ctx, err)
 						failure = true
 					}
 					z.poolMetaMutex.Lock()
@@ -879,7 +877,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						}); err != nil {
 							stopFn(err)
 							failure = true
-							logger.LogIf(ctx, err)
+							decomLogIf(ctx, err)
 							continue
 						}
 						stopFn(nil)
@@ -908,20 +906,20 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						if bi.Name == minioMetaBucket && strings.Contains(version.Name, dataUsageCacheName) {
 							ignore = true
 							stopFn(err)
-							logger.LogIf(ctx, err)
+							decomLogIf(ctx, err)
 							break
 						}
 					}
 					if err != nil {
 						failure = true
-						logger.LogIf(ctx, err)
+						decomLogIf(ctx, err)
 						stopFn(err)
 						continue
 					}
 					if err = z.decommissionObject(ctx, bi.Name, gr); err != nil {
 						stopFn(err)
 						failure = true
-						logger.LogIf(ctx, err)
+						decomLogIf(ctx, err)
 						continue
 					}
 					stopFn(nil)
@@ -955,13 +953,13 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				stopFn(err)
 				auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err)
 				if err != nil {
-					logger.LogIf(ctx, err)
+					decomLogIf(ctx, err)
 				}
 			}
 			z.poolMetaMutex.Lock()
 			z.poolMeta.TrackCurrentBucketObject(idx, bi.Name, entry.name)
 			ok, err := z.poolMeta.updateAfter(ctx, idx, z.serverPools, 30*time.Second)
-			logger.LogIf(ctx, err)
+			decomLogIf(ctx, err)
 			if ok {
 				globalNotificationSys.ReloadPoolMeta(ctx)
 			}
@@ -989,7 +987,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				}
 				setN := humanize.Ordinal(setIdx + 1)
 				retryDur := time.Duration(rand.Float64() * float64(5*time.Second))
-				logger.LogOnceIf(ctx, fmt.Errorf("listing objects from %s set failed with %v, retrying in %v", setN, err, retryDur), "decom-listing-failed"+setN)
+				decomLogOnceIf(ctx, fmt.Errorf("listing objects from %s set failed with %v, retrying in %v", setN, err, retryDur), "decom-listing-failed"+setN)
 				time.Sleep(retryDur)
 			}
 		}(setIdx)
@@ -1055,8 +1053,10 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 			}
 
 			z.poolMetaMutex.Lock()
-			z.poolMeta.BucketDone(idx, bucket) // remove from pendingBuckets and persist.
-			z.poolMeta.save(ctx, z.serverPools)
+			if z.poolMeta.BucketDone(idx, bucket) {
+				// remove from pendingBuckets and persist.
+				decomLogIf(ctx, z.poolMeta.save(ctx, z.serverPools))
+			}
 			z.poolMetaMutex.Unlock()
 			continue
 		}
@@ -1071,8 +1071,9 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 		stopFn(nil)
 
 		z.poolMetaMutex.Lock()
-		z.poolMeta.BucketDone(idx, bucket)
-		z.poolMeta.save(ctx, z.serverPools)
+		if z.poolMeta.BucketDone(idx, bucket) {
+			decomLogIf(ctx, z.poolMeta.save(ctx, z.serverPools))
+		}
 		z.poolMetaMutex.Unlock()
 	}
 	return nil
@@ -1169,8 +1170,8 @@ func (z *erasureServerPools) doDecommissionInRoutine(ctx context.Context, idx in
 	dctx = logger.SetReqInfo(dctx, &logger.ReqInfo{})
 
 	if err := z.decommissionInBackground(dctx, idx); err != nil {
-		logger.LogIf(GlobalContext, err)
-		logger.LogIf(GlobalContext, z.DecommissionFailed(dctx, idx))
+		decomLogIf(GlobalContext, err)
+		decomLogIf(GlobalContext, z.DecommissionFailed(dctx, idx))
 		return
 	}
 
@@ -1180,20 +1181,20 @@ func (z *erasureServerPools) doDecommissionInRoutine(ctx context.Context, idx in
 	z.poolMetaMutex.Unlock()
 
 	if !failed {
-		logger.Event(dctx, "Decommissioning complete for pool '%s', verifying for any pending objects", poolCmdLine)
+		decomLogEvent(dctx, "Decommissioning complete for pool '%s', verifying for any pending objects", poolCmdLine)
 		err := z.checkAfterDecom(dctx, idx)
 		if err != nil {
-			logger.LogIf(ctx, err)
+			decomLogIf(ctx, err)
 			failed = true
 		}
 	}
 
 	if failed {
 		// Decommission failed indicate as such.
-		logger.LogIf(GlobalContext, z.DecommissionFailed(dctx, idx))
+		decomLogIf(GlobalContext, z.DecommissionFailed(dctx, idx))
 	} else {
 		// Complete the decommission..
-		logger.LogIf(GlobalContext, z.CompleteDecommission(dctx, idx))
+		decomLogIf(GlobalContext, z.CompleteDecommission(dctx, idx))
 	}
 }
 
@@ -1403,16 +1404,18 @@ func (z *erasureServerPools) getBucketsToDecommission(ctx context.Context) ([]de
 
 	// Buckets data are dispersed in multiple zones/sets, make
 	// sure to decommission the necessary metadata.
-	decomBuckets = append(decomBuckets, decomBucketInfo{
-		Name:   minioMetaBucket,
-		Prefix: minioConfigPrefix,
-	})
-	decomBuckets = append(decomBuckets, decomBucketInfo{
-		Name:   minioMetaBucket,
-		Prefix: bucketMetaPrefix,
-	})
+	decomMetaBuckets := []decomBucketInfo{
+		{
+			Name:   minioMetaBucket,
+			Prefix: minioConfigPrefix,
+		},
+		{
+			Name:   minioMetaBucket,
+			Prefix: bucketMetaPrefix,
+		},
+	}
 
-	return decomBuckets, nil
+	return append(decomMetaBuckets, decomBuckets...), nil
 }
 
 func (z *erasureServerPools) StartDecommission(ctx context.Context, indices ...int) (err error) {

cmd/erasure-server-pool-rebalance.go

@@ -146,7 +146,7 @@ func (z *erasureServerPools) updateRebalanceStats(ctx context.Context) error {
 		lock := z.serverPools[0].NewNSLock(minioMetaBucket, rebalMetaName)
 		lkCtx, err := lock.GetLock(ctx, globalOperationTimeout)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("failed to acquire write lock on %s/%s: %w", minioMetaBucket, rebalMetaName, err))
+			rebalanceLogIf(ctx, fmt.Errorf("failed to acquire write lock on %s/%s: %w", minioMetaBucket, rebalMetaName, err))
 			return err
 		}
 		defer lock.Unlock(lkCtx)
@@ -423,7 +423,7 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
 			stopFn := globalRebalanceMetrics.log(rebalanceMetricSaveMetadata, poolIdx, traceMsg)
 			err := z.saveRebalanceStats(ctx, poolIdx, rebalSaveStats)
 			stopFn(err)
-			logger.LogIf(ctx, err)
+			rebalanceLogIf(ctx, err)
 			timer.Reset(randSleepFor())
 
 			if rebalDone {
@@ -432,7 +432,7 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
 		}
 	}()
 
-	logger.Event(ctx, "Pool %d rebalancing is started", poolIdx+1)
+	rebalanceLogEvent(ctx, "Pool %d rebalancing is started", poolIdx+1)
 
 	for {
 		select {
@@ -451,14 +451,14 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
 		err = z.rebalanceBucket(ctx, bucket, poolIdx)
 		if err != nil {
 			stopFn(err)
-			logger.LogIf(ctx, err)
+			rebalanceLogIf(ctx, err)
 			return
 		}
 		stopFn(nil)
 		z.bucketRebalanceDone(bucket, poolIdx)
 	}
 
-	logger.Event(ctx, "Pool %d rebalancing is done", poolIdx+1)
+	rebalanceLogEvent(ctx, "Pool %d rebalancing is done", poolIdx+1)
 
 	return err
 }
@@ -535,7 +535,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 	const envRebalanceWorkers = "_MINIO_REBALANCE_WORKERS"
 	workerSize, err := env.GetInt(envRebalanceWorkers, len(pool.sets))
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("invalid workers value err: %v, defaulting to %d", err, len(pool.sets)))
+		rebalanceLogIf(ctx, fmt.Errorf("invalid workers value err: %v, defaulting to %d", err, len(pool.sets)))
 		workerSize = len(pool.sets)
 	}
 
@@ -630,7 +630,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 						})
 					var failure bool
 					if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-						logger.LogIf(ctx, err)
+						rebalanceLogIf(ctx, err)
 						failure = true
 					}
 
@@ -665,14 +665,14 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 					}
 					if err != nil {
 						failure = true
-						logger.LogIf(ctx, err)
+						rebalanceLogIf(ctx, err)
 						stopFn(err)
 						continue
 					}
 
 					if err = z.rebalanceObject(ctx, bucket, gr); err != nil {
 						failure = true
-						logger.LogIf(ctx, err)
+						rebalanceLogIf(ctx, err)
 						stopFn(err)
 						continue
 					}
@@ -706,7 +706,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 				stopFn(err)
 				auditLogRebalance(ctx, "Rebalance:DeleteObject", bucket, entry.name, "", err)
 				if err != nil {
-					logger.LogIf(ctx, err)
+					rebalanceLogIf(ctx, err)
 				}
 			}
 		}
@@ -724,7 +724,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 				return
 			}
 			setN := humanize.Ordinal(setIdx + 1)
-			logger.LogOnceIf(ctx, fmt.Errorf("listing objects from %s set failed with %v", setN, err), "rebalance-listing-failed"+setN)
+			rebalanceLogIf(ctx, fmt.Errorf("listing objects from %s set failed with %v", setN, err), "rebalance-listing-failed"+setN)
 		}(setIdx)
 	}
 
@@ -743,7 +743,7 @@ func (z *erasureServerPools) saveRebalanceStats(ctx context.Context, poolIdx int
 	lock := z.serverPools[0].NewNSLock(minioMetaBucket, rebalMetaName)
 	lkCtx, err := lock.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("failed to acquire write lock on %s/%s: %w", minioMetaBucket, rebalMetaName, err))
+		rebalanceLogIf(ctx, fmt.Errorf("failed to acquire write lock on %s/%s: %w", minioMetaBucket, rebalMetaName, err))
 		return err
 	}
 	defer lock.Unlock(lkCtx)

cmd/erasure-server-pool-rebalance_gen.go

@@ -614,6 +614,7 @@ func (z *rebalanceMetrics) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z rebalanceMetrics) EncodeMsg(en *msgp.Writer) (err error) {
 	// map header, size 0
+	_ = z
 	err = en.Append(0x80)
 	if err != nil {
 		return
@@ -625,6 +626,7 @@ func (z rebalanceMetrics) EncodeMsg(en *msgp.Writer) (err error) {
 func (z rebalanceMetrics) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// map header, size 0
+	_ = z
 	o = append(o, 0x80)
 	return
 }

cmd/erasure-server-pool.go

@@ -92,15 +92,15 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		n = 2048
 	}
 
-	if globalIsCICD {
-		n = 256 // 256MiB for CI/CD environments is sufficient
-	}
-
 	// Avoid allocating more than half of the available memory
 	if maxN := availableMemory() / (blockSizeV2 * 2); n > maxN {
 		n = maxN
 	}
 
+	if globalIsCICD || strconv.IntSize == 32 {
+		n = 256 // 256MiB for CI/CD environments is sufficient or on 32bit platforms.
+	}
+
 	// Initialize byte pool once for all sets, bpool size is set to
 	// setCount * setDriveCount with each memory upto blockSizeV2.
 	globalBytePoolCap = bpool.NewBytePoolCap(n, blockSizeV2, blockSizeV2*2)
@@ -207,7 +207,7 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 				logger.Fatal(err, "Unable to initialize backend")
 			}
 			retry := time.Duration(r.Float64() * float64(5*time.Second))
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize backend: %w, retrying in %s", err, retry))
+			storageLogIf(ctx, fmt.Errorf("Unable to initialize backend: %w, retrying in %s", err, retry))
 			time.Sleep(retry)
 			attempt++
 			continue
@@ -376,7 +376,7 @@ func (z *erasureServerPools) getAvailablePoolIdx(ctx context.Context, bucket, ob
 		}
 	}
 	// Should not happen, but print values just in case.
-	logger.LogIf(ctx, fmt.Errorf("reached end of serverPools (total: %v, atTotal: %v, choose: %v)", total, atTotal, choose))
+	storageLogIf(ctx, fmt.Errorf("reached end of serverPools (total: %v, atTotal: %v, choose: %v)", total, atTotal, choose))
 	return -1
 }
 
@@ -610,7 +610,7 @@ func (z *erasureServerPools) Shutdown(ctx context.Context) error {
 
 	for _, err := range g.Wait() {
 		if err != nil {
-			logger.LogIf(ctx, err)
+			storageLogIf(ctx, err)
 		}
 		// let's the rest shutdown
 	}
@@ -688,12 +688,17 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataU
 		updates <- DataUsageInfo{} // no buckets found update data usage to reflect latest state
 		return nil
 	}
-
+	totalResults := 0
+	resultIndex := -1
+	for _, z := range z.serverPools {
+		totalResults += len(z.sets)
+	}
+	results = make([]dataUsageCache, totalResults)
 	// Collect for each set in serverPools.
 	for _, z := range z.serverPools {
 		for _, erObj := range z.sets {
+			resultIndex++
 			wg.Add(1)
-			results = append(results, dataUsageCache{})
 			go func(i int, erObj *erasureObjects) {
 				updates := make(chan dataUsageCache, 1)
 				defer xioutil.SafeClose(updates)
@@ -709,7 +714,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataU
 				// Start scanner. Blocks until done.
 				err := erObj.nsScanner(ctx, allBuckets, wantCycle, updates, healScanMode)
 				if err != nil {
-					logger.LogIf(ctx, err)
+					scannerLogIf(ctx, err)
 					mu.Lock()
 					if firstErr == nil {
 						firstErr = err
@@ -719,7 +724,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataU
 					mu.Unlock()
 					return
 				}
-			}(len(results)-1, erObj)
+			}(resultIndex, erObj)
 		}
 	}
 	updateCloser := make(chan chan struct{})
@@ -1266,13 +1271,17 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 	return z.serverPools[poolIdx].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
 }
 
+func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+	return z.listObjectsGeneric(ctx, bucket, prefix, marker, delimiter, maxKeys, true)
+}
+
 func (z *erasureServerPools) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (ListObjectsV2Info, error) {
 	marker := continuationToken
 	if marker == "" {
 		marker = startAfter
 	}
 
-	loi, err := z.ListObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
+	loi, err := z.listObjectsGeneric(ctx, bucket, prefix, marker, delimiter, maxKeys, false)
 	if err != nil {
 		return ListObjectsV2Info{}, err
 	}
@@ -1320,7 +1329,7 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 	merged, err := z.listPath(ctx, &opts)
 	if err != nil && err != io.EOF {
 		if !isErrBucketNotFound(err) {
-			logger.LogOnceIf(ctx, err, "erasure-list-objects-path-"+bucket)
+			storageLogOnceIf(ctx, err, "erasure-list-objects-path-"+bucket)
 		}
 		return loi, toObjectErr(err, bucket)
 	}
@@ -1381,9 +1390,10 @@ func maxKeysPlusOne(maxKeys int, addOne bool) int {
 	return maxKeys
 }
 
-func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
+func (z *erasureServerPools) listObjectsGeneric(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int, v1 bool) (ListObjectsInfo, error) {
 	var loi ListObjectsInfo
 	opts := listPathOptions{
+		V1:          v1,
 		Bucket:      bucket,
 		Prefix:      prefix,
 		Separator:   delimiter,
@@ -1513,7 +1523,7 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	merged, err := z.listPath(ctx, &opts)
 	if err != nil && err != io.EOF {
 		if !isErrBucketNotFound(err) {
-			logger.LogOnceIf(ctx, err, "erasure-list-objects-path-"+bucket)
+			storageLogOnceIf(ctx, err, "erasure-list-objects-path-"+bucket)
 		}
 		return loi, toObjectErr(err, bucket)
 	}
@@ -1554,8 +1564,23 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	}
 	if loi.IsTruncated {
 		last := objects[len(objects)-1]
-		loi.NextMarker = opts.encodeMarker(last.Name)
+		loi.NextMarker = last.Name
 	}
+
+	if merged.lastSkippedEntry != "" {
+		if merged.lastSkippedEntry > loi.NextMarker {
+			// An object hidden by ILM was found during listing. Since the number of entries
+			// fetched from drives is limited, set IsTruncated to true to ask the s3 client
+			// to continue listing if it wishes in order to find if there is more objects.
+			loi.IsTruncated = true
+			loi.NextMarker = merged.lastSkippedEntry
+		}
+	}
+
+	if loi.NextMarker != "" {
+		loi.NextMarker = opts.encodeMarker(loi.NextMarker)
+	}
+
 	return loi, nil
 }
 
@@ -1920,7 +1945,7 @@ func (z *erasureServerPools) HealFormat(ctx context.Context, dryRun bool) (madmi
 	for _, pool := range z.serverPools {
 		result, err := pool.HealFormat(ctx, dryRun)
 		if err != nil && !errors.Is(err, errNoHealRequired) {
-			logger.LogOnceIf(ctx, err, "erasure-heal-format")
+			healingLogOnceIf(ctx, err, "erasure-heal-format")
 			continue
 		}
 		// Count errNoHealRequired across all serverPools,
@@ -2111,7 +2136,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 					}
 
 					if err := listPathRaw(ctx, lopts); err != nil {
-						logger.LogIf(ctx, fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts))
+						storageLogIf(ctx, fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts))
 						cancel()
 						return
 					}
@@ -2156,7 +2181,7 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 		if opts.Remove && !opts.DryRun {
 			err := z.CheckAbandonedParts(ctx, bucket, entry.name, opts)
 			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", bucket, entry.name, err))
+				healingLogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", bucket, entry.name, err))
 			}
 		}
 		for _, version := range fivs.Versions {
@@ -2360,7 +2385,7 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 	// Check if disks are healing on in-case of VMware vsphere deployments.
 	if opts.Maintenance && opts.DeploymentType == vmware {
 		if drivesHealing > 0 {
-			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Total drives to be healed %d", drivesHealing))
+			healingLogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Total drives to be healed %d", drivesHealing))
 		}
 	}
 
@@ -2420,15 +2445,15 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 
 			healthy := erasureSetUpCount[poolIdx][setIdx].online >= poolWriteQuorums[poolIdx]
 			if !healthy {
-				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
+				storageLogIf(logger.SetReqInfo(ctx, reqInfo),
 					fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
-						poolIdx, setIdx, poolWriteQuorums[poolIdx]))
+						poolIdx, setIdx, poolWriteQuorums[poolIdx]), logger.FatalKind)
 			}
 			result.Healthy = result.Healthy && healthy
 
 			healthyRead := erasureSetUpCount[poolIdx][setIdx].online >= poolReadQuorums[poolIdx]
 			if !healthyRead {
-				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
+				storageLogIf(logger.SetReqInfo(ctx, reqInfo),
 					fmt.Errorf("Read quorum may be lost on pool: %d, set: %d, expected read quorum: %d",
 						poolIdx, setIdx, poolReadQuorums[poolIdx]))
 			}

cmd/erasure-sets.go

@@ -448,7 +448,7 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 					diskID, err := disk.GetDiskID()
 					if err != nil {
 						if !errors.Is(err, errUnformattedDisk) {
-							logger.LogIf(ctx, err)
+							bootLogIf(ctx, err)
 						}
 						return
 					}
@@ -457,11 +457,11 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 					}
 					m, n, err := findDiskIndexByDiskID(format, diskID)
 					if err != nil {
-						logger.LogIf(ctx, err)
+						bootLogIf(ctx, err)
 						return
 					}
 					if m != i || n != j {
-						logger.LogIf(ctx, fmt.Errorf("Detected unexpected drive ordering refusing to use the drive - poolID: %s, found drive mounted at (set=%s, drive=%s) expected mount at (set=%s, drive=%s): %s(%s)", humanize.Ordinal(poolIdx+1), humanize.Ordinal(m+1), humanize.Ordinal(n+1), humanize.Ordinal(i+1), humanize.Ordinal(j+1), disk, diskID))
+						bootLogIf(ctx, fmt.Errorf("Detected unexpected drive ordering refusing to use the drive - poolID: %s, found drive mounted at (set=%s, drive=%s) expected mount at (set=%s, drive=%s): %s(%s)", humanize.Ordinal(poolIdx+1), humanize.Ordinal(m+1), humanize.Ordinal(n+1), humanize.Ordinal(i+1), humanize.Ordinal(j+1), disk, diskID))
 						s.erasureDisks[i][j] = &unrecognizedDisk{storage: disk}
 						return
 					}
@@ -1083,7 +1083,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 
 	if !reflect.DeepEqual(s.format, refFormat) {
 		// Format is corrupted and unrecognized by the running instance.
-		logger.LogIf(ctx, fmt.Errorf("Unable to heal the newly replaced drives due to format.json inconsistencies, please engage MinIO support for further assistance: %w",
+		healingLogIf(ctx, fmt.Errorf("Unable to heal the newly replaced drives due to format.json inconsistencies, please engage MinIO support for further assistance: %w",
 			errCorruptedFormat))
 		return res, errCorruptedFormat
 	}
@@ -1112,7 +1112,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 				continue
 			}
 			if err := saveFormatErasure(storageDisks[index], format, formatOpID); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Drive %s failed to write updated 'format.json': %v", storageDisks[index], err))
+				healingLogIf(ctx, fmt.Errorf("Drive %s failed to write updated 'format.json': %v", storageDisks[index], err))
 				storageDisks[index].Close()
 				tmpNewFormats[index] = nil // this disk failed to write new format
 			}
@@ -1127,7 +1127,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 
 			m, n, err := findDiskIndexByDiskID(refFormat, format.Erasure.This)
 			if err != nil {
-				logger.LogIf(ctx, err)
+				healingLogIf(ctx, err)
 				continue
 			}
 

cmd/erasure.go

@@ -31,7 +31,6 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/dsync"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/sync/errgroup"
 )
 
@@ -384,7 +383,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 	// Collect disks we can use.
 	disks, healing := er.getOnlineDisksWithHealing(false)
 	if len(disks) == 0 {
-		logger.LogIf(ctx, errors.New("data-scanner: all drives are offline or being healed, skipping scanner cycle"))
+		scannerLogIf(ctx, errors.New("data-scanner: all drives are offline or being healed, skipping scanner cycle"))
 		return nil
 	}
 
@@ -449,7 +448,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				if cache.Info.LastUpdate.Equal(lastSave) {
 					continue
 				}
-				logger.LogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-cache-update")
+				scannerLogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-cache-update")
 				updates <- cache.clone()
 
 				lastSave = cache.Info.LastUpdate
@@ -458,7 +457,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 					// Save final state...
 					cache.Info.NextCycle = wantCycle
 					cache.Info.LastUpdate = time.Now()
-					logger.LogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-channel-closed")
+					scannerLogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-channel-closed")
 					updates <- cache.clone()
 					return
 				}
@@ -494,7 +493,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				// Load cache for bucket
 				cacheName := pathJoin(bucket.Name, dataUsageCacheName)
 				cache := dataUsageCache{}
-				logger.LogIf(ctx, cache.load(ctx, er, cacheName))
+				scannerLogIf(ctx, cache.load(ctx, er, cacheName))
 				if cache.Info.Name == "" {
 					cache.Info.Name = bucket.Name
 				}
@@ -530,9 +529,9 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				cache, err = disk.NSScanner(ctx, cache, updates, healScanMode, nil)
 				if err != nil {
 					if !cache.Info.LastUpdate.IsZero() && cache.Info.LastUpdate.After(before) {
-						logger.LogIf(ctx, cache.save(ctx, er, cacheName))
+						scannerLogIf(ctx, cache.save(ctx, er, cacheName))
 					} else {
-						logger.LogIf(ctx, err)
+						scannerLogIf(ctx, err)
 					}
 					// This ensures that we don't close
 					// bucketResults channel while the
@@ -543,9 +542,13 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				}
 
 				wg.Wait()
+				// Flatten for upstream, but save full state.
 				var root dataUsageEntry
 				if r := cache.root(); r != nil {
 					root = cache.flatten(*r)
+					if root.ReplicationStats.empty() {
+						root.ReplicationStats = nil
+					}
 				}
 				select {
 				case <-ctx.Done():
@@ -558,7 +561,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				}
 
 				// Save cache
-				logger.LogIf(ctx, cache.save(ctx, er, cacheName))
+				scannerLogIf(ctx, cache.save(ctx, er, cacheName))
 			}
 		}(i)
 	}

cmd/etcd.go

@@ -22,7 +22,6 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/minio/minio/internal/logger"
 	etcd "go.etcd.io/etcd/client/v3"
 )
 
@@ -48,7 +47,7 @@ func saveKeyEtcdWithTTL(ctx context.Context, client *etcd.Client, key string, da
 		return etcdErrToErr(err, client.Endpoints())
 	}
 	_, err = client.Put(timeoutCtx, key, string(data), etcd.WithLease(lease.ID))
-	logger.LogIf(ctx, err)
+	etcdLogIf(ctx, err)
 	return etcdErrToErr(err, client.Endpoints())
 }
 
@@ -59,7 +58,7 @@ func saveKeyEtcd(ctx context.Context, client *etcd.Client, key string, data []by
 		return saveKeyEtcdWithTTL(ctx, client, key, data, opts[0].ttl)
 	}
 	_, err := client.Put(timeoutCtx, key, string(data))
-	logger.LogIf(ctx, err)
+	etcdLogIf(ctx, err)
 	return etcdErrToErr(err, client.Endpoints())
 }
 
@@ -68,7 +67,7 @@ func deleteKeyEtcd(ctx context.Context, client *etcd.Client, key string) error {
 	defer cancel()
 
 	_, err := client.Delete(timeoutCtx, key)
-	logger.LogIf(ctx, err)
+	etcdLogIf(ctx, err)
 	return etcdErrToErr(err, client.Endpoints())
 }
 
@@ -77,7 +76,7 @@ func readKeyEtcd(ctx context.Context, client *etcd.Client, key string) ([]byte,
 	defer cancel()
 	resp, err := client.Get(timeoutCtx, key)
 	if err != nil {
-		logger.LogOnceIf(ctx, err, "etcd-retrieve-keys")
+		etcdLogOnceIf(ctx, err, "etcd-retrieve-keys")
 		return nil, etcdErrToErr(err, client.Endpoints())
 	}
 	if resp.Count == 0 {

cmd/event-notification.go

@@ -28,7 +28,6 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/pubsub"
 	"github.com/minio/pkg/v2/policy"
 )
@@ -83,7 +82,7 @@ func (evnot *EventNotifier) set(bucket BucketInfo, meta BucketMetadata) {
 	config.SetRegion(globalSite.Region)
 	if err := config.Validate(globalSite.Region, globalEventNotifier.targetList); err != nil {
 		if _, ok := err.(*event.ErrARNNotFound); !ok {
-			logger.LogIf(GlobalContext, err)
+			internalLogIf(GlobalContext, err)
 		}
 	}
 	evnot.AddRulesMap(bucket.Name, config.ToRulesMap())

cmd/format-erasure.go

@@ -278,7 +278,7 @@ func formatErasureMigrateV2ToV3(data []byte, export, version string) ([]byte, er
 	tmpOld := pathJoin(export, minioMetaTmpDeletedBucket, mustGetUUID())
 	if err := renameAll(pathJoin(export, minioMetaMultipartBucket),
 		tmpOld, export); err != nil && err != errFileNotFound {
-		logger.LogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate",
+		bootLogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate",
 			pathJoin(export, minioMetaMultipartBucket),
 			tmpOld,
 			osErrToFileErr(err)))
@@ -570,7 +570,7 @@ func formatErasureFixLocalDeploymentID(endpoints Endpoints, storageDisks []Stora
 				format.ID = refFormat.ID
 				// Heal the drive if we fixed its deployment ID.
 				if err := saveFormatErasure(storageDisks[index], format, mustGetUUID()); err != nil {
-					logger.LogIf(GlobalContext, err)
+					bootLogIf(GlobalContext, err)
 					return fmt.Errorf("Unable to save format.json, %w", err)
 				}
 			}

cmd/ftp-server-driver.go

@@ -33,7 +33,6 @@ import (
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio/internal/auth"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	ftp "goftp.io/server/v2"
 )
 
@@ -323,7 +322,7 @@ func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error)
 			}
 
 			// Call hook for site replication.
-			logger.LogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{
+			replLogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{
 				Type: madmin.SRIAMItemSTSAcc,
 				STSCredential: &madmin.SRSTSCredential{
 					AccessKey:    cred.AccessKey,

cmd/ftp-server.go

@@ -136,7 +136,7 @@ func startFTPServer(args []string) {
 
 	ftpServer, err := ftp.NewServer(&ftp.Options{
 		Name:           name,
-		WelcomeMessage: fmt.Sprintf("Welcome to MinIO FTP Server Version='%s' License='GNU AGPLv3'", Version),
+		WelcomeMessage: fmt.Sprintf("Welcome to '%s' FTP Server Version='%s' License='%s'", MinioStoreName, MinioLicense, Version),
 		Driver:         NewFTPDriver(),
 		Port:           port,
 		Perm:           ftp.NewSimplePerm("nobody", "nobody"),

cmd/generic-handlers.go

@@ -33,6 +33,8 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/grid"
 	xnet "github.com/minio/pkg/v2/net"
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
 
 	"github.com/minio/minio/internal/amztime"
 	"github.com/minio/minio/internal/config/dns"
@@ -73,6 +75,9 @@ const (
 // and must not set by clients
 func containsReservedMetadata(header http.Header) bool {
 	for key := range header {
+		if slices.Contains(maps.Keys(validSSEReplicationHeaders), key) {
+			return false
+		}
 		if stringsHasPrefixFold(key, ReservedMetadataPrefix) {
 			return true
 		}
@@ -233,7 +238,8 @@ func guessIsMetricsReq(req *http.Request) bool {
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ClusterPath ||
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2NodePath ||
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2BucketPath ||
-		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ResourcePath
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ResourcePath ||
+		strings.HasPrefix(req.URL.Path, minioReservedBucketPath+metricsV3Path)
 }
 
 // guessIsRPCReq - returns true if the request is for an RPC endpoint.

cmd/generic-handlers_test.go

@@ -108,15 +108,15 @@ var containsReservedMetadataTests = []struct {
 	},
 	{
 		header:     http.Header{crypto.MetaIV: []string{"iv"}},
-		shouldFail: true,
+		shouldFail: false,
 	},
 	{
 		header:     http.Header{crypto.MetaAlgorithm: []string{crypto.InsecureSealAlgorithm}},
-		shouldFail: true,
+		shouldFail: false,
 	},
 	{
 		header:     http.Header{crypto.MetaSealedKeySSEC: []string{"mac"}},
-		shouldFail: true,
+		shouldFail: false,
 	},
 	{
 		header:     http.Header{ReservedMetadataPrefix + "Key": []string{"value"}},

cmd/global-heal.go

@@ -153,7 +153,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		_, err := objAPI.HealBucket(ctx, bucket, madmin.HealOpts{ScanMode: scanMode})
 		if err != nil {
 			// Log bucket healing error if any, we shall retry again.
-			logger.LogIf(ctx, err)
+			healingLogIf(ctx, err)
 		}
 	}
 
@@ -177,7 +177,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		numHealers = uint64(v)
 	}
 
-	logger.Event(ctx, fmt.Sprintf("Healing drive '%s' - use %d parallel workers.", tracker.disk.String(), numHealers))
+	healingLogEvent(ctx, fmt.Sprintf("Healing drive '%s' - use %d parallel workers.", tracker.disk.String(), numHealers))
 
 	jt, _ := workers.New(int(numHealers))
 
@@ -204,7 +204,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		if _, err := objAPI.HealBucket(ctx, bucket, madmin.HealOpts{
 			ScanMode: scanMode,
 		}); err != nil {
-			logger.LogIf(ctx, err)
+			healingLogIf(ctx, err)
 			continue
 		}
 
@@ -226,7 +226,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		if len(disks) == 0 {
 			// No object healing necessary
 			tracker.bucketDone(bucket)
-			logger.LogIf(ctx, tracker.update(ctx))
+			healingLogIf(ctx, tracker.update(ctx))
 			continue
 		}
 
@@ -293,7 +293,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				if res.entryDone {
 					tracker.setObject(res.name)
 					if time.Since(tracker.getLastUpdate()) > time.Minute {
-						logger.LogIf(ctx, tracker.update(ctx))
+						healingLogIf(ctx, tracker.update(ctx))
 					}
 					continue
 				}
@@ -306,7 +306,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			select {
 			case <-ctx.Done():
 				if !contextCanceled(ctx) {
-					logger.LogIf(ctx, ctx.Err())
+					healingLogIf(ctx, ctx.Err())
 				}
 				return false
 			case results <- result:
@@ -360,7 +360,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 						return
 					}
 					result = healEntryFailure(0)
-					logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, entry.name, err))
+					healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, entry.name, err))
 				} else {
 					result = healEntrySuccess(0)
 				}
@@ -399,9 +399,9 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					// If not deleted, assume they failed.
 					result = healEntryFailure(uint64(version.Size))
 					if version.VersionID != "" {
-						logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
+						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
 					} else {
-						logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, version.Name, err))
+						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, version.Name, err))
 					}
 				} else {
 					result = healEntrySuccess(uint64(version.Size))
@@ -465,7 +465,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			// we let the caller retry this disk again for the
 			// buckets it failed to list.
 			retErr = err
-			logger.LogIf(ctx, err)
+			healingLogIf(ctx, err)
 			continue
 		}
 
@@ -475,7 +475,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			return ctx.Err()
 		default:
 			tracker.bucketDone(bucket)
-			logger.LogIf(ctx, tracker.update(ctx))
+			healingLogIf(ctx, tracker.update(ctx))
 		}
 	}
 

cmd/globals.go

@@ -171,6 +171,7 @@ type serverCtxt struct {
 	ReadHeaderTimeout   time.Duration
 	MaxIdleConnsPerHost int
 
+	CrossDomainXML string
 	// The layout of disks as interpreted
 	Layout disksLayout
 }
@@ -470,6 +471,7 @@ var (
 
 	// Indicates if server was started as `--address ":0"`
 	globalDynamicAPIPort bool
+
 	// Add new variable global values here.
 )
 

cmd/grid.go

@@ -35,13 +35,9 @@ var globalGrid atomic.Pointer[grid.Manager]
 var globalGridStart = make(chan struct{})
 
 func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error {
-	lookupHost := globalDNSCache.LookupHost
-	if IsKubernetes() || IsDocker() {
-		lookupHost = nil
-	}
 	hosts, local := eps.GridHosts()
 	g, err := grid.NewManager(ctx, grid.ManagerOptions{
-		Dialer:       grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions))),
+		Dialer:       grid.ContextDialer(xhttp.DialContextWithLookupHost(globalDNSCache.LookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions))),
 		Local:        local,
 		Hosts:        hosts,
 		AddAuth:      newCachedAuthToken(),

cmd/handler-api.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"math"
 	"net/http"
 	"os"
 	"runtime"
@@ -55,6 +56,7 @@ type apiConfig struct {
 	gzipObjects                 bool
 	rootAccess                  bool
 	syncEvents                  bool
+	objectMaxVersions           int64
 }
 
 const (
@@ -87,14 +89,14 @@ func cgroupMemLimit() (limit uint64) {
 }
 
 func availableMemory() (available uint64) {
-	available = 8 << 30 // Default to 8 GiB when we can't find the limits.
+	available = 2048 * blockSizeV2 * 2 // Default to 4 GiB when we can't find the limits.
 
 	if runtime.GOOS == "linux" {
 		// Useful in container mode
 		limit := cgroupMemLimit()
 		if limit > 0 {
-			// A valid value is found
-			available = limit
+			// A valid value is found, return its 75%
+			available = (limit * 3) / 4
 			return
 		}
 	} // for all other platforms limits are based on virtual memory.
@@ -103,7 +105,8 @@ func availableMemory() (available uint64) {
 	if err != nil {
 		return
 	}
-	available = memStats.Available / 2
+	// A valid value is available return its 75%
+	available = (memStats.Available * 3) / 4
 	return
 }
 
@@ -132,6 +135,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 
 	var apiRequestsMaxPerNode int
 	if cfg.RequestsMax <= 0 {
+		// Returns 75% of max memory allowed
 		maxMem := availableMemory()
 
 		// max requests per node is calculated as
@@ -184,6 +188,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	t.gzipObjects = cfg.GzipObjects
 	t.rootAccess = cfg.RootAccess
 	t.syncEvents = cfg.SyncEvents
+	t.objectMaxVersions = cfg.ObjectMaxVersions
 }
 
 func (t *apiConfig) odirectEnabled() bool {
@@ -287,7 +292,11 @@ func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
 	defer t.mu.RUnlock()
 
 	if t.requestsPool == nil {
-		return nil, time.Duration(0)
+		return nil, 10 * time.Second
+	}
+
+	if t.requestsDeadline <= 0 {
+		return t.requestsPool, 10 * time.Second
 	}
 
 	return t.requestsPool, t.requestsDeadline
@@ -384,3 +393,15 @@ func (t *apiConfig) isSyncEventsEnabled() bool {
 
 	return t.syncEvents
 }
+
+func (t *apiConfig) getObjectMaxVersions() int64 {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	if t.objectMaxVersions <= 0 {
+		// defaults to 'IntMax' when unset.
+		return math.MaxInt64
+	}
+
+	return t.objectMaxVersions
+}

cmd/handler-utils.go

@@ -33,6 +33,8 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/mcontext"
 	xnet "github.com/minio/pkg/v2/net"
+	"golang.org/x/exp/maps"
+	"golang.org/x/exp/slices"
 )
 
 const (
@@ -49,7 +51,7 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError
 	locationConstraint := createBucketLocationConfiguration{}
 	err := xmlDecoder(r.Body, &locationConstraint, r.ContentLength)
 	if err != nil && r.ContentLength != 0 {
-		logger.LogOnceIf(GlobalContext, err, "location-constraint-xml-parsing")
+		internalLogOnceIf(GlobalContext, err, "location-constraint-xml-parsing")
 		// Treat all other failures as XML parsing errors.
 		return "", ErrMalformedXML
 	} // else for both err as nil or io.EOF
@@ -82,6 +84,31 @@ var supportedHeaders = []string{
 	xhttp.AmzObjectTagging,
 	"expires",
 	xhttp.AmzBucketReplicationStatus,
+	"X-Minio-Replication-Server-Side-Encryption-Sealed-Key",
+	"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm",
+	"X-Minio-Replication-Server-Side-Encryption-Iv",
+	"X-Minio-Replication-Encrypted-Multipart",
+	"X-Minio-Replication-Actual-Object-Size",
+	// Add more supported headers here.
+}
+
+// mapping of internal headers to allowed replication headers
+var validSSEReplicationHeaders = map[string]string{
+	"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "X-Minio-Replication-Server-Side-Encryption-Sealed-Key",
+	"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm",
+	"X-Minio-Internal-Server-Side-Encryption-Iv":             "X-Minio-Replication-Server-Side-Encryption-Iv",
+	"X-Minio-Internal-Encrypted-Multipart":                   "X-Minio-Replication-Encrypted-Multipart",
+	"X-Minio-Internal-Actual-Object-Size":                    "X-Minio-Replication-Actual-Object-Size",
+	// Add more supported headers here.
+}
+
+// mapping of replication headers to internal headers
+var replicationToInternalHeaders = map[string]string{
+	"X-Minio-Replication-Server-Side-Encryption-Sealed-Key":     "X-Minio-Internal-Server-Side-Encryption-Sealed-Key",
+	"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm",
+	"X-Minio-Replication-Server-Side-Encryption-Iv":             "X-Minio-Internal-Server-Side-Encryption-Iv",
+	"X-Minio-Replication-Encrypted-Multipart":                   "X-Minio-Internal-Encrypted-Multipart",
+	"X-Minio-Replication-Actual-Object-Size":                    "X-Minio-Internal-Actual-Object-Size",
 	// Add more supported headers here.
 }
 
@@ -164,7 +191,7 @@ func extractMetadata(ctx context.Context, mimesHeader ...textproto.MIMEHeader) (
 // extractMetadata extracts metadata from map values.
 func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
 	if v == nil {
-		logger.LogIf(ctx, errInvalidArgument)
+		bugLogIf(ctx, errInvalidArgument)
 		return errInvalidArgument
 	}
 
@@ -178,7 +205,11 @@ func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[
 	for _, supportedHeader := range supportedHeaders {
 		value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]
 		if ok {
-			m[supportedHeader] = strings.Join(value, ",")
+			if slices.Contains(maps.Keys(replicationToInternalHeaders), supportedHeader) {
+				m[replicationToInternalHeaders[supportedHeader]] = strings.Join(value, ",")
+			} else {
+				m[supportedHeader] = strings.Join(value, ",")
+			}
 		}
 	}
 
@@ -296,8 +327,8 @@ func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 
 		bucket, _ := path2BucketObject(resource)
 
-		_, err = globalBucketMetadataSys.Get(bucket) // check if this bucket exists.
-		countBktStat := bucket != "" && bucket != minioReservedBucket && err == nil
+		meta, err := globalBucketMetadataSys.Get(bucket) // check if this bucket exists.
+		countBktStat := bucket != "" && bucket != minioReservedBucket && err == nil && !meta.Created.IsZero()
 		if countBktStat {
 			globalBucketHTTPStats.updateHTTPStats(bucket, api, nil)
 		}
@@ -366,12 +397,6 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 	}
 	desc := "Do not upgrade one server at a time - please follow the recommended guidelines mentioned here https://github.com/minio/minio#upgrading-minio for your environment"
 	switch {
-	case strings.HasPrefix(r.URL.Path, peerS3Prefix):
-		writeErrorResponseString(r.Context(), w, APIError{
-			Code:           "XMinioPeerS3VersionMismatch",
-			Description:    desc,
-			HTTPStatusCode: http.StatusUpgradeRequired,
-		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, peerRESTPrefix):
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioPeerVersionMismatch",
@@ -436,7 +461,7 @@ func proxyRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, e
 		ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
 			success = false
 			if err != nil && !errors.Is(err, context.Canceled) {
-				logger.LogIf(GlobalContext, err)
+				replLogIf(GlobalContext, err)
 			}
 		},
 	})

cmd/healthcheck-handler.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
 )
 
 const unavailable = "offline"
@@ -134,7 +135,7 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 		ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
 		defer cancel()
 
-		if _, err := GlobalKMS.Stat(ctx); err != nil {
+		if _, err := GlobalKMS.GenerateKey(ctx, "", kms.Context{"healthcheck": ""}); err != nil {
 			switch r.Method {
 			case http.MethodHead:
 				apiErr := toAPIError(r.Context(), err)

cmd/http-stats.go

@@ -269,6 +269,28 @@ func (s *bucketConnStats) getS3InOutBytes() map[string]inOutBytes {
 	return bucketStats
 }
 
+// Return S3 total input/output bytes for each
+func (s *bucketConnStats) getBucketS3InOutBytes(buckets []string) map[string]inOutBytes {
+	s.RLock()
+	defer s.RUnlock()
+
+	if len(s.stats) == 0 || len(buckets) == 0 {
+		return nil
+	}
+
+	bucketStats := make(map[string]inOutBytes, len(buckets))
+	for _, bucket := range buckets {
+		if stats, ok := s.stats[bucket]; ok {
+			bucketStats[bucket] = inOutBytes{
+				In:  stats.s3InputBytes,
+				Out: stats.s3OutputBytes,
+			}
+		}
+	}
+
+	return bucketStats
+}
+
 // delete metrics once bucket is deleted.
 func (s *bucketConnStats) delete(bucket string) {
 	s.Lock()

cmd/iam-etcd-store.go

@@ -30,7 +30,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
+	"github.com/puzpuzpuz/xsync/v3"
 	"go.etcd.io/etcd/api/v3/mvccpb"
 	etcd "go.etcd.io/etcd/client/v3"
 )
@@ -325,11 +325,11 @@ func (ies *IAMEtcdStore) loadGroups(ctx context.Context, m map[string]GroupInfo)
 	return nil
 }
 
-func (ies *IAMEtcdStore) loadMappedPolicyWithRetry(ctx context.Context, name string, userType IAMUserType, isGroup bool, m map[string]MappedPolicy, _ int) error {
+func (ies *IAMEtcdStore) loadMappedPolicyWithRetry(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy], retries int) error {
 	return ies.loadMappedPolicy(ctx, name, userType, isGroup, m)
 }
 
-func (ies *IAMEtcdStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
+func (ies *IAMEtcdStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
 	var p MappedPolicy
 	err := ies.loadIAMConfig(ctx, &p, getMappedPolicyPath(name, userType, isGroup))
 	if err != nil {
@@ -338,11 +338,11 @@ func (ies *IAMEtcdStore) loadMappedPolicy(ctx context.Context, name string, user
 		}
 		return err
 	}
-	m[name] = p
+	m.Store(name, p)
 	return nil
 }
 
-func getMappedPolicy(ctx context.Context, kv *mvccpb.KeyValue, userType IAMUserType, isGroup bool, m map[string]MappedPolicy, basePrefix string) error {
+func getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy], basePrefix string) error {
 	var p MappedPolicy
 	err := getIAMConfig(&p, kv.Value, string(kv.Key))
 	if err != nil {
@@ -352,11 +352,11 @@ func getMappedPolicy(ctx context.Context, kv *mvccpb.KeyValue, userType IAMUserT
 		return err
 	}
 	name := extractPathPrefixAndSuffix(string(kv.Key), basePrefix, ".json")
-	m[name] = p
+	m.Store(name, p)
 	return nil
 }
 
-func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
+func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
 	cctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
 	defer cancel()
 	var basePrefix string
@@ -381,7 +381,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
 
 	// Parse all policies mapping to create the proper data model
 	for _, kv := range r.Kvs {
-		if err = getMappedPolicy(ctx, kv, userType, isGroup, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) {
+		if err = getMappedPolicy(kv, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) {
 			return err
 		}
 	}
@@ -459,7 +459,7 @@ func (ies *IAMEtcdStore) watch(ctx context.Context, keyPath string) <-chan iamWa
 						goto outerLoop
 					}
 					if err := watchResp.Err(); err != nil {
-						logger.LogIf(ctx, err)
+						iamLogIf(ctx, err)
 						// log and retry.
 						time.Sleep(1 * time.Second)
 						// Upon an error on watch channel

cmd/iam-object-store.go

@@ -25,16 +25,16 @@ import (
 	"path"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 	"unicode/utf8"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/madmin-go/v3"
-	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
+	"github.com/puzpuzpuz/xsync/v3"
 )
 
 // IAMObjectStore implements IAMStorageAPI
@@ -44,6 +44,8 @@ type IAMObjectStore struct {
 
 	*iamCache
 
+	cachedIAMListing atomic.Value
+
 	usersSysType UsersSysType
 
 	objAPI ObjectLayer
@@ -325,9 +327,7 @@ func (iamOS *IAMObjectStore) loadGroups(ctx context.Context, m map[string]GroupI
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadMappedPolicyWithRetry(ctx context.Context, name string, userType IAMUserType, isGroup bool,
-	m map[string]MappedPolicy, retries int,
-) error {
+func (iamOS *IAMObjectStore) loadMappedPolicyWithRetry(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy], retries int) error {
 	for {
 	retry:
 		var p MappedPolicy
@@ -344,14 +344,12 @@ func (iamOS *IAMObjectStore) loadMappedPolicyWithRetry(ctx context.Context, name
 			goto retry
 		}
 
-		m[name] = p
+		m.Store(name, p)
 		return nil
 	}
 }
 
-func (iamOS *IAMObjectStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool,
-	m map[string]MappedPolicy,
-) error {
+func (iamOS *IAMObjectStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
 	var p MappedPolicy
 	err := iamOS.loadIAMConfig(ctx, &p, getMappedPolicyPath(name, userType, isGroup))
 	if err != nil {
@@ -361,11 +359,11 @@ func (iamOS *IAMObjectStore) loadMappedPolicy(ctx context.Context, name string,
 		return err
 	}
 
-	m[name] = p
+	m.Store(name, p)
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
+func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
 	var basePath string
 	if isGroup {
 		basePath = iamConfigPolicyDBGroupsPrefix
@@ -396,53 +394,52 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IA
 }
 
 var (
-	usersListKey                   = "users/"
-	svcAccListKey                  = "service-accounts/"
-	groupsListKey                  = "groups/"
-	policiesListKey                = "policies/"
-	stsListKey                     = "sts/"
-	policyDBUsersListKey           = "policydb/users/"
-	policyDBSTSUsersListKey        = "policydb/sts-users/"
-	policyDBServiceAccountsListKey = "policydb/service-accounts/"
-	policyDBGroupsListKey          = "policydb/groups/"
-
-	// List of directories from which to read iam data into memory.
-	allListKeys = []string{
-		usersListKey,
-		svcAccListKey,
-		groupsListKey,
-		policiesListKey,
-		stsListKey,
-		policyDBUsersListKey,
-		policyDBSTSUsersListKey,
-		policyDBServiceAccountsListKey,
-		policyDBGroupsListKey,
-	}
-
-	// List of directories to skip: we do not read STS directories for better
-	// performance. STS credentials would be stored in memory when they are
-	// first used.
-	iamLoadSkipListKeySet = set.CreateStringSet(
-		stsListKey,
-		policyDBSTSUsersListKey,
-	)
+	usersListKey            = "users/"
+	svcAccListKey           = "service-accounts/"
+	groupsListKey           = "groups/"
+	policiesListKey         = "policies/"
+	stsListKey              = "sts/"
+	policyDBPrefix          = "policydb/"
+	policyDBUsersListKey    = "policydb/users/"
+	policyDBSTSUsersListKey = "policydb/sts-users/"
+	policyDBGroupsListKey   = "policydb/groups/"
 )
 
+// splitPath splits a path into a top-level directory and a child item. The
+// parent directory retains the trailing slash.
+func splitPath(s string, lastIndex bool) (string, string) {
+	var i int
+	if lastIndex {
+		i = strings.LastIndex(s, "/")
+	} else {
+		i = strings.Index(s, "/")
+	}
+	if i == -1 {
+		return s, ""
+	}
+	// Include the trailing slash in the parent directory.
+	return s[:i+1], s[i+1:]
+}
+
 func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (map[string][]string, error) {
 	res := make(map[string][]string)
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
-	for _, listKey := range allListKeys {
-		if iamLoadSkipListKeySet.Contains(listKey) {
+	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+SlashSeparator) {
+		if item.Err != nil {
+			return nil, item.Err
+		}
+
+		lastIndex := strings.HasPrefix(item.Item, policyDBPrefix)
+		listKey, trimmedItem := splitPath(item.Item, lastIndex)
+		if listKey == iamFormatFile {
 			continue
 		}
-		for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+SlashSeparator+listKey) {
-			if item.Err != nil {
-				return nil, item.Err
-			}
-			res[listKey] = append(res[listKey], item.Item)
-		}
+
+		res[listKey] = append(res[listKey], trimmedItem)
 	}
+	// Store the listing for later re-use.
+	iamOS.cachedIAMListing.Store(res)
 	return res, nil
 }
 
@@ -453,22 +450,47 @@ func (iamOS *IAMObjectStore) PurgeExpiredSTS(ctx context.Context) error {
 	}
 
 	bootstrapTraceMsg("purging expired STS credentials")
+
+	iamListing, ok := iamOS.cachedIAMListing.Load().(map[string][]string)
+	if !ok {
+		// There has been no store yet. This should never happen!
+		iamLogIf(GlobalContext, errors.New("WARNING: no cached IAM listing found"))
+		return nil
+	}
+
 	// Scan STS users on disk and purge expired ones. We do not need to hold a
 	// lock with store.lock() here.
-	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+SlashSeparator+stsListKey) {
-		if item.Err != nil {
-			return item.Err
-		}
-		userName := path.Dir(item.Item)
-		// loadUser() will delete expired user during the load - we do not need
-		// to keep the loaded user around in memory, so we reinitialize the map
-		// each time.
-		m := map[string]UserIdentity{}
-		if err := iamOS.loadUser(ctx, userName, stsUser, m); err != nil && err != errNoSuchUser {
-			logger.LogIf(GlobalContext, fmt.Errorf("unable to load user during STS purge: %w (%s)", err, item.Item))
+	stsAccountsFromStore := map[string]UserIdentity{}
+	stsAccPoliciesFromStore := xsync.NewMapOf[string, MappedPolicy]()
+	for _, item := range iamListing[stsListKey] {
+		userName := path.Dir(item)
+		// loadUser() will delete expired user during the load.
+		err := iamOS.loadUser(ctx, userName, stsUser, stsAccountsFromStore)
+		if err != nil && !errors.Is(err, errNoSuchUser) {
+			iamLogIf(GlobalContext,
+				fmt.Errorf("unable to load user during STS purge: %w (%s)", err, item))
 		}
 
 	}
+	// Loading the STS policy mappings from disk ensures that stale entries
+	// (removed during loadUser() in the loop above) are removed from memory.
+	for _, item := range iamListing[policyDBSTSUsersListKey] {
+		stsName := strings.TrimSuffix(item, ".json")
+		err := iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, stsAccPoliciesFromStore)
+		if err != nil && !errors.Is(err, errNoSuchPolicy) {
+			iamLogIf(GlobalContext,
+				fmt.Errorf("unable to load policies during STS purge: %w (%s)", err, item))
+		}
+
+	}
+
+	// Store the newly populated map in the iam cache. This takes care of
+	// removing stale entries from the existing map.
+	iamOS.Lock()
+	defer iamOS.Unlock()
+	iamOS.iamCache.iamSTSAccountsMap = stsAccountsFromStore
+	iamOS.iamCache.iamSTSPolicyMap = stsAccPoliciesFromStore
+
 	return nil
 }
 
@@ -629,10 +651,8 @@ type itemOrErr struct {
 	Err  error
 }
 
-// Lists files or dirs in the minioMetaBucket at the given path
-// prefix. If dirs is true, only directories are listed, otherwise
-// only objects are listed. All returned items have the pathPrefix
-// removed from their names.
+// Lists objects in the minioMetaBucket at the given path prefix. All returned
+// items have the pathPrefix removed from their names.
 func listIAMConfigItems(ctx context.Context, objAPI ObjectLayer, pathPrefix string) <-chan itemOrErr {
 	ch := make(chan itemOrErr)