ilm: Handle DeleteAllVersions action differently for DEL markers (#19481)

i.e., this rule element doesn't apply to DEL markers.

This is a breaking change to how ExpiredObejctDeleteAllVersions
functions today. This is necessary to avoid the following highly probable
footgun scenario in the future.

Scenario:
The user uses tags-based filtering to select an object's time to live(TTL). 
The application sometimes deletes objects, too, making its latest
version a DEL marker. The previous implementation skipped tag-based filters
if the newest version was DEL marker, voiding the tag-based TTL. The user is
surprised to find objects that have expired sooner than expected.

* Add DelMarkerExpiration action

This ILM action removes all versions of an object if its
the latest version is a DEL marker.

```xml
<DelMarkerObjectExpiration>
    <Days> 10 </Days>
</DelMarkerObjectExpiration>
```

1. Applies only to objects whose,
  • The latest version is a DEL marker.
  • satisfies the number of days criteria
2. Deletes all versions of this object
3. Associated rule can't have tag-based filtering

Includes,
- New bucket event type for deletion due to DelMarkerExpiration

.github/workflows/go-cross.yml

@@ -3,12 +3,12 @@ name: Crosscompile
 on:
   pull_request:
     branches:
-    - master
-    - next
+      - master
+      - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
-concurrency: 
+concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
   cancel-in-progress: true
 
@@ -24,8 +24,8 @@ jobs:
         go-version: [1.21.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
-      - uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go-version }}
           check-latest: true

.github/workflows/iam-integrations.yaml

@@ -3,8 +3,8 @@ name: IAM integration
 on:
   pull_request:
     branches:
-    - master
-    - next
+      - master
+      - next
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -112,6 +112,12 @@ jobs:
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           go run docs/iam/access-manager-plugin.go &
           make test-iam
+      - name: Test MinIO Old Version data to IAM import current version
+        if: matrix.ldap == 'ldaphost:389'
+        env:
+          _MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
+        run: |
+          make test-iam-ldap-upgrade-import
       - name: Test LDAP for automatic site replication
         if: matrix.ldap == 'localhost:389'
         run: |

.github/workflows/multipart/migrate.sh

@@ -24,10 +24,7 @@ if [ ! -f ./mc ]; then
 		chmod +x mc
 fi
 
-(
-	cd ./docs/debugging/s3-check-md5
-	go install -v
-)
+go install -v github.com/minio/minio/docs/debugging/s3-check-md5@latest
 
 export RELEASE=RELEASE.2023-08-29T23-07-35Z
 
@@ -109,6 +106,43 @@ if [ $failed_count_site2 -ne 0 ]; then
 	exit 1
 fi
 
+# Add user group test
+./mc admin user add site1 site-replication-issue-user site-replication-issue-password
+./mc admin group add site1 site-replication-issue-group site-replication-issue-user
+
+max_wait_attempts=30
+wait_interval=5
+
+attempt=1
+while true; do
+	diff <(./mc admin group info site1 site-replication-issue-group) <(./mc admin group info site2 site-replication-issue-group)
+
+	if [[ $? -eq 0 ]]; then
+		echo "Outputs are consistent."
+		break
+	fi
+
+	remaining_attempts=$((max_wait_attempts - attempt))
+	if ((attempt >= max_wait_attempts)); then
+		echo "Outputs remain inconsistent after $max_wait_attempts attempts. Exiting with error."
+		exit 1
+	else
+		echo "Outputs are inconsistent. Waiting for $wait_interval seconds (attempt $attempt/$max_wait_attempts)."
+		sleep $wait_interval
+	fi
+
+	((attempt++))
+done
+
+status=$(./mc admin group info site1 site-replication-issue-group --json | jq .groupStatus | tr -d '"')
+
+if [[ $status == "enabled" ]]; then
+	echo "Success"
+else
+	echo "Expected status: enabled, actual status: $status"
+	exit 1
+fi
+
 cleanup
 
 ## change working directory

.github/workflows/replication.yaml

@@ -36,6 +36,12 @@ jobs:
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-decom
 
+      - name: Test ILM
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make test-ilm
+
       - name: Test Config File
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0

.github/workflows/typos.yml

@@ -1,5 +1,5 @@
 ---
-name: Test GitHub Action
+name: Spelling
 on: [pull_request]
 
 jobs:

.github/workflows/vulncheck.yml

@@ -21,11 +21,11 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: 1.21.8
+          go-version: 1.21.9
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest
         shell: bash
       - name: Run govulncheck
-        run: govulncheck ./...
+        run: govulncheck -show verbose ./...
         shell: bash

.typos.toml

@@ -12,20 +12,27 @@ extend-ignore-re = [
     "[0-9A-Za-z/+=]{64}",
     "ZXJuZXQxDjAMBgNVBA-some-junk-Q4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF",
     "eyJmb28iOiJiYXIifQ",
+    "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.*",
+    "MIIDBTCCAe2gAwIBAgIQWHw7h.*",
     'http\.Header\{"X-Amz-Server-Side-Encryptio":',
-    'sessionToken',
+    "ZoEoZdLlzVbOlT9rbhD7ZN7TLyiYXSAlB79uGEge",
 ]
 
 [default.extend-words]
 "encrypter" = "encrypter"
+"kms" = "kms"
 "requestor" = "requestor"
 
 [default.extend-identifiers]
-"bui" = "bui"
-"toi" = "toi"
-"ot" = "ot"
-"dm2nd" = "dm2nd"
 "HashiCorp" = "HashiCorp"
+
+[type.go.extend-identifiers]
+"bui" = "bui"
+"dm2nd" = "dm2nd"
+"ot" = "ot"
 "ParseND" = "ParseND"
 "ParseNDStream" = "ParseNDStream"
+"pn" = "pn"
 "TestGetPartialObjectMisAligned" = "TestGetPartialObjectMisAligned"
+"thr" = "thr"
+"toi" = "toi"

Makefile

@@ -53,6 +53,10 @@ test-root-disable: install-race
 	@echo "Running minio root lockdown tests"
 	@env bash $(PWD)/buildscripts/disable-root.sh
 
+test-ilm: install-race
+	@echo "Running ILM tests"
+	@env bash $(PWD)/docs/bucket/replication/setup_ilm_expiry_replication.sh
+
 test-decom: install-race
 	@echo "Running minio decom tests"
 	@env bash $(PWD)/docs/distributed/decom.sh
@@ -81,6 +85,10 @@ test-iam: build ## verify IAM (external IDP, etcd backends)
 	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
 	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
 
+test-iam-ldap-upgrade-import: build ## verify IAM (external LDAP IDP)
+	@echo "Running upgrade tests for IAM (LDAP backend)"
+	@env bash $(PWD)/buildscripts/minio-iam-ldap-upgrade-import-test.sh
+
 test-sio-error:
 	@(env bash $(PWD)/docs/bucket/replication/sio-error.sh)
 
@@ -123,6 +131,7 @@ verify-healing: ## verify healing and replacing disks with minio binary
 	@echo "Verify healing build with race"
 	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
+	@(env bash $(PWD)/buildscripts/verify-healing-empty-erasure-set.sh)
 	@(env bash $(PWD)/buildscripts/heal-inconsistent-versions.sh)
 
 verify-healing-with-root-disks: ## verify healing root disks

buildscripts/minio-iam-ldap-upgrade-import-test.sh

@@ -0,0 +1,125 @@
+#!/bin/bash
+
+# This script is used to test the migration of IAM content from old minio
+# instance to new minio instance.
+#
+# To run it locally, start the LDAP server in github.com/minio/minio-iam-testing
+# repo (e.g. make podman-run), and then run this script.
+#
+# This script assumes that LDAP server is at:
+#
+#   `localhost:1389`
+#
+# if this is not the case, set the environment variable
+# `_MINIO_LDAP_TEST_SERVER`.
+
+OLD_VERSION=RELEASE.2024-03-26T22-10-45Z
+OLD_BINARY_LINK=https://dl.min.io/server/minio/release/linux-amd64/archive/minio.${OLD_VERSION}
+
+__init__() {
+	if which curl &>/dev/null; then
+		echo "curl is already installed"
+	else
+		echo "Installing curl:"
+		sudo apt install curl -y
+	fi
+
+	export GOPATH=/tmp/gopath
+	export PATH="${PATH}":"${GOPATH}"/bin
+
+	if which mc &>/dev/null; then
+		echo "mc is already installed"
+	else
+		echo "Installing mc:"
+		go install github.com/minio/mc@latest
+	fi
+
+	if [ ! -x ./minio.${OLD_VERSION} ]; then
+		echo "Downloading minio.${OLD_VERSION} binary"
+		curl -o minio.${OLD_VERSION} ${OLD_BINARY_LINK}
+		chmod +x minio.${OLD_VERSION}
+	fi
+
+	if [ -z "$_MINIO_LDAP_TEST_SERVER" ]; then
+		export _MINIO_LDAP_TEST_SERVER=localhost:1389
+		echo "Using default LDAP endpoint: $_MINIO_LDAP_TEST_SERVER"
+	fi
+
+	rm -rf /tmp/data
+}
+
+create_iam_content_in_old_minio() {
+	echo "Creating IAM content in old minio instance."
+
+	MINIO_CI_CD=1 ./minio.${OLD_VERSION} server /tmp/data/{1...4} &
+	sleep 5
+
+	set -x
+	mc alias set old-minio http://localhost:9000 minioadmin minioadmin
+	mc idp ldap add old-minio \
+		server_addr=localhost:1389 \
+		server_insecure=on \
+		lookup_bind_dn=cn=admin,dc=min,dc=io \
+		lookup_bind_password=admin \
+		user_dn_search_base_dn=dc=min,dc=io \
+		user_dn_search_filter="(uid=%s)" \
+		group_search_base_dn=ou=swengg,dc=min,dc=io \
+		group_search_filter="(&(objectclass=groupOfNames)(member=%d))"
+	mc admin service restart old-minio
+
+	mc idp ldap policy attach old-minio readwrite --user=UID=dillon,ou=people,ou=swengg,dc=min,dc=io
+	mc idp ldap policy attach old-minio readwrite --group=CN=project.c,ou=groups,ou=swengg,dc=min,dc=io
+
+	mc idp ldap policy entities old-minio
+
+	mc admin cluster iam export old-minio
+	set +x
+
+	mc admin service stop old-minio
+}
+
+import_iam_content_in_new_minio() {
+	echo "Importing IAM content in new minio instance."
+	# Assume current minio binary exists.
+	MINIO_CI_CD=1 ./minio server /tmp/data/{1...4} &
+	sleep 5
+
+	set -x
+	mc alias set new-minio http://localhost:9000 minioadmin minioadmin
+	echo "BEFORE IMPORT mappings:"
+	mc idp ldap policy entities new-minio
+	mc admin cluster iam import new-minio ./old-minio-iam-info.zip
+	echo "AFTER IMPORT mappings:"
+	mc idp ldap policy entities new-minio
+	set +x
+
+	# mc admin service stop new-minio
+}
+
+verify_iam_content_in_new_minio() {
+	output=$(mc idp ldap policy entities new-minio --json)
+
+	groups=$(echo "$output" | jq -r '.result.policyMappings[] | select(.policy == "readwrite") | .groups[]')
+	if [ "$groups" != "cn=project.c,ou=groups,ou=swengg,dc=min,dc=io" ]; then
+		echo "Failed to verify groups: $groups"
+		exit 1
+	fi
+
+	users=$(echo "$output" | jq -r '.result.policyMappings[] | select(.policy == "readwrite") | .users[]')
+	if [ "$users" != "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" ]; then
+		echo "Failed to verify users: $users"
+		exit 1
+	fi
+
+	mc admin service stop new-minio
+}
+
+main() {
+	create_iam_content_in_old_minio
+
+	import_iam_content_in_new_minio
+
+	verify_iam_content_in_new_minio
+}
+
+(__init__ "$@" && main "$@")

buildscripts/minio-upgrade.sh

@@ -55,6 +55,11 @@ __init__() {
 
 	go install github.com/minio/mc@latest
 
+	## this is needed because github actions don't have
+	## docker-compose on all runners
+	go install github.com/docker/compose/v2/cmd@latest
+	mv -v /tmp/gopath/bin/cmd /tmp/gopath/bin/docker-compose
+
 	TAG=minio/minio:dev make docker
 
 	MINIO_VERSION=RELEASE.2019-12-19T22-52-26Z docker-compose \

buildscripts/rewrite-old-new.sh

@@ -87,10 +87,7 @@ function verify_rewrite() {
 		exit 1
 	fi
 
-	(
-		cd ./docs/debugging/s3-check-md5
-		go install -v
-	)
+	go install -v github.com/minio/minio/docs/debugging/s3-check-md5@latest
 
 	if ! s3-check-md5 \
 		-debug \

buildscripts/verify-healing-empty-erasure-set.sh

@@ -0,0 +1,140 @@
+#!/bin/bash -e
+#
+
+set -E
+set -o pipefail
+
+if [ ! -x "$PWD/minio" ]; then
+	echo "minio executable binary not found in current directory"
+	exit 1
+fi
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
+
+function start_minio_3_node() {
+	export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_PASSWORD=minio123
+	export MINIO_ERASURE_SET_DRIVE_COUNT=6
+	export MINIO_CI_CD=1
+
+	start_port=$2
+	args=""
+	for i in $(seq 1 3); do
+		args="$args http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/1/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/2/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/3/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/4/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/5/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/6/"
+	done
+
+	"${MINIO[@]}" --address ":$((start_port + 1))" $args >"${WORK_DIR}/dist-minio-server1.log" 2>&1 &
+	pid1=$!
+	disown ${pid1}
+
+	"${MINIO[@]}" --address ":$((start_port + 2))" $args >"${WORK_DIR}/dist-minio-server2.log" 2>&1 &
+	pid2=$!
+	disown $pid2
+
+	"${MINIO[@]}" --address ":$((start_port + 3))" $args >"${WORK_DIR}/dist-minio-server3.log" 2>&1 &
+	pid3=$!
+	disown $pid3
+
+	sleep "$1"
+
+	if ! ps -p $pid1 1>&2 >/dev/null; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/dist-minio-server1.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi
+
+	if ! ps -p $pid2 1>&2 >/dev/null; then
+		echo "server2 log:"
+		cat "${WORK_DIR}/dist-minio-server2.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi
+
+	if ! ps -p $pid3 1>&2 >/dev/null; then
+		echo "server3 log:"
+		cat "${WORK_DIR}/dist-minio-server3.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi
+
+	if ! pkill minio; then
+		for i in $(seq 1 3); do
+			echo "server$i log:"
+			cat "${WORK_DIR}/dist-minio-server$i.log"
+		done
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi
+
+	sleep 1
+	if pgrep minio; then
+		# forcibly killing, to proceed further properly.
+		if ! pkill -9 minio; then
+			echo "no minio process running anymore, proceed."
+		fi
+	fi
+}
+
+function check_online() {
+	if ! grep -q 'Status:' ${WORK_DIR}/dist-minio-*.log; then
+		echo "1"
+	fi
+}
+
+function purge() {
+	rm -rf "$1"
+}
+
+function __init__() {
+	echo "Initializing environment"
+	mkdir -p "$WORK_DIR"
+	mkdir -p "$MINIO_CONFIG_DIR"
+
+	## version is purposefully set to '3' for minio to migrate configuration file
+	echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json"
+}
+
+function perform_test() {
+	start_minio_3_node 120 $2
+
+	echo "Testing Distributed Erasure setup healing of drives"
+	echo "Remove the contents of the disks belonging to '${1}' erasure set"
+
+	rm -rf ${WORK_DIR}/${1}/*/
+
+	set -x
+	start_minio_3_node 120 $2
+
+	rv=$(check_online)
+	if [ "$rv" == "1" ]; then
+		for i in $(seq 1 3); do
+			echo "server$i log:"
+			cat "${WORK_DIR}/dist-minio-server$i.log"
+		done
+		pkill -9 minio
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi
+}
+
+function main() {
+	# use same ports for all tests
+	start_port=$(shuf -i 10000-65000 -n 1)
+
+	perform_test "2" ${start_port}
+	perform_test "1" ${start_port}
+	perform_test "3" ${start_port}
+}
+
+(__init__ "$@" && main "$@")
+rv=$?
+purge "$WORK_DIR"
+exit "$rv"

buildscripts/verify-healing.sh

@@ -12,6 +12,7 @@ fi
 WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
 MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
+GOPATH=/tmp/gopath
 
 function start_minio_3_node() {
 	export MINIO_ROOT_USER=minio
@@ -19,10 +20,14 @@ function start_minio_3_node() {
 	export MINIO_ERASURE_SET_DRIVE_COUNT=6
 	export MINIO_CI_CD=1
 
+	first_time=$(find ${WORK_DIR}/ | grep format.json | wc -l)
+
 	start_port=$2
 	args=""
-	for i in $(seq 1 3); do
-		args="$args http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/1/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/2/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/3/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/4/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/5/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/6/"
+	for d in $(seq 1 3 5); do
+		args="$args http://127.0.0.1:$((start_port + 1))${WORK_DIR}/1/${d}/ http://127.0.0.1:$((start_port + 2))${WORK_DIR}/2/${d}/ http://127.0.0.1:$((start_port + 3))${WORK_DIR}/3/${d}/ "
+		d=$((d + 1))
+		args="$args http://127.0.0.1:$((start_port + 1))${WORK_DIR}/1/${d}/ http://127.0.0.1:$((start_port + 2))${WORK_DIR}/2/${d}/ http://127.0.0.1:$((start_port + 3))${WORK_DIR}/3/${d}/ "
 	done
 
 	"${MINIO[@]}" --address ":$((start_port + 1))" $args >"${WORK_DIR}/dist-minio-server1.log" 2>&1 &
@@ -39,6 +44,8 @@ function start_minio_3_node() {
 
 	sleep "$1"
 
+	[ ${first_time} -eq 0 ] && upload_objects $start_port
+
 	if ! ps -p $pid1 1>&2 >/dev/null; then
 		echo "server1 log:"
 		cat "${WORK_DIR}/dist-minio-server1.log"
@@ -82,10 +89,23 @@ function start_minio_3_node() {
 	fi
 }
 
-function check_online() {
+function check_heal() {
 	if ! grep -q 'Status:' ${WORK_DIR}/dist-minio-*.log; then
-		echo "1"
+		return 1
 	fi
+
+	for ((i = 0; i < 20; i++)); do
+		test -f ${WORK_DIR}/$1/1/.minio.sys/format.json
+		v1=$?
+		nextInES=$(($1 + 1)) && [ $nextInES -gt 3 ] && nextInES=1
+		foundFiles1=$(find ${WORK_DIR}/$1/1/ | grep -v .minio.sys | grep xl.meta | wc -l)
+		foundFiles2=$(find ${WORK_DIR}/$nextInES/1/ | grep -v .minio.sys | grep xl.meta | wc -l)
+		test $foundFiles1 -eq $foundFiles2
+		v2=$?
+		[ $v1 == 0 -a $v2 == 0 ] && return 0
+		sleep 10
+	done
+	return 1
 }
 
 function purge() {
@@ -99,20 +119,39 @@ function __init__() {
 
 	## version is purposefully set to '3' for minio to migrate configuration file
 	echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json"
+
+	if [ ! -f /tmp/mc ]; then
+		wget --quiet -O /tmp/mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
+			chmod +x /tmp/mc
+	fi
+}
+
+function upload_objects() {
+	start_port=$1
+
+	/tmp/mc alias set myminio http://127.0.0.1:$((start_port + 1)) minio minio123 --api=s3v4
+	/tmp/mc ready myminio
+	/tmp/mc mb myminio/testbucket/
+	for ((i = 0; i < 20; i++)); do
+		echo "my content" | /tmp/mc pipe myminio/testbucket/file-$i
+	done
 }
 
 function perform_test() {
-	start_minio_3_node 120 $2
+	start_port=$2
+
+	start_minio_3_node 120 $start_port
 
 	echo "Testing Distributed Erasure setup healing of drives"
-	echo "Remove the contents of the disks belonging to '${1}' erasure set"
+	echo "Remove the contents of the disks belonging to '${1}' node"
 
 	rm -rf ${WORK_DIR}/${1}/*/
 
 	set -x
-	start_minio_3_node 120 $2
+	start_minio_3_node 120 $start_port
 
-	rv=$(check_online)
+	check_heal ${1}
+	rv=$?
 	if [ "$rv" == "1" ]; then
 		for i in $(seq 1 3); do
 			echo "server$i log:"

cmd/admin-bucket-handlers.go

@@ -39,7 +39,6 @@ import (
 	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/policy"
 )
@@ -99,7 +98,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 	}
 
 	// Call site replication hook.
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta))
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta))
 
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
@@ -431,7 +430,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 			case bucketNotificationConfig:
 				config, err := globalBucketMetadataSys.GetNotificationConfig(bucket)
 				if err != nil {
-					logger.LogIf(ctx, err)
+					adminLogIf(ctx, err)
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
@@ -447,7 +446,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					if errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) {
 						continue
 					}
-					logger.LogIf(ctx, err)
+					adminLogIf(ctx, err)
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
@@ -736,7 +735,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				rpt.SetStatus(bucket, fileName, fmt.Errorf("An Object Lock configuration is present on this bucket, so the versioning state cannot be suspended."))
 				continue
 			}
-			if _, err := getReplicationConfig(ctx, bucket); err == nil && v.Suspended() {
+			if rcfg, _ := getReplicationConfig(ctx, bucket); rcfg != nil && v.Suspended() {
 				rpt.SetStatus(bucket, fileName, fmt.Errorf("A replication configuration is present on this bucket, so the versioning state cannot be suspended."))
 				continue
 			}

cmd/admin-handlers-config-kv.go

@@ -58,7 +58,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.ErrorKind)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
@@ -162,7 +162,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.ErrorKind)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
@@ -443,7 +443,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.ErrorKind)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}

cmd/admin-handlers-idp-config.go

@@ -31,7 +31,6 @@ import (
 	"github.com/minio/minio/internal/config"
 	cfgldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/ldap"
 	"github.com/minio/pkg/v2/policy"
@@ -60,7 +59,7 @@ func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.R
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.ErrorKind)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}

cmd/admin-handlers-idp-ldap.go

@@ -20,14 +20,12 @@ package cmd
 import (
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io"
 	"net/http"
 	"strings"
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/policy"
 )
@@ -105,6 +103,12 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 		return
 	}
 
+	// fail if ldap is not enabled
+	if !globalIAMSys.LDAPConfig.Enabled() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminLDAPNotEnabled), r.URL)
+		return
+	}
+
 	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
 		// More than maxConfigSize bytes were available
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
@@ -132,7 +136,7 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.ErrorKind)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
@@ -192,7 +196,7 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
 
 	// fail if ldap is not enabled
 	if !globalIAMSys.LDAPConfig.Enabled() {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errors.New("LDAP not enabled")), r.URL)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminLDAPNotEnabled), r.URL)
 		return
 	}
 
@@ -218,9 +222,8 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
 		err          error
 	)
 
-	// If we are creating svc account for request sender, ensure
-	// that targetUser is a real user (i.e. not derived
-	// credentials).
+	// If we are creating svc account for request sender, ensure that targetUser
+	// is a real user (i.e. not derived credentials).
 	if isSvcAccForRequestor {
 		if requestorIsDerivedCredential {
 			if requestorParentUser == "" {
@@ -254,15 +257,23 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
 			opts.claims[k] = v
 		}
 	} else {
-		isDN := globalIAMSys.LDAPConfig.IsLDAPUserDN(targetUser)
+		// We still need to ensure that the target user is a valid LDAP user.
+		//
+		// The target user may be supplied as a (short) username or a DN.
+		// However, for now, we only support using the short username.
 
+		isDN := globalIAMSys.LDAPConfig.ParsesAsDN(targetUser)
 		opts.claims[ldapUserN] = targetUser // simple username
 		targetUser, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
 		if err != nil {
 			// if not found, check if DN
-			if strings.Contains(err.Error(), "not found") && isDN {
-				// warn user that DNs are not allowed
-				err = fmt.Errorf("Must use short username to add service account. %w", err)
+			if strings.Contains(err.Error(), "User DN not found for:") {
+				if isDN {
+					// warn user that DNs are not allowed
+					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminLDAPExpectedLoginName, err), r.URL)
+				} else {
+					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminNoSuchUser, err), r.URL)
+				}
 			}
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -301,7 +312,7 @@ func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.R
 	// Call hook for cluster-replication if the service account is not for a
 	// root user.
 	if newCred.ParentUser != globalActiveCred.AccessKey {
-		logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
 				Create: &madmin.SRSvcAccCreate{
@@ -378,7 +389,8 @@ func (a adminAPIHandlers) ListAccessKeysLDAP(w http.ResponseWriter, r *http.Requ
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
-	} else if userDN == "" {
+	}
+	if targetAccount == "" {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
 		return
 	}

cmd/admin-handlers-pools.go

@@ -26,7 +26,6 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/env"
 	"github.com/minio/pkg/v2/policy"
@@ -210,7 +209,7 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	logger.LogIf(r.Context(), json.NewEncoder(w).Encode(&status))
+	adminLogIf(r.Context(), json.NewEncoder(w).Encode(&status))
 }
 
 func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
@@ -243,7 +242,7 @@ func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
 		poolsStatus[idx] = status
 	}
 
-	logger.LogIf(r.Context(), json.NewEncoder(w).Encode(poolsStatus))
+	adminLogIf(r.Context(), json.NewEncoder(w).Encode(poolsStatus))
 }
 
 func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request) {
@@ -350,11 +349,11 @@ func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceNotStarted), r.URL)
 			return
 		}
-		logger.LogIf(ctx, fmt.Errorf("failed to fetch rebalance status: %w", err))
+		adminLogIf(ctx, fmt.Errorf("failed to fetch rebalance status: %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	logger.LogIf(r.Context(), json.NewEncoder(w).Encode(rs))
+	adminLogIf(r.Context(), json.NewEncoder(w).Encode(rs))
 }
 
 func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request) {
@@ -374,7 +373,7 @@ func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request)
 	// Cancel any ongoing rebalance operation
 	globalNotificationSys.StopRebalance(r.Context())
 	writeSuccessResponseHeadersOnly(w)
-	logger.LogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt))
+	adminLogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt))
 }
 
 func proxyDecommissionRequest(ctx context.Context, defaultEndPoint Endpoint, w http.ResponseWriter, r *http.Request) (proxy bool) {

cmd/admin-handlers-site-replication.go

@@ -32,7 +32,6 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v3"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/policy"
 )
@@ -55,7 +54,7 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 	opts := getSRAddOptions(r)
 	status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites, opts)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -93,7 +92,7 @@ func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := globalSiteReplicationSys.PeerJoinReq(ctx, joinArg); err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -140,7 +139,7 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
 		globalSiteReplicationSys.purgeDeletedBucket(ctx, objectAPI, bucket)
 	}
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -192,7 +191,7 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
 		err = globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo, item.UpdatedAt)
 	}
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -263,7 +262,7 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 		err = globalSiteReplicationSys.PeerBucketLCConfigHandler(ctx, item.Bucket, item.ExpiryLCConfig, item.UpdatedAt)
 	}
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -316,7 +315,6 @@ func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptio
 	if encryptionKey != "" {
 		data, err = madmin.DecryptData(encryptionKey, bytes.NewReader(data))
 		if err != nil {
-			logger.LogIf(ctx, err)
 			return SRError{
 				Cause: err,
 				Code:  ErrSiteReplicationInvalidRequest,
@@ -349,6 +347,18 @@ func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.R
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
+	// Report the ILMExpiryStats only if at least one site has replication of ILM expiry enabled
+	var replicateILMExpiry bool
+	for _, site := range info.Sites {
+		if site.ReplicateILMExpiry {
+			replicateILMExpiry = true
+			break
+		}
+	}
+	if !replicateILMExpiry {
+		// explicitly send nil for ILMExpiryStats
+		info.ILMExpiryStats = nil
+	}
 
 	if err = json.NewEncoder(w).Encode(info); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -396,7 +406,7 @@ func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Req
 	opts := getSREditOptions(r)
 	status, err := globalSiteReplicationSys.EditPeerCluster(ctx, site, opts)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -433,7 +443,7 @@ func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := globalSiteReplicationSys.PeerEditReq(ctx, pi); err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -456,7 +466,7 @@ func (a adminAPIHandlers) SRStateEdit(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if err := globalSiteReplicationSys.PeerStateEditReq(ctx, state); err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -493,7 +503,7 @@ func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.R
 	}
 	status, err := globalSiteReplicationSys.RemovePeerCluster(ctx, objectAPI, rreq)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -524,7 +534,7 @@ func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if err := globalSiteReplicationSys.InternalRemoveReq(ctx, objectAPI, req); err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -586,7 +596,7 @@ func (a adminAPIHandlers) SiteReplicationDevNull(w http.ResponseWriter, r *http.
 			// If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec)
 			// would mean the network is not stable. Logging here will help in debugging network issues.
 			if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
-				logger.LogIf(ctx, err)
+				adminLogIf(ctx, err)
 			}
 		}
 		if err != nil {
@@ -609,5 +619,5 @@ func (a adminAPIHandlers) SiteReplicationNetPerf(w http.ResponseWriter, r *http.
 		duration = globalNetPerfMinDuration
 	}
 	result := siteNetperf(r.Context(), duration)
-	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(result))
+	adminLogIf(r.Context(), gob.NewEncoder(w).Encode(result))
 }

cmd/admin-handlers-users.go

@@ -35,7 +35,6 @@ import (
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/cachevalue"
 	"github.com/minio/minio/internal/config/dns"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/policy"
 	"github.com/puzpuzpuz/xsync/v3"
@@ -75,7 +74,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemIAMUser,
 		IAMUser: &madmin.SRIAMUser{
 			AccessKey:   accessKey,
@@ -272,14 +271,21 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 				return
 			}
 		}
-		updatedAt, err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members)
+
+		if globalIAMSys.LDAPConfig.Enabled() {
+			// We don't allow internal group manipulation in this API when LDAP
+			// is enabled for now.
+			err = errIAMActionNotAllowed
+		} else {
+			updatedAt, err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members)
+		}
 	}
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemGroupInfo,
 		GroupInfo: &madmin.SRGroupInfo{
 			UpdateReq: updReq,
@@ -369,7 +375,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemGroupInfo,
 		GroupInfo: &madmin.SRGroupInfo{
 			UpdateReq: madmin.GroupAddRemove{
@@ -407,7 +413,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemIAMUser,
 		IAMUser: &madmin.SRIAMUser{
 			AccessKey:   accessKey,
@@ -496,25 +502,31 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	password := cred.SecretKey
 	configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
 
 	var ureq madmin.AddOrUpdateUserReq
 	if err = json.Unmarshal(configBytes, &ureq); err != nil {
-		logger.LogIf(ctx, err)
+		adminLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
 
+	// We don't allow internal user creation with LDAP enabled for now.
+	if globalIAMSys.LDAPConfig.Enabled() {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
+		return
+	}
+
 	updatedAt, err := globalIAMSys.CreateUser(ctx, accessKey, ureq)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemIAMUser,
 		IAMUser: &madmin.SRIAMUser{
 			AccessKey:   accessKey,
@@ -732,7 +744,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	// Call hook for cluster-replication if the service account is not for a
 	// root user.
 	if newCred.ParentUser != globalActiveCred.AccessKey {
-		logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
 				Create: &madmin.SRSvcAccCreate{
@@ -854,7 +866,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 
 	// Call site replication hook - non-root user accounts are replicated.
 	if svcAccount.ParentUser != globalActiveCred.AccessKey {
-		logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
 				Update: &madmin.SRSvcAccUpdate{
@@ -1116,7 +1128,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 
 	// Call site replication hook - non-root user accounts are replicated.
 	if svcAccount.ParentUser != "" && svcAccount.ParentUser != globalActiveCred.AccessKey {
-		logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemSvcAcc,
 			SvcAccChange: &madmin.SRSvcAccChange{
 				Delete: &madmin.SRSvcAccDelete{
@@ -1274,7 +1286,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	default:
 		policies, err := globalIAMSys.PolicyDBGet(accountName, cred.Groups...)
 		if err != nil {
-			logger.LogIf(ctx, err)
+			adminLogIf(ctx, err)
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
@@ -1426,7 +1438,7 @@ func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Requ
 	for name, p := range policies {
 		_, err = json.Marshal(p)
 		if err != nil {
-			logger.LogIf(ctx, err)
+			adminLogIf(ctx, err)
 			continue
 		}
 		newPolicies[name] = p
@@ -1456,7 +1468,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 	for name, p := range policies {
 		_, err = json.Marshal(p)
 		if err != nil {
-			logger.LogIf(ctx, err)
+			adminLogIf(ctx, err)
 			continue
 		}
 		newPolicies[name] = p
@@ -1486,7 +1498,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 
 	// Call cluster-replication policy creation hook to replicate policy deletion to
 	// other minio clusters.
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type:      madmin.SRIAMItemPolicy,
 		Name:      policyName,
 		UpdatedAt: UTCNow(),
@@ -1549,7 +1561,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 
 	// Call cluster-replication policy creation hook to replicate policy to
 	// other minio clusters.
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type:      madmin.SRIAMItemPolicy,
 		Name:      policyName,
 		Policy:    iamPolicyBytes,
@@ -1557,7 +1569,12 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 	}))
 }
 
-// SetPolicyForUserOrGroup - PUT /minio/admin/v3/set-policy?policy=xxx&user-or-group=?[&is-group]
+// SetPolicyForUserOrGroup - sets a policy on a user or a group.
+//
+// PUT /minio/admin/v3/set-policy?policy=xxx&user-or-group=?[&is-group]
+//
+// Deprecated: This API is replaced by attach/detach policy APIs for specific
+// type of users (builtin or LDAP).
 func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
@@ -1609,6 +1626,32 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 	userType := regUser
 	if globalIAMSys.GetUsersSysType() == LDAPUsersSysType {
 		userType = stsUser
+
+		// Validate that the user or group exists in LDAP and use the normalized
+		// form of the entityName (which will be an LDAP DN).
+		var err error
+		if isGroup {
+			var foundGroupDN string
+			var underBaseDN bool
+			if foundGroupDN, underBaseDN, err = globalIAMSys.LDAPConfig.GetValidatedGroupDN(nil, entityName); err != nil {
+				iamLogIf(ctx, err)
+			} else if foundGroupDN == "" || !underBaseDN {
+				err = errNoSuchGroup
+			}
+			entityName = foundGroupDN
+		} else {
+			var foundUserDN string
+			if foundUserDN, err = globalIAMSys.LDAPConfig.GetValidatedDNForUsername(entityName); err != nil {
+				iamLogIf(ctx, err)
+			} else if foundUserDN == "" {
+				err = errNoSuchUser
+			}
+			entityName = foundUserDN
+		}
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
 	}
 
 	updatedAt, err := globalIAMSys.PolicyDBSet(ctx, entityName, policyName, userType, isGroup)
@@ -1617,7 +1660,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 		return
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemPolicyMapping,
 		PolicyMapping: &madmin.SRPolicyMapping{
 			UserOrGroup: entityName,
@@ -1757,17 +1800,27 @@ func (a adminAPIHandlers) AttachDetachPolicyBuiltin(w http.ResponseWriter, r *ht
 }
 
 const (
-	allPoliciesFile            = "policies.json"
-	allUsersFile               = "users.json"
-	allGroupsFile              = "groups.json"
-	allSvcAcctsFile            = "svcaccts.json"
-	userPolicyMappingsFile     = "user_mappings.json"
-	groupPolicyMappingsFile    = "group_mappings.json"
-	stsUserPolicyMappingsFile  = "stsuser_mappings.json"
-	stsGroupPolicyMappingsFile = "stsgroup_mappings.json"
-	iamAssetsDir               = "iam-assets"
+	allPoliciesFile           = "policies.json"
+	allUsersFile              = "users.json"
+	allGroupsFile             = "groups.json"
+	allSvcAcctsFile           = "svcaccts.json"
+	userPolicyMappingsFile    = "user_mappings.json"
+	groupPolicyMappingsFile   = "group_mappings.json"
+	stsUserPolicyMappingsFile = "stsuser_mappings.json"
+
+	iamAssetsDir = "iam-assets"
 )
 
+var iamExportFiles = []string{
+	allPoliciesFile,
+	allUsersFile,
+	allGroupsFile,
+	allSvcAcctsFile,
+	userPolicyMappingsFile,
+	groupPolicyMappingsFile,
+	stsUserPolicyMappingsFile,
+}
+
 // ExportIAMHandler - exports all iam info as a zipped file
 func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
@@ -1791,38 +1844,28 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			sys:     nil,
 		})
 		if zerr != nil {
-			logger.LogIf(ctx, zerr)
+			adminLogIf(ctx, zerr)
 			return nil
 		}
 		header.Method = zip.Deflate
 		zwriter, zerr := zipWriter.CreateHeader(header)
 		if zerr != nil {
-			logger.LogIf(ctx, zerr)
+			adminLogIf(ctx, zerr)
 			return nil
 		}
 		if _, err := io.Copy(zwriter, r); err != nil {
-			logger.LogIf(ctx, err)
+			adminLogIf(ctx, err)
 		}
 		return nil
 	}
 
-	iamFiles := []string{
-		allPoliciesFile,
-		allUsersFile,
-		allGroupsFile,
-		allSvcAcctsFile,
-		userPolicyMappingsFile,
-		groupPolicyMappingsFile,
-		stsUserPolicyMappingsFile,
-		stsGroupPolicyMappingsFile,
-	}
-	for _, f := range iamFiles {
+	for _, f := range iamExportFiles {
 		iamFile := pathJoin(iamAssetsDir, f)
 		switch f {
 		case allPoliciesFile:
 			allPolicies, err := globalIAMSys.ListPolicies(ctx, "")
 			if err != nil {
-				logger.LogIf(ctx, err)
+				adminLogIf(ctx, err)
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
 			}
@@ -1901,7 +1944,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 					writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 					return
 				}
-				_, policy, err := globalIAMSys.GetServiceAccount(ctx, acc.Credentials.AccessKey)
+				sa, policy, err := globalIAMSys.GetServiceAccount(ctx, acc.Credentials.AccessKey)
 				if err != nil {
 					writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 					return
@@ -1923,6 +1966,9 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 					Claims:        claims,
 					SessionPolicy: json.RawMessage(policyJSON),
 					Status:        acc.Credentials.Status,
+					Name:          sa.Name,
+					Description:   sa.Description,
+					Expiration:    &sa.Expiration,
 				}
 			}
 
@@ -1986,22 +2032,6 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
 				return
 			}
-		case stsGroupPolicyMappingsFile:
-			groupPolicyMap := xsync.NewMapOf[string, MappedPolicy]()
-			err := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, true, groupPolicyMap)
-			if err != nil {
-				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
-				return
-			}
-			grpPolData, err := json.Marshal(mappedPoliciesToMap(groupPolicyMap))
-			if err != nil {
-				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
-				return
-			}
-			if err = rawDataFn(bytes.NewReader(grpPolData), iamFile, len(grpPolData)); err != nil {
-				writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
-				return
-			}
 		}
 	}
 }
@@ -2171,12 +2201,12 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					// If group does not exist, then check if the group has beginning and end space characters
 					// we will reject such group names.
 					if errors.Is(gerr, errNoSuchGroup) && hasSpaceBE(group) {
-						writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminResourceInvalidArgument, err, allGroupsFile, group), r.URL)
+						writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminResourceInvalidArgument, gerr, allGroupsFile, group), r.URL)
 						return
 					}
 				}
 				if _, gerr := globalIAMSys.AddUsersToGroup(ctx, group, grpInfo.Members); gerr != nil {
-					writeErrorResponseJSON(ctx, w, importError(ctx, err, allGroupsFile, group), r.URL)
+					writeErrorResponseJSON(ctx, w, importError(ctx, gerr, allGroupsFile, group), r.URL)
 					return
 				}
 			}
@@ -2203,6 +2233,16 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, allSvcAcctsFile, ""), r.URL)
 				return
 			}
+
+			// Validations for LDAP enabled deployments.
+			if globalIAMSys.LDAPConfig.Enabled() {
+				err := globalIAMSys.NormalizeLDAPAccessKeypairs(ctx, serviceAcctReqs)
+				if err != nil {
+					writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, ""), r.URL)
+					return
+				}
+			}
+
 			for user, svcAcctReq := range serviceAcctReqs {
 				var sp *policy.Policy
 				var err error
@@ -2213,7 +2253,8 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 						return
 					}
 				}
-				// service account access key cannot have space characters beginning and end of the string.
+				// service account access key cannot have space characters
+				// beginning and end of the string.
 				if hasSpaceBE(svcAcctReq.AccessKey) {
 					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
 					return
@@ -2239,20 +2280,14 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					updateReq = false
 				}
 				if updateReq {
-					opts := updateServiceAccountOpts{
-						secretKey:     svcAcctReq.SecretKey,
-						status:        svcAcctReq.Status,
-						name:          svcAcctReq.Name,
-						description:   svcAcctReq.Description,
-						expiration:    svcAcctReq.Expiration,
-						sessionPolicy: sp,
-					}
-					_, err = globalIAMSys.UpdateServiceAccount(ctx, svcAcctReq.AccessKey, opts)
+					// If the service account exists, we remove it to ensure a
+					// clean import.
+					err := globalIAMSys.DeleteServiceAccount(ctx, svcAcctReq.AccessKey, true)
 					if err != nil {
-						writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
+						delErr := fmt.Errorf("failed to delete existing service account(%s) before importing it: %w", svcAcctReq.AccessKey, err)
+						writeErrorResponseJSON(ctx, w, importError(ctx, delErr, allSvcAcctsFile, user), r.URL)
 						return
 					}
-					continue
 				}
 				opts := newServiceAccountOpts{
 					accessKey:                  user,
@@ -2265,18 +2300,6 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					allowSiteReplicatorAccount: false,
 				}
 
-				// In case of LDAP we need to resolve the targetUser to a DN and
-				// query their groups:
-				if globalIAMSys.LDAPConfig.Enabled() {
-					opts.claims[ldapUserN] = svcAcctReq.AccessKey // simple username
-					targetUser, _, err := globalIAMSys.LDAPConfig.LookupUserDN(svcAcctReq.AccessKey)
-					if err != nil {
-						writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
-						return
-					}
-					opts.claims[ldapUser] = targetUser // username DN
-				}
-
 				if _, _, err = globalIAMSys.NewServiceAccount(ctx, svcAcctReq.Parent, svcAcctReq.Groups, opts); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
 					return
@@ -2345,6 +2368,17 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, groupPolicyMappingsFile, ""), r.URL)
 				return
 			}
+
+			// Validations for LDAP enabled deployments.
+			if globalIAMSys.LDAPConfig.Enabled() {
+				isGroup := true
+				err := globalIAMSys.NormalizeLDAPMappingImport(ctx, isGroup, grpPolicyMap)
+				if err != nil {
+					writeErrorResponseJSON(ctx, w, importError(ctx, err, groupPolicyMappingsFile, ""), r.URL)
+					return
+				}
+			}
+
 			for g, pm := range grpPolicyMap {
 				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, unknownIAMUserType, true); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, groupPolicyMappingsFile, g), r.URL)
@@ -2374,6 +2408,16 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, stsUserPolicyMappingsFile, ""), r.URL)
 				return
 			}
+
+			// Validations for LDAP enabled deployments.
+			if globalIAMSys.LDAPConfig.Enabled() {
+				isGroup := true
+				err := globalIAMSys.NormalizeLDAPMappingImport(ctx, !isGroup, userPolicyMap)
+				if err != nil {
+					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, ""), r.URL)
+					return
+				}
+			}
 			for u, pm := range userPolicyMap {
 				// disallow setting policy mapping if user is a temporary user
 				ok, _, err := globalIAMSys.IsTempUser(u)
@@ -2385,6 +2429,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					writeErrorResponseJSON(ctx, w, importError(ctx, errIAMActionNotAllowed, stsUserPolicyMappingsFile, u), r.URL)
 					return
 				}
+
 				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, stsUser, false); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, u), r.URL)
 					return
@@ -2392,35 +2437,6 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 	}
-
-	// import sts group policy mappings
-	{
-		f, err := zr.Open(pathJoin(iamAssetsDir, stsGroupPolicyMappingsFile))
-		switch {
-		case errors.Is(err, os.ErrNotExist):
-		case err != nil:
-			writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, stsGroupPolicyMappingsFile, ""), r.URL)
-			return
-		default:
-			defer f.Close()
-			var grpPolicyMap map[string]MappedPolicy
-			data, err := io.ReadAll(f)
-			if err != nil {
-				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, stsGroupPolicyMappingsFile, ""), r.URL)
-				return
-			}
-			if err = json.Unmarshal(data, &grpPolicyMap); err != nil {
-				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, stsGroupPolicyMappingsFile, ""), r.URL)
-				return
-			}
-			for g, pm := range grpPolicyMap {
-				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, unknownIAMUserType, true); err != nil {
-					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsGroupPolicyMappingsFile, g), r.URL)
-					return
-				}
-			}
-		}
-	}
 }
 
 func addExpirationToCondValues(exp *time.Time, condValues map[string][]string) {

cmd/admin-handlers.go

@@ -36,6 +36,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
 	"regexp"
 	"runtime"
 	"sort"
@@ -130,7 +131,7 @@ func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.R
 	// Download Binary Once
 	binC, bin, err := downloadBinary(u, mode)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -354,7 +355,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 	// Download Binary Once
 	binC, bin, err := downloadBinary(u, mode)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -368,7 +369,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 				StatusCode: http.StatusInternalServerError,
 			}
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+			adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
@@ -376,7 +377,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 
 	err = verifyBinary(u, sha256Sum, releaseInfo, mode, bytes.NewReader(bin))
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -389,7 +390,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 				StatusCode: http.StatusInternalServerError,
 			}
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+			adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
@@ -397,7 +398,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 
 	err = commitBinary()
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -420,7 +421,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 	for _, nerr := range globalNotificationSys.SignalService(serviceRestart) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
+			adminLogIf(ctx, nerr.Err)
 		}
 	}
 
@@ -451,7 +452,7 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 	case madmin.ServiceActionUnfreeze:
 		serviceSig = serviceUnFreeze
 	default:
-		logger.LogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind)
+		adminLogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
 		return
 	}
@@ -473,7 +474,7 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 	for _, nerr := range globalNotificationSys.SignalService(serviceSig) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
+			adminLogIf(ctx, nerr.Err)
 		}
 	}
 
@@ -534,7 +535,7 @@ func (a adminAPIHandlers) ServiceV2Handler(w http.ResponseWriter, r *http.Reques
 	case madmin.ServiceActionUnfreeze:
 		serviceSig = serviceUnFreeze
 	default:
-		logger.LogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind)
+		adminLogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
 		return
 	}
@@ -1239,7 +1240,7 @@ func extractHealInitParams(vars map[string]string, qParams url.Values, r io.Read
 	if hip.clientToken == "" {
 		jerr := json.NewDecoder(r).Decode(&hip.hs)
 		if jerr != nil {
-			logger.LogIf(GlobalContext, jerr, logger.ErrorKind)
+			adminLogIf(GlobalContext, jerr, logger.ErrorKind)
 			err = ErrRequestBodyParse
 			return
 		}
@@ -1433,7 +1434,7 @@ func getAggregatedBackgroundHealState(ctx context.Context, o ObjectLayer) (madmi
 		var errCount int
 		for _, nerr := range nerrs {
 			if nerr.Err != nil {
-				logger.LogIf(ctx, nerr.Err)
+				adminLogIf(ctx, nerr.Err)
 				errCount++
 			}
 		}
@@ -1561,7 +1562,7 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
 		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 			// would mean the network is not stable. Logging here will help in debugging network issues.
 			if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
-				logger.LogIf(ctx, err)
+				adminLogIf(ctx, err)
 			}
 		}
 		totalRx += n
@@ -1575,7 +1576,8 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
 
 // NetperfHandler - perform mesh style network throughput test
 func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
+	ctx, cancel := context.WithCancel(r.Context())
+	defer cancel()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
@@ -1596,6 +1598,15 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
 	ctx = lkctx.Context()
 	defer nsLock.Unlock(lkctx)
 
+	// Freeze all incoming S3 API calls before running speedtest.
+	globalNotificationSys.ServiceFreeze(ctx, true)
+
+	// Unfreeze as soon as request context is canceled or when the function returns.
+	go func() {
+		<-ctx.Done()
+		globalNotificationSys.ServiceFreeze(ctx, false)
+	}()
+
 	durationStr := r.Form.Get(peerRESTDuration)
 	duration, err := time.ParseDuration(durationStr)
 	if err != nil {
@@ -1621,7 +1632,8 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
 // increasing concurrency and stopping when we have reached the limits on the
 // system.
 func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
+	ctx, cancel := context.WithCancel(r.Context())
+	defer cancel()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
@@ -1691,8 +1703,11 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 	// Freeze all incoming S3 API calls before running speedtest.
 	globalNotificationSys.ServiceFreeze(ctx, true)
 
-	// unfreeze all incoming S3 API calls after speedtest.
-	defer globalNotificationSys.ServiceFreeze(ctx, false)
+	// Unfreeze as soon as request context is canceled or when the function returns.
+	go func() {
+		<-ctx.Done()
+		globalNotificationSys.ServiceFreeze(ctx, false)
+	}()
 
 	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
 	defer keepAliveTicker.Stop()
@@ -1792,7 +1807,8 @@ func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo,
 
 // DriveSpeedtestHandler - reports throughput of drives available in the cluster
 func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
+	ctx, cancel := context.WithCancel(r.Context())
+	defer cancel()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
@@ -1802,8 +1818,11 @@ func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.R
 	// Freeze all incoming S3 API calls before running speedtest.
 	globalNotificationSys.ServiceFreeze(ctx, true)
 
-	// unfreeze all incoming S3 API calls after speedtest.
-	defer globalNotificationSys.ServiceFreeze(ctx, false)
+	// Unfreeze as soon as request context is canceled or when the function returns.
+	go func() {
+		<-ctx.Done()
+		globalNotificationSys.ServiceFreeze(ctx, false)
+	}()
 
 	serial := r.Form.Get("serial") == "true"
 	blockSizeStr := r.Form.Get("blocksize")
@@ -2375,7 +2394,7 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
 	}
 
 	client := &http.Client{
-		Transport: globalHealthChkTransport,
+		Transport: globalRemoteTargetTransport,
 		Timeout:   10 * time.Second,
 	}
 
@@ -2800,7 +2819,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 			w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID))
 		encodedErrorResponse := encodeResponse(errorResponse)
 		healthInfo.Error = string(encodedErrorResponse)
-		logger.LogIf(ctx, enc.Encode(healthInfo))
+		adminLogIf(ctx, enc.Encode(healthInfo))
 	}
 
 	deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics.
@@ -3113,7 +3132,7 @@ func getClusterMetaInfo(ctx context.Context) []byte {
 	case ci := <-resultCh:
 		out, err := json.MarshalIndent(ci, "", "  ")
 		if err != nil {
-			logger.LogIf(ctx, err)
+			bugLogIf(ctx, err)
 			return nil
 		}
 		return out
@@ -3172,11 +3191,11 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
-	file = strings.ReplaceAll(file, string(os.PathSeparator), "/")
 
+	file = filepath.ToSlash(file)
 	// Reject attempts to traverse parent or absolute paths.
-	if strings.Contains(file, "..") || strings.Contains(volume, "..") {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+	if hasBadPathComponent(volume) || hasBadPathComponent(file) {
+		writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL)
 		return
 	}
 
@@ -3195,6 +3214,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 			return
 		}
 	}
+	addErr := func(msg string) {}
 
 	// Write a version for making *incompatible* changes.
 	// The AdminClient will reject any version it does not know.
@@ -3206,18 +3226,18 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 
 		clusterKey, err := bytesToPublicKey(getSubnetAdminPublicKey())
 		if err != nil {
-			logger.LogIf(ctx, stream.AddError(err.Error()))
+			bugLogIf(ctx, stream.AddError(err.Error()))
 			return
 		}
 		err = stream.AddKeyEncrypted(clusterKey)
 		if err != nil {
-			logger.LogIf(ctx, stream.AddError(err.Error()))
+			bugLogIf(ctx, stream.AddError(err.Error()))
 			return
 		}
 		if b := getClusterMetaInfo(ctx); len(b) > 0 {
 			w, err := stream.AddEncryptedStream("cluster.info", nil)
 			if err != nil {
-				logger.LogIf(ctx, err)
+				bugLogIf(ctx, err)
 				return
 			}
 			w.Write(b)
@@ -3226,14 +3246,19 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 
 		// Add new key for inspect data.
 		if err := stream.AddKeyEncrypted(publicKey); err != nil {
-			logger.LogIf(ctx, stream.AddError(err.Error()))
+			bugLogIf(ctx, stream.AddError(err.Error()))
 			return
 		}
 		encStream, err := stream.AddEncryptedStream("inspect.zip", nil)
 		if err != nil {
-			logger.LogIf(ctx, stream.AddError(err.Error()))
+			bugLogIf(ctx, stream.AddError(err.Error()))
 			return
 		}
+		addErr = func(msg string) {
+			inspectZipW.Close()
+			encStream.Close()
+			stream.AddError(msg)
+		}
 		defer encStream.Close()
 
 		inspectZipW = zip.NewWriter(encStream)
@@ -3244,7 +3269,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		// MUST use crypto/rand
 		n, err := crand.Read(key[:])
 		if err != nil || n != len(key) {
-			logger.LogIf(ctx, err)
+			bugLogIf(ctx, err)
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
@@ -3258,7 +3283,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 
 		stream, err := sio.AES_256_GCM.Stream(key[:])
 		if err != nil {
-			logger.LogIf(ctx, err)
+			bugLogIf(ctx, err)
 			return
 		}
 		// Zero nonce, we only use each key once, and 32 bytes is plenty.
@@ -3272,7 +3297,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		defer inspectZipW.Close()
 
 		if b := getClusterMetaInfo(ctx); len(b) > 0 {
-			logger.LogIf(ctx, embedFileInZip(inspectZipW, "cluster.info", b, 0o600))
+			adminLogIf(ctx, embedFileInZip(inspectZipW, "cluster.info", b, 0o600))
 		}
 	}
 
@@ -3300,32 +3325,20 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 			sys:     nil,
 		})
 		if zerr != nil {
-			logger.LogIf(ctx, zerr)
+			bugLogIf(ctx, zerr)
 			return nil
 		}
 		header.Method = zip.Deflate
 		zwriter, zerr := inspectZipW.CreateHeader(header)
 		if zerr != nil {
-			logger.LogIf(ctx, zerr)
+			bugLogIf(ctx, zerr)
 			return nil
 		}
 		if _, err := io.Copy(zwriter, r); err != nil {
-			logger.LogIf(ctx, err)
+			adminLogIf(ctx, err)
 		}
 		return nil
 	}
-	err := o.GetRawData(ctx, volume, file, rawDataFn)
-	if !errors.Is(err, errFileNotFound) {
-		logger.LogIf(ctx, err)
-	}
-
-	// save the format.json as part of inspect by default
-	if !(volume == minioMetaBucket && file == formatConfigFile) {
-		err = o.GetRawData(ctx, minioMetaBucket, formatConfigFile, rawDataFn)
-	}
-	if !errors.Is(err, errFileNotFound) {
-		logger.LogIf(ctx, err)
-	}
 
 	// save args passed to inspect command
 	var sb bytes.Buffer
@@ -3336,7 +3349,25 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		sb.WriteString(pool.CmdLine)
 	}
 	sb.WriteString("\n")
-	logger.LogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes(), 0o600))
+	adminLogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes(), 0o600))
+
+	err := o.GetRawData(ctx, volume, file, rawDataFn)
+	if err != nil {
+		if errors.Is(err, errFileNotFound) {
+			addErr("GetRawData: No files matched the given pattern")
+			return
+		}
+		embedFileInZip(inspectZipW, "GetRawData-err.txt", []byte(err.Error()), 0o600)
+		adminLogIf(ctx, err)
+	}
+
+	// save the format.json as part of inspect by default
+	if !(volume == minioMetaBucket && file == formatConfigFile) {
+		err = o.GetRawData(ctx, minioMetaBucket, formatConfigFile, rawDataFn)
+	}
+	if !errors.Is(err, errFileNotFound) {
+		adminLogIf(ctx, err)
+	}
 
 	scheme := "https"
 	if !globalIsTLS {
@@ -3370,7 +3401,7 @@ function main() {
 }
 
 main "$@"`, scheme)
-	logger.LogIf(ctx, embedFileInZip(inspectZipW, "start-minio.sh", scrb.Bytes(), 0o755))
+	adminLogIf(ctx, embedFileInZip(inspectZipW, "start-minio.sh", scrb.Bytes(), 0o755))
 }
 
 func getSubnetAdminPublicKey() []byte {

cmd/admin-heal-ops.go

@@ -329,21 +329,25 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence, objAPI ObjectLay
 	// Add heal state and start sequence
 	ahs.healSeqMap[hpath] = h
 
-	// Launch top-level background heal go-routine
-	go h.healSequenceStart(objAPI)
-
 	clientToken := h.clientToken
 	if globalIsDistErasure {
 		clientToken = fmt.Sprintf("%s:%d", h.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
 	}
 
+	if h.clientToken == bgHealingUUID {
+		// For background heal do nothing, do not spawn an unnecessary goroutine.
+	} else {
+		// Launch top-level background heal go-routine
+		go h.healSequenceStart(objAPI)
+	}
+
 	b, err := json.Marshal(madmin.HealStartSuccess{
 		ClientToken:   clientToken,
 		ClientAddress: h.clientAddress,
 		StartTime:     h.startTime,
 	})
 	if err != nil {
-		logger.LogIf(h.ctx, err)
+		bugLogIf(h.ctx, err)
 		return nil, toAdminAPIErr(h.ctx, err), ""
 	}
 	return b, noError, ""
@@ -390,7 +394,7 @@ func (ahs *allHealState) PopHealStatusJSON(hpath string,
 	if err != nil {
 		h.currentStatus.Items = nil
 
-		logger.LogIf(h.ctx, err)
+		bugLogIf(h.ctx, err)
 		return nil, ErrInternalError
 	}
 
@@ -537,9 +541,9 @@ func (h *healSequence) getHealedItemsMap() map[madmin.HealItemType]int64 {
 	return retMap
 }
 
-// gethealFailedItemsMap - returns map of all items where heal failed against
+// getHealFailedItemsMap - returns map of all items where heal failed against
 // drive endpoint and status
-func (h *healSequence) gethealFailedItemsMap() map[string]int64 {
+func (h *healSequence) getHealFailedItemsMap() map[string]int64 {
 	h.mutex.RLock()
 	defer h.mutex.RUnlock()
 
@@ -552,6 +556,32 @@ func (h *healSequence) gethealFailedItemsMap() map[string]int64 {
 	return retMap
 }
 
+func (h *healSequence) countFailed(res madmin.HealResultItem) {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
+
+	for _, d := range res.After.Drives {
+		// For failed items we report the endpoint and drive state
+		// This will help users take corrective actions for drives
+		h.healFailedItemsMap[d.Endpoint+","+d.State]++
+	}
+
+	h.lastHealActivity = UTCNow()
+}
+
+func (h *healSequence) countHeals(healType madmin.HealItemType, healed bool) {
+	h.mutex.Lock()
+	defer h.mutex.Unlock()
+
+	if !healed {
+		h.scannedItemsMap[healType]++
+	} else {
+		h.healedItemsMap[healType]++
+	}
+
+	h.lastHealActivity = UTCNow()
+}
+
 // isQuitting - determines if the heal sequence is quitting (due to an
 // external signal)
 func (h *healSequence) isQuitting() bool {
@@ -704,10 +734,7 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		task.opts.ScanMode = madmin.HealNormalScan
 	}
 
-	h.mutex.Lock()
-	h.scannedItemsMap[healType]++
-	h.lastHealActivity = UTCNow()
-	h.mutex.Unlock()
+	h.countHeals(healType, false)
 
 	if source.noWait {
 		select {
@@ -744,32 +771,11 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 				return nil
 			}
 
-			h.mutex.Lock()
-			defer h.mutex.Unlock()
-
-			// Progress is not reported in case of background heal processing.
-			// Instead we increment relevant counter based on the heal result
-			// for prometheus reporting.
-			if res.err != nil {
-				for _, d := range res.result.After.Drives {
-					// For failed items we report the endpoint and drive state
-					// This will help users take corrective actions for drives
-					h.healFailedItemsMap[d.Endpoint+","+d.State]++
-				}
-			} else {
-				// Only object type reported for successful healing
-				h.healedItemsMap[res.result.Type]++
-			}
-
 			// Report caller of any failure
 			return res.err
 		}
 		res.result.Type = healType
 		if res.err != nil {
-			// Only report object error
-			if healType != madmin.HealItemObject {
-				return res.err
-			}
 			res.result.Detail = res.err.Error()
 		}
 		return h.pushHealResultItem(res.result)

cmd/admin-server-info.go

@@ -31,7 +31,6 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
 )
 
 // getLocalServerProperty - returns madmin.ServerProperties for only the
@@ -67,7 +66,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 				} else {
 					network[nodeName] = string(madmin.ItemOffline)
 					// log once the error
-					logger.LogOnceIf(context.Background(), err, nodeName)
+					peersLogOnceIf(context.Background(), err, nodeName)
 				}
 			}
 		}

cmd/api-errors.go

@@ -263,6 +263,7 @@ const (
 	ErrInvalidResourceName
 	ErrInvalidLifecycleQueryParameter
 	ErrServerNotInitialized
+	ErrBucketMetadataNotInitialized
 	ErrRequestTimedout
 	ErrClientDisconnected
 	ErrTooManyRequests
@@ -278,6 +279,7 @@ const (
 	ErrMalformedJSON
 	ErrAdminNoSuchUser
 	ErrAdminNoSuchUserLDAPWarn
+	ErrAdminLDAPExpectedLoginName
 	ErrAdminNoSuchGroup
 	ErrAdminGroupNotEmpty
 	ErrAdminGroupDisabled
@@ -300,6 +302,7 @@ const (
 	ErrAdminConfigIDPCfgNameDoesNotExist
 	ErrInsecureClientRequest
 	ErrObjectTampered
+	ErrAdminLDAPNotEnabled
 
 	// Site-Replication errors
 	ErrSiteReplicationInvalidRequest
@@ -1293,7 +1296,12 @@ var errorCodes = errorCodeMap{
 	},
 	ErrServerNotInitialized: {
 		Code:           "XMinioServerNotInitialized",
-		Description:    "Server not initialized, please try again.",
+		Description:    "Server not initialized yet, please try again.",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrBucketMetadataNotInitialized: {
+		Code:           "XMinioBucketMetadataNotInitialized",
+		Description:    "Bucket metadata not initialized yet, please try again.",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
 	ErrMalformedJSON: {
@@ -2079,7 +2087,16 @@ var errorCodes = errorCodeMap{
 		Description:    "Invalid attribute name specified.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	// Add your error structure here.
+	ErrAdminLDAPNotEnabled: {
+		Code:           "XMinioLDAPNotEnabled",
+		Description:    "LDAP is not enabled. LDAP must be enabled to make LDAP requests.",
+		HTTPStatusCode: http.StatusNotImplemented,
+	},
+	ErrAdminLDAPExpectedLoginName: {
+		Code:           "XMinioLDAPExpectedLoginName",
+		Description:    "Expected LDAP short username but was given full DN.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 }
 
 // toAPIErrorCode - Converts embedded errors. Convenience
@@ -2200,6 +2217,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrInvalidMaxParts
 	case ioutil.ErrOverread:
 		apiErr = ErrExcessData
+	case errServerNotInitialized:
+		apiErr = ErrServerNotInitialized
+	case errBucketMetadataNotInitialized:
+		apiErr = ErrBucketMetadataNotInitialized
 	}
 
 	// Compression errors
@@ -2519,7 +2540,7 @@ func toAPIError(ctx context.Context, err error) APIError {
 		// Make sure to log the errors which we cannot translate
 		// to a meaningful S3 API errors. This is added to aid in
 		// debugging unexpected/unhandled errors.
-		logger.LogIf(ctx, err)
+		internalLogIf(ctx, err)
 	}
 
 	return apiErr

cmd/api-headers.go

@@ -30,7 +30,6 @@ import (
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 	xxml "github.com/minio/xxml"
 )
 
@@ -68,7 +67,7 @@ func encodeResponse(response interface{}) []byte {
 	var buf bytes.Buffer
 	buf.WriteString(xml.Header)
 	if err := xml.NewEncoder(&buf).Encode(response); err != nil {
-		logger.LogIf(GlobalContext, err)
+		bugLogIf(GlobalContext, err)
 		return nil
 	}
 	return buf.Bytes()
@@ -86,7 +85,7 @@ func encodeResponseList(response interface{}) []byte {
 	var buf bytes.Buffer
 	buf.WriteString(xxml.Header)
 	if err := xxml.NewEncoder(&buf).Encode(response); err != nil {
-		logger.LogIf(GlobalContext, err)
+		bugLogIf(GlobalContext, err)
 		return nil
 	}
 	return buf.Bytes()

cmd/api-response.go

@@ -891,7 +891,7 @@ func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType
 	}
 	// Similar check to http.checkWriteHeaderCode
 	if statusCode < 100 || statusCode > 999 {
-		logger.LogIf(context.Background(), fmt.Errorf("invalid WriteHeader code %v", statusCode))
+		bugLogIf(context.Background(), fmt.Errorf("invalid WriteHeader code %v", statusCode))
 		statusCode = http.StatusInternalServerError
 	}
 	setCommonHeaders(w)
@@ -961,7 +961,7 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 
 	// Similar check to http.checkWriteHeaderCode
 	if err.HTTPStatusCode < 100 || err.HTTPStatusCode > 999 {
-		logger.LogIf(ctx, fmt.Errorf("invalid WriteHeader code %v from %v", err.HTTPStatusCode, err.Code))
+		bugLogIf(ctx, fmt.Errorf("invalid WriteHeader code %v from %v", err.HTTPStatusCode, err.Code))
 		err.HTTPStatusCode = http.StatusInternalServerError
 	}
 

cmd/auth-handler.go

@@ -126,7 +126,7 @@ func getRequestAuthType(r *http.Request) (at authType) {
 		var err error
 		r.Form, err = url.ParseQuery(r.URL.RawQuery)
 		if err != nil {
-			logger.LogIf(r.Context(), err)
+			authNLogIf(r.Context(), err)
 			return authTypeUnknown
 		}
 	}
@@ -257,7 +257,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 		if err != nil {
 			// Base64 decoding fails, we should log to indicate
 			// something is malforming the request sent by client.
-			logger.LogIf(GlobalContext, err, logger.ErrorKind)
+			authNLogIf(GlobalContext, err, logger.ErrorKind)
 			return nil, errAuthentication
 		}
 		claims.MapClaims[sessionPolicyNameExtracted] = string(spBytes)
@@ -353,7 +353,7 @@ func checkRequestAuthTypeWithVID(ctx context.Context, r *http.Request, action po
 
 func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
 	if logger.GetReqInfo(ctx) == nil {
-		logger.LogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.ErrorKind)
+		bugLogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.ErrorKind)
 		return ErrAccessDenied
 	}
 
@@ -392,7 +392,7 @@ func authenticateRequest(ctx context.Context, r *http.Request, action policy.Act
 		// To extract region from XML in request body, get copy of request body.
 		payload, err := io.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
 		if err != nil {
-			logger.LogIf(ctx, err, logger.ErrorKind)
+			authZLogIf(ctx, err, logger.ErrorKind)
 			return ErrMalformedXML
 		}
 

cmd/background-heal-ops.go

@@ -25,7 +25,6 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go/v3"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
 )
 
@@ -101,16 +100,17 @@ func waitForLowHTTPReq() {
 }
 
 func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
+	bgSeq := newBgHealSequence()
 	// Run the background healer
 	for i := 0; i < globalBackgroundHealRoutine.workers; i++ {
-		go globalBackgroundHealRoutine.AddWorker(ctx, objAPI)
+		go globalBackgroundHealRoutine.AddWorker(ctx, objAPI, bgSeq)
 	}
 
-	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence(), objAPI)
+	globalBackgroundHealState.LaunchNewHealSequence(bgSeq, objAPI)
 }
 
 // Wait for heal requests and process them
-func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer) {
+func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer, bgSeq *healSequence) {
 	for {
 		select {
 		case task, ok := <-h.tasks:
@@ -133,6 +133,15 @@ func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer) {
 				}
 			}
 
+			if bgSeq != nil {
+				// We increment relevant counter based on the heal result for prometheus reporting.
+				if err != nil {
+					bgSeq.countFailed(res)
+				} else {
+					bgSeq.countHeals(res.Type, false)
+				}
+			}
+
 			if task.respCh != nil {
 				task.respCh <- healResult{result: res, err: err}
 			}
@@ -148,7 +157,7 @@ func newHealRoutine() *healRoutine {
 
 	if envHealWorkers := env.Get("_MINIO_HEAL_WORKERS", ""); envHealWorkers != "" {
 		if numHealers, err := strconv.Atoi(envHealWorkers); err != nil {
-			logger.LogIf(context.Background(), fmt.Errorf("invalid _MINIO_HEAL_WORKERS value: %w", err))
+			bugLogIf(context.Background(), fmt.Errorf("invalid _MINIO_HEAL_WORKERS value: %w", err))
 		} else {
 			workers = numHealers
 		}

cmd/background-newdisks-heal-ops.go

@@ -33,7 +33,6 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
 )
 
@@ -409,11 +408,11 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		if errors.Is(err, errFileNotFound) {
 			return nil
 		}
-		logger.LogIf(ctx, fmt.Errorf("Unable to load healing tracker on '%s': %w, re-initializing..", disk, err))
+		healingLogIf(ctx, fmt.Errorf("Unable to load healing tracker on '%s': %w, re-initializing..", disk, err))
 		tracker = initHealingTracker(disk, mustGetUUID())
 	}
 
-	logger.Event(ctx, "Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint)
+	healingLogEvent(ctx, "Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint)
 
 	buckets, _ := z.ListBuckets(ctx, BucketOptions{})
 	// Buckets data are dispersed in multiple pools/sets, make
@@ -452,7 +451,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		return err
 	}
 
-	logger.Event(ctx, "Healing of drive '%s' is finished (healed: %d, skipped: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
+	healingLogEvent(ctx, "Healing of drive '%s' is finished (healed: %d, skipped: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
 
 	if len(tracker.QueuedBuckets) > 0 {
 		return fmt.Errorf("not all buckets were healed: %v", tracker.QueuedBuckets)
@@ -464,7 +463,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	if tracker.HealID == "" { // HealID was empty only before Feb 2023
-		logger.LogIf(ctx, tracker.delete(ctx))
+		bugLogIf(ctx, tracker.delete(ctx))
 		return nil
 	}
 
@@ -482,7 +481,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		t, err := loadHealingTracker(ctx, disk)
 		if err != nil {
 			if !errors.Is(err, errFileNotFound) {
-				logger.LogIf(ctx, err)
+				healingLogIf(ctx, err)
 			}
 			continue
 		}
@@ -517,7 +516,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
 			// Reformat disks immediately
 			_, err := z.HealFormat(context.Background(), false)
 			if err != nil && !errors.Is(err, errNoHealRequired) {
-				logger.LogIf(ctx, err)
+				healingLogIf(ctx, err)
 				// Reset for next interval.
 				diskCheckTimer.Reset(defaultMonitorNewDiskInterval)
 				continue

cmd/batch-expire.go

@@ -33,7 +33,6 @@ import (
 	"github.com/minio/minio/internal/bucket/versioning"
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
 	"github.com/minio/pkg/v2/wildcard"
 	"github.com/minio/pkg/v2/workers"
@@ -156,7 +155,7 @@ func (ef BatchJobExpireFilter) Matches(obj ObjectInfo, now time.Time) bool {
 		}
 	default:
 		// we should never come here, Validate should have caught this.
-		logger.LogOnceIf(context.Background(), fmt.Errorf("invalid filter type: %s", ef.Type), ef.Type)
+		batchLogOnceIf(context.Background(), fmt.Errorf("invalid filter type: %s", ef.Type), ef.Type)
 		return false
 	}
 
@@ -321,7 +320,7 @@ func (r BatchJobExpire) Notify(ctx context.Context, body io.Reader) error {
 		req.Header.Set("Authorization", r.NotificationCfg.Token)
 	}
 
-	clnt := http.Client{Transport: getRemoteInstanceTransport}
+	clnt := http.Client{Transport: getRemoteInstanceTransport()}
 	resp, err := clnt.Do(req)
 	if err != nil {
 		return err
@@ -433,7 +432,7 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
 					})
 					if err != nil {
 						stopFn(exp, err)
-						logger.LogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", toExpire[i].Bucket, toExpire[i].Name, toExpire[i].VersionID, err, attempts))
+						batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", toExpire[i].Bucket, toExpire[i].Name, toExpire[i].VersionID, err, attempts))
 					} else {
 						stopFn(exp, err)
 						success = true
@@ -471,7 +470,7 @@ func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo
 				for i, err := range errs {
 					if err != nil {
 						stopFn(toDelCopy[i], err)
-						logger.LogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID, err, attempts))
+						batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID, err, attempts))
 						failed++
 						if attempts == retryAttempts { // all retry attempts failed, record failure
 							if oi, ok := oiCache.Get(toDelCopy[i]); ok {
@@ -557,16 +556,16 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 			select {
 			case <-saveTicker.C:
 				// persist in-memory state to disk after every 10secs.
-				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 
 			case <-ctx.Done():
 				// persist in-memory state immediately before exiting due to context cancellation.
-				logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+				batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 				return
 
 			case <-saverQuitCh:
 				// persist in-memory state immediately to disk.
-				logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+				batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 				return
 			}
 		}
@@ -670,7 +669,7 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 	// Notify expire jobs final status to the configured endpoint
 	buf, _ := json.Marshal(ri)
 	if err := r.Notify(context.Background(), bytes.NewReader(buf)); err != nil {
-		logger.LogIf(context.Background(), fmt.Errorf("unable to notify %v", err))
+		batchLogIf(context.Background(), fmt.Errorf("unable to notify %v", err))
 	}
 
 	return nil

cmd/batch-handlers.go

@@ -48,7 +48,6 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/ioutil"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 	"github.com/minio/pkg/v2/env"
 	"github.com/minio/pkg/v2/policy"
@@ -92,7 +91,7 @@ func notifyEndpoint(ctx context.Context, ri *batchJobInfo, endpoint, token strin
 	}
 	req.Header.Set("Content-Type", "application/json")
 
-	clnt := http.Client{Transport: getRemoteInstanceTransport}
+	clnt := http.Client{Transport: getRemoteInstanceTransport()}
 	resp, err := clnt.Do(req)
 	if err != nil {
 		return err
@@ -206,7 +205,7 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a
 				if aerr == nil {
 					return
 				}
-				logger.LogIf(ctx,
+				batchLogIf(ctx,
 					fmt.Errorf("trying %s: Unable to cleanup failed multipart replication %s on remote %s/%s: %w - this may consume space on remote cluster",
 						humanize.Ordinal(attempts), res.UploadID, tgtBucket, tgtObject, aerr))
 				attempts++
@@ -351,7 +350,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	c, err := miniogo.New(u.Host, &miniogo.Options{
 		Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
 		Secure:       u.Scheme == "https",
-		Transport:    getRemoteInstanceTransport,
+		Transport:    getRemoteInstanceTransport(),
 		BucketLookup: lookupStyle(r.Source.Path),
 	})
 	if err != nil {
@@ -402,7 +401,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 					} else {
 						if !isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) &&
 							!isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) {
-							logger.LogIf(ctx, err)
+							batchLogIf(ctx, err)
 						}
 						continue
 					}
@@ -414,7 +413,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 					} else {
 						if !isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) &&
 							!isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) {
-							logger.LogIf(ctx, err)
+							batchLogIf(ctx, err)
 						}
 						continue
 					}
@@ -443,7 +442,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 						return
 					}
 					stopFn(oi, err)
-					logger.LogIf(ctx, err)
+					batchLogIf(ctx, err)
 					success = false
 				} else {
 					stopFn(oi, nil)
@@ -451,7 +450,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 				ri.trackCurrentBucketObject(r.Target.Bucket, oi, success)
 				globalBatchJobsMetrics.save(job.ID, ri)
 				// persist in-memory state to disk after every 10secs.
-				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 
 				if wait := globalBatchConfig.ReplicationWait(); wait > 0 {
 					time.Sleep(wait)
@@ -466,10 +465,10 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 
 		globalBatchJobsMetrics.save(job.ID, ri)
 		// persist in-memory state to disk.
-		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+		batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 
 		if err := r.Notify(ctx, ri); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
+			batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
 		}
 
 		cancel()
@@ -553,7 +552,7 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
 					VersionID: entry.VersionID,
 				})
 			if err != nil {
-				logger.LogIf(ctx, err)
+				batchLogIf(ctx, err)
 				continue
 			}
 
@@ -572,7 +571,7 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
 
 			opts, err := batchReplicationOpts(ctx, "", gr.ObjInfo)
 			if err != nil {
-				logger.LogIf(ctx, err)
+				batchLogIf(ctx, err)
 				continue
 			}
 
@@ -1048,7 +1047,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	c, err := miniogo.NewCore(u.Host, &miniogo.Options{
 		Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
 		Secure:       u.Scheme == "https",
-		Transport:    getRemoteInstanceTransport,
+		Transport:    getRemoteInstanceTransport(),
 		BucketLookup: lookupStyle(r.Target.Path),
 	})
 	if err != nil {
@@ -1068,11 +1067,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			cl, err := miniogo.New(u.Host, &miniogo.Options{
 				Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
 				Secure:       u.Scheme == "https",
-				Transport:    getRemoteInstanceTransport,
+				Transport:    getRemoteInstanceTransport(),
 				BucketLookup: lookupStyle(r.Target.Path),
 			})
 			if err != nil {
-				logger.LogIf(ctx, err)
+				batchLogIf(ctx, err)
 				return
 			}
 
@@ -1083,7 +1082,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			writeFn := func(batch []ObjectInfo) {
 				if len(batch) > 0 {
 					if err := r.writeAsArchive(ctx, api, cl, batch); err != nil {
-						logger.LogIf(ctx, err)
+						batchLogIf(ctx, err)
 						for _, b := range batch {
 							slowCh <- b
 						}
@@ -1091,7 +1090,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 						ri.trackCurrentBucketBatch(r.Source.Bucket, batch)
 						globalBatchJobsMetrics.save(job.ID, ri)
 						// persist in-memory state to disk after every 10secs.
-						logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+						batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 					}
 				}
 			}
@@ -1179,7 +1178,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 						return
 					}
 					stopFn(result, err)
-					logger.LogIf(ctx, err)
+					batchLogIf(ctx, err)
 					success = false
 				} else {
 					stopFn(result, nil)
@@ -1187,7 +1186,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				ri.trackCurrentBucketObject(r.Source.Bucket, result, success)
 				globalBatchJobsMetrics.save(job.ID, ri)
 				// persist in-memory state to disk after every 10secs.
-				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 
 				if wait := globalBatchConfig.ReplicationWait(); wait > 0 {
 					time.Sleep(wait)
@@ -1202,10 +1201,10 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 		globalBatchJobsMetrics.save(job.ID, ri)
 		// persist in-memory state to disk.
-		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+		batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 
 		if err := r.Notify(ctx, ri); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
+			batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
 		}
 
 		cancel()
@@ -1248,9 +1247,18 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 		return errInvalidArgument
 	}
 
-	if r.Source.Bucket == "" {
+	if r.Source.Endpoint != "" && r.Target.Endpoint != "" {
 		return errInvalidArgument
 	}
+
+	if r.Source.Creds.Empty() && r.Target.Creds.Empty() {
+		return errInvalidArgument
+	}
+
+	if r.Source.Bucket == "" || r.Target.Bucket == "" {
+		return errInvalidArgument
+	}
+
 	var isRemoteToLocal bool
 	localBkt := r.Source.Bucket
 	if r.Source.Endpoint != "" {
@@ -1275,9 +1283,6 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 	if err := r.Source.Snowball.Validate(); err != nil {
 		return err
 	}
-	if r.Source.Creds.Empty() && r.Target.Creds.Empty() {
-		return errInvalidArgument
-	}
 
 	if !r.Source.Creds.Empty() {
 		if err := r.Source.Creds.Validate(); err != nil {
@@ -1299,9 +1304,6 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 	if r.Target.Endpoint != "" && !r.Target.Type.isMinio() && !r.Target.ValidPath() {
 		return errInvalidArgument
 	}
-	if r.Target.Bucket == "" {
-		return errInvalidArgument
-	}
 
 	if !r.Target.Creds.Empty() {
 		if err := r.Target.Creds.Validate(); err != nil {
@@ -1309,10 +1311,6 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 		}
 	}
 
-	if r.Source.Creds.Empty() && r.Target.Creds.Empty() {
-		return errInvalidArgument
-	}
-
 	if err := r.Target.Type.Validate(); err != nil {
 		return err
 	}
@@ -1354,7 +1352,7 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 	c, err := miniogo.NewCore(u.Host, &miniogo.Options{
 		Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
 		Secure:       u.Scheme == "https",
-		Transport:    getRemoteInstanceTransport,
+		Transport:    getRemoteInstanceTransport(),
 		BucketLookup: lookupStyle(pathStyle),
 	})
 	if err != nil {
@@ -1500,7 +1498,7 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
 		req := &BatchJobRequest{}
 		if err := req.load(ctx, objectAPI, result.Name); err != nil {
 			if !errors.Is(err, errNoSuchJob) {
-				logger.LogIf(ctx, err)
+				batchLogIf(ctx, err)
 			}
 			continue
 		}
@@ -1516,7 +1514,7 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
 		}
 	}
 
-	logger.LogIf(ctx, json.NewEncoder(w).Encode(&listResult))
+	batchLogIf(ctx, json.NewEncoder(w).Encode(&listResult))
 }
 
 var errNoSuchJob = errors.New("no such job")
@@ -1539,7 +1537,7 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
 	req := &BatchJobRequest{}
 	if err := req.load(ctx, objectAPI, pathJoin(batchJobPrefix, jobID)); err != nil {
 		if !errors.Is(err, errNoSuchJob) {
-			logger.LogIf(ctx, err)
+			batchLogIf(ctx, err)
 		}
 
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1548,7 +1546,7 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
 
 	buf, err := yaml.Marshal(req)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		batchLogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1707,7 +1705,7 @@ func (j *BatchJobPool) resume() {
 	ctx, cancel := context.WithCancel(j.ctx)
 	defer cancel()
 	if err := j.objLayer.Walk(ctx, minioMetaBucket, batchJobPrefix, results, WalkOptions{}); err != nil {
-		logger.LogIf(j.ctx, err)
+		batchLogIf(j.ctx, err)
 		return
 	}
 	for result := range results {
@@ -1717,7 +1715,7 @@ func (j *BatchJobPool) resume() {
 		}
 		req := &BatchJobRequest{}
 		if err := req.load(ctx, j.objLayer, result.Name); err != nil {
-			logger.LogIf(ctx, err)
+			batchLogIf(ctx, err)
 			continue
 		}
 		_, nodeIdx := parseRequestToken(req.ID)
@@ -1726,7 +1724,7 @@ func (j *BatchJobPool) resume() {
 			continue
 		}
 		if err := j.queueJob(req); err != nil {
-			logger.LogIf(ctx, err)
+			batchLogIf(ctx, err)
 			continue
 		}
 	}
@@ -1750,7 +1748,7 @@ func (j *BatchJobPool) AddWorker() {
 				if job.Replicate.RemoteToLocal() {
 					if err := job.Replicate.StartFromSource(job.ctx, j.objLayer, *job); err != nil {
 						if !isErrBucketNotFound(err) {
-							logger.LogIf(j.ctx, err)
+							batchLogIf(j.ctx, err)
 							j.canceler(job.ID, false)
 							continue
 						}
@@ -1759,7 +1757,7 @@ func (j *BatchJobPool) AddWorker() {
 				} else {
 					if err := job.Replicate.Start(job.ctx, j.objLayer, *job); err != nil {
 						if !isErrBucketNotFound(err) {
-							logger.LogIf(j.ctx, err)
+							batchLogIf(j.ctx, err)
 							j.canceler(job.ID, false)
 							continue
 						}
@@ -1769,14 +1767,14 @@ func (j *BatchJobPool) AddWorker() {
 			case job.KeyRotate != nil:
 				if err := job.KeyRotate.Start(job.ctx, j.objLayer, *job); err != nil {
 					if !isErrBucketNotFound(err) {
-						logger.LogIf(j.ctx, err)
+						batchLogIf(j.ctx, err)
 						continue
 					}
 				}
 			case job.Expire != nil:
 				if err := job.Expire.Start(job.ctx, j.objLayer, *job); err != nil {
 					if !isErrBucketNotFound(err) {
-						logger.LogIf(j.ctx, err)
+						batchLogIf(j.ctx, err)
 						continue
 					}
 				}

cmd/batch-rotate.go

@@ -33,7 +33,6 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
 	"github.com/minio/pkg/v2/workers"
 )
@@ -383,7 +382,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				success := true
 				if err := r.KeyRotate(ctx, api, result); err != nil {
 					stopFn(result, err)
-					logger.LogIf(ctx, err)
+					batchLogIf(ctx, err)
 					success = false
 				} else {
 					stopFn(result, nil)
@@ -392,7 +391,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				ri.RetryAttempts = attempts
 				globalBatchJobsMetrics.save(job.ID, ri)
 				// persist in-memory state to disk after every 10secs.
-				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 				if success {
 					break
 				}
@@ -412,10 +411,10 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	ri.Failed = ri.ObjectsFailed > 0
 	globalBatchJobsMetrics.save(job.ID, ri)
 	// persist in-memory state to disk.
-	logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+	batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 
 	if err := r.Notify(ctx, ri); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
+		batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
 	}
 
 	cancel()

cmd/bootstrap-peer-server.go

@@ -198,7 +198,7 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 				if err != nil {
 					bootstrapTraceMsg(fmt.Sprintf("clnt.Verify: %v, endpoint: %s", err, clnt))
 					if !isNetworkError(err) {
-						logger.LogOnceIf(context.Background(), fmt.Errorf("%s has incorrect configuration: %w", clnt, err), "incorrect_"+clnt.String())
+						bootLogOnceIf(context.Background(), fmt.Errorf("%s has incorrect configuration: %w", clnt, err), "incorrect_"+clnt.String())
 						incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt, err))
 					} else {
 						offlineEndpoints = append(offlineEndpoints, fmt.Errorf("%s is unreachable: %w", clnt, err))

cmd/bucket-encryption-handlers.go

@@ -114,7 +114,7 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeSSEConfig,
 		Bucket:    bucket,
 		SSEConfig: &cfgStr,
@@ -203,7 +203,7 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 	}
 
 	// Call site replication hook.
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeSSEConfig,
 		Bucket:    bucket,
 		SSEConfig: nil,

cmd/bucket-handlers.go

@@ -98,7 +98,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	// Get buckets in the DNS
 	dnsBuckets, err := globalDNSConfig.List()
 	if err != nil && !IsErrIgnored(err, dns.ErrNoEntriesFound, dns.ErrNotImplemented, dns.ErrDomainMissing) {
-		logger.LogIf(GlobalContext, err)
+		dnsLogIf(GlobalContext, err)
 		return
 	}
 
@@ -160,13 +160,13 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 	ctx := GlobalContext
 	for _, err := range g.Wait() {
 		if err != nil {
-			logger.LogIf(ctx, err)
+			dnsLogIf(ctx, err)
 			return
 		}
 	}
 
 	for _, bucket := range bucketsInConflict.ToSlice() {
-		logger.LogIf(ctx, fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket by a different tenant. This local bucket will be ignored. Bucket names are globally unique in federated deployments. Use path style requests on following addresses '%v' to access this bucket", bucket, globalDomainIPs.ToSlice()))
+		dnsLogIf(ctx, fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket by a different tenant. This local bucket will be ignored. Bucket names are globally unique in federated deployments. Use path style requests on following addresses '%v' to access this bucket", bucket, globalDomainIPs.ToSlice()))
 	}
 
 	var wg sync.WaitGroup
@@ -187,7 +187,7 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 			// We go to here, so we know the bucket no longer exists,
 			// but is registered in DNS to this server
 			if err := globalDNSConfig.Delete(bucket); err != nil {
-				logger.LogIf(GlobalContext, fmt.Errorf("Failed to remove DNS entry for %s due to %w",
+				dnsLogIf(GlobalContext, fmt.Errorf("Failed to remove DNS entry for %s due to %w",
 					bucket, err))
 			}
 		}(bucket)
@@ -790,7 +790,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 
 	// check if client is attempting to create more buckets, complain about it.
 	if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
-		logger.LogIf(ctx, fmt.Errorf("Please avoid creating more buckets %d beyond recommended %d", currBuckets+1, maxBuckets))
+		internalLogIf(ctx, fmt.Errorf("Please avoid creating more buckets %d beyond recommended %d", currBuckets+1, maxBuckets), logger.WarningKind)
 	}
 
 	opts := MakeBucketOptions{
@@ -871,7 +871,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 
 	// Call site replication hook
-	logger.LogIf(ctx, globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts))
+	replLogIf(ctx, globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts))
 
 	// Make sure to add Location information here only for bucket
 	w.Header().Set(xhttp.Location, pathJoin(SlashSeparator, bucket))
@@ -1661,7 +1661,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 					writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 					return
 				}
-			case rcfg.HasActiveRules("", true):
+			case rcfg != nil && rcfg.HasActiveRules("", true):
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 				return
 			}
@@ -1693,7 +1693,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	if globalDNSConfig != nil {
 		if err := globalDNSConfig.Delete(bucket); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually, bucket on MinIO no longer exists", err))
+			dnsLogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually, bucket on MinIO no longer exists", err))
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
@@ -1703,7 +1703,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	globalReplicationPool.deleteResyncMetadata(ctx, bucket)
 
 	// Call site replication hook.
-	logger.LogIf(ctx, globalSiteReplicationSys.DeleteBucketHook(ctx, bucket, forceDelete))
+	replLogIf(ctx, globalSiteReplicationSys.DeleteBucketHook(ctx, bucket, forceDelete))
 
 	// Write success response.
 	writeSuccessNoContent(w)
@@ -1776,7 +1776,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:             madmin.SRBucketMetaTypeObjectLockConfig,
 		Bucket:           bucket,
 		ObjectLockConfig: &cfgStr,
@@ -1880,7 +1880,7 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeTags,
 		Bucket:    bucket,
 		Tags:      &cfgStr,
@@ -1956,7 +1956,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeTags,
 		Bucket:    bucket,
 		UpdatedAt: updatedAt,

cmd/bucket-lifecycle.go

@@ -336,7 +336,7 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
 			case newerNoncurrentTask:
 				deleteObjectVersions(es.ctx, es.objAPI, v.bucket, v.versions, v.event)
 			case jentry:
-				logger.LogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName))
+				transitionLogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName))
 			case freeVersionTask:
 				oi := v.ObjectInfo
 				traceFn := globalLifecycleSys.trace(oi)
@@ -355,7 +355,7 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
 				// Remove the remote object
 				err := deleteObjectFromRemoteTier(es.ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier)
 				if ignoreNotFoundErr(err) != nil {
-					logger.LogIf(es.ctx, err)
+					transitionLogIf(es.ctx, err)
 					return
 				}
 
@@ -368,10 +368,10 @@ func (es *expiryState) Worker(input <-chan expiryOp) {
 					auditLogLifecycle(es.ctx, oi, ILMFreeVersionDelete, nil, traceFn)
 				}
 				if ignoreNotFoundErr(err) != nil {
-					logger.LogIf(es.ctx, err)
+					transitionLogIf(es.ctx, err)
 				}
 			default:
-				logger.LogIf(es.ctx, fmt.Errorf("Invalid work type - %v", v))
+				bugLogIf(es.ctx, fmt.Errorf("Invalid work type - %v", v))
 			}
 		}
 	}
@@ -486,7 +486,7 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 			if err := transitionObject(t.ctx, objectAPI, task.objInfo, newLifecycleAuditEvent(task.src, task.event)); err != nil {
 				if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) && !xnet.IsNetworkOrHostDown(err, false) {
 					if !strings.Contains(err.Error(), "use of closed network connection") {
-						logger.LogIf(t.ctx, fmt.Errorf("Transition to %s failed for %s/%s version:%s with %w",
+						transitionLogIf(t.ctx, fmt.Errorf("Transition to %s failed for %s/%s version:%s with %w",
 							task.event.StorageClass, task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
 					}
 				}
@@ -614,7 +614,7 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 		// remote object
 		opts.SkipFreeVersion = true
 	} else {
-		logger.LogIf(ctx, err)
+		transitionLogIf(ctx, err)
 	}
 
 	// Now, delete object from hot-tier namespace
@@ -721,9 +721,9 @@ func auditTierActions(ctx context.Context, tier string, bytes int64) func(err er
 
 // getTransitionedObjectReader returns a reader from the transitioned tier.
 func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, oi ObjectInfo, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	tgtClient, err := globalTierConfigMgr.getDriver(oi.TransitionedObject.Tier)
+	tgtClient, err := globalTierConfigMgr.getDriver(ctx, oi.TransitionedObject.Tier)
 	if err != nil {
-		return nil, fmt.Errorf("transition storage class not configured")
+		return nil, fmt.Errorf("transition storage class not configured: %w", err)
 	}
 
 	fn, off, length, err := NewGetObjectReader(rs, oi, opts)
@@ -879,7 +879,7 @@ func postRestoreOpts(ctx context.Context, r *http.Request, bucket, object string
 	if vid != "" && vid != nullVersionID {
 		_, err := uuid.Parse(vid)
 		if err != nil {
-			logger.LogIf(ctx, err)
+			s3LogIf(ctx, err)
 			return opts, InvalidVersionID{
 				Bucket:    bucket,
 				Object:    object,

cmd/bucket-metadata-sys.go

@@ -46,6 +46,7 @@ type BucketMetadataSys struct {
 	objAPI ObjectLayer
 
 	sync.RWMutex
+	initialized bool
 	metadataMap map[string]BucketMetadata
 }
 
@@ -433,6 +434,8 @@ func (sys *BucketMetadataSys) GetConfigFromDisk(ctx context.Context, bucket stri
 	return loadBucketMetadata(ctx, objAPI, bucket)
 }
 
+var errBucketMetadataNotInitialized = errors.New("bucket metadata not initialized yet")
+
 // GetConfig returns a specific configuration from the bucket metadata.
 // The returned object may not be modified.
 // reloaded will be true if metadata refreshed from disk
@@ -454,6 +457,10 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (met
 	}
 	meta, err = loadBucketMetadata(ctx, objAPI, bucket)
 	if err != nil {
+		if !sys.Initialized() {
+			// bucket metadata not yet initialized
+			return newBucketMetadata(bucket), reloaded, errBucketMetadataNotInitialized
+		}
 		return meta, reloaded, err
 	}
 	sys.Lock()
@@ -498,9 +505,10 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 	}
 
 	errs := g.Wait()
-	for _, err := range errs {
+	for index, err := range errs {
 		if err != nil {
-			logger.LogIf(ctx, err)
+			internalLogOnceIf(ctx, fmt.Errorf("Unable to load bucket metadata, will be retried: %w", err),
+				"load-bucket-metadata-"+buckets[index].Name, logger.WarningKind)
 		}
 	}
 
@@ -542,7 +550,7 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, fa
 		case <-t.C:
 			buckets, err := sys.objAPI.ListBuckets(ctx, BucketOptions{})
 			if err != nil {
-				logger.LogIf(ctx, err)
+				internalLogIf(ctx, err, logger.WarningKind)
 				break
 			}
 
@@ -560,7 +568,7 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, fa
 
 				meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[i].Name)
 				if err != nil {
-					logger.LogIf(ctx, err)
+					internalLogIf(ctx, err, logger.WarningKind)
 					wait() // wait to proceed to next entry.
 					continue
 				}
@@ -583,6 +591,14 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, fa
 	}
 }
 
+// Initialized indicates if bucket metadata sys is initialized atleast once.
+func (sys *BucketMetadataSys) Initialized() bool {
+	sys.RLock()
+	defer sys.RUnlock()
+
+	return sys.initialized
+}
+
 // Loads bucket metadata for all buckets into BucketMetadataSys.
 func (sys *BucketMetadataSys) init(ctx context.Context, buckets []BucketInfo) {
 	count := 100 // load 100 bucket metadata at a time.
@@ -596,6 +612,10 @@ func (sys *BucketMetadataSys) init(ctx context.Context, buckets []BucketInfo) {
 		buckets = buckets[count:]
 	}
 
+	sys.Lock()
+	sys.initialized = true
+	sys.Unlock()
+
 	if globalIsDistErasure {
 		go sys.refreshBucketsMetadataLoop(ctx, failedBuckets)
 	}

cmd/bucket-metadata.go

@@ -145,7 +145,7 @@ func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
 // If an error is returned the returned metadata will be default initialized.
 func readBucketMetadata(ctx context.Context, api ObjectLayer, name string) (BucketMetadata, error) {
 	if name == "" {
-		logger.LogIf(ctx, errors.New("bucket name cannot be empty"))
+		internalLogIf(ctx, errors.New("bucket name cannot be empty"), logger.WarningKind)
 		return BucketMetadata{}, errInvalidArgument
 	}
 	b := newBucketMetadata(name)
@@ -400,7 +400,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 	for legacyFile := range configs {
 		configFile := path.Join(bucketMetaPrefix, b.Name, legacyFile)
 		if err := deleteConfig(ctx, objectAPI, configFile); err != nil && !errors.Is(err, errConfigNotFound) {
-			logger.LogIf(ctx, err)
+			internalLogIf(ctx, err, logger.WarningKind)
 		}
 	}
 

cmd/bucket-object-lock.go

@@ -44,7 +44,6 @@ func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention,
 		if errors.Is(err, errInvalidArgument) {
 			return r, err
 		}
-		logger.CriticalIf(context.Background(), err)
 		return r, err
 	}
 	return config.ToRetention(), nil
@@ -66,7 +65,7 @@ func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locke
 	if ret.Mode.Valid() && (ret.Mode == objectlock.RetCompliance || ret.Mode == objectlock.RetGovernance) {
 		t, err := objectlock.UTCNowNTP()
 		if err != nil {
-			logger.LogIf(ctx, err)
+			internalLogIf(ctx, err, logger.WarningKind)
 			return true
 		}
 		if ret.RetainUntilDate.After(t) {
@@ -114,7 +113,7 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 			// duration of the retention period.
 			t, err := objectlock.UTCNowNTP()
 			if err != nil {
-				logger.LogIf(ctx, err)
+				internalLogIf(ctx, err, logger.WarningKind)
 				return ObjectLocked{}
 			}
 
@@ -140,7 +139,7 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 			if !byPassSet {
 				t, err := objectlock.UTCNowNTP()
 				if err != nil {
-					logger.LogIf(ctx, err)
+					internalLogIf(ctx, err, logger.WarningKind)
 					return ObjectLocked{}
 				}
 
@@ -170,7 +169,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
 
 	t, err := objectlock.UTCNowNTP()
 	if err != nil {
-		logger.LogIf(ctx, err)
+		internalLogIf(ctx, err, logger.WarningKind)
 		return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
 	}
 
@@ -277,7 +276,7 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
 			r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
 			t, err := objectlock.UTCNowNTP()
 			if err != nil {
-				logger.LogIf(ctx, err)
+				internalLogIf(ctx, err, logger.WarningKind)
 				return mode, retainDate, legalHold, ErrObjectLocked
 			}
 			if r.Mode == objectlock.RetCompliance && r.RetainUntilDate.After(t) {
@@ -324,7 +323,7 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
 
 		t, err := objectlock.UTCNowNTP()
 		if err != nil {
-			logger.LogIf(ctx, err)
+			internalLogIf(ctx, err, logger.WarningKind)
 			return mode, retainDate, legalHold, ErrObjectLocked
 		}
 

cmd/bucket-policy-handlers.go

@@ -113,7 +113,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Call site replication hook.
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypePolicy,
 		Bucket:    bucket,
 		Policy:    bucketPolicyBytes,
@@ -157,7 +157,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	}
 
 	// Call site replication hook.
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypePolicy,
 		Bucket:    bucket,
 		UpdatedAt: updatedAt,

cmd/bucket-policy.go

@@ -53,7 +53,7 @@ func (sys *PolicySys) IsAllowed(args policy.BucketPolicyArgs) bool {
 
 	// Log unhandled errors.
 	if _, ok := err.(BucketPolicyNotFound); !ok {
-		logger.LogIf(GlobalContext, err)
+		internalLogIf(GlobalContext, err, logger.WarningKind)
 	}
 
 	// As policy is not available for given bucket name, returns IsOwner i.e.

cmd/bucket-quota.go

@@ -64,9 +64,9 @@ func (sys *BucketQuotaSys) GetBucketUsageInfo(bucket string) (BucketUsageInfo, e
 	timedout := OperationTimedOut{}
 	if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &timedout) {
 		if len(dui.BucketsUsage) > 0 {
-			logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err), "bucket-usage-cache-"+bucket)
+			internalLogOnceIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err), "bucket-usage-cache-"+bucket)
 		} else {
-			logger.LogOnceIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"), "bucket-usage-empty-"+bucket)
+			internalLogOnceIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"), "bucket-usage-empty-"+bucket)
 		}
 	}
 
@@ -87,7 +87,7 @@ func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota,
 	}
 	if !quotaCfg.IsValid() {
 		if quotaCfg.Type == "fifo" {
-			logger.LogIf(GlobalContext, errors.New("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore. Please clear your quota configs using 'mc admin bucket quota alias/bucket --clear' and use 'mc ilm add' for expiration of objects"))
+			internalLogIf(GlobalContext, errors.New("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore. Please clear your quota configs using 'mc admin bucket quota alias/bucket --clear' and use 'mc ilm add' for expiration of objects"), logger.WarningKind)
 			return quotaCfg, fmt.Errorf("invalid quota type 'fifo'")
 		}
 		return quotaCfg, fmt.Errorf("Invalid quota config %#v", quotaCfg)

cmd/bucket-replication.go

@@ -81,13 +81,10 @@ const (
 // gets replication config associated to a given bucket name.
 func getReplicationConfig(ctx context.Context, bucketName string) (rc *replication.Config, err error) {
 	rCfg, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucketName)
-	if err != nil {
-		if errors.Is(err, BucketReplicationConfigNotFound{Bucket: bucketName}) || errors.Is(err, errInvalidArgument) {
-			return rCfg, err
-		}
-		logger.CriticalIf(ctx, err)
+	if err != nil && !errors.Is(err, BucketReplicationConfigNotFound{Bucket: bucketName}) {
+		return rCfg, err
 	}
-	return rCfg, err
+	return rCfg, nil
 }
 
 // validateReplicationDestination returns error if replication destination bucket missing or not configured
@@ -261,10 +258,16 @@ func mustReplicate(ctx context.Context, bucket, object string, mopts mustReplica
 	if mopts.replicationRequest { // incoming replication request on target cluster
 		return
 	}
+
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
+		replLogOnceIf(ctx, err, bucket)
 		return
 	}
+	if cfg == nil {
+		return
+	}
+
 	opts := replication.ObjectOpts{
 		Name:           object,
 		SSEC:           crypto.SSEC.IsEncrypted(mopts.meta),
@@ -312,6 +315,7 @@ var standardHeaders = []string{
 func hasReplicationRules(ctx context.Context, bucket string, objects []ObjectToDelete) bool {
 	c, err := getReplicationConfig(ctx, bucket)
 	if err != nil || c == nil {
+		replLogOnceIf(ctx, err, bucket)
 		return false
 	}
 	for _, obj := range objects {
@@ -331,6 +335,7 @@ func isStandardHeader(matchHeaderKey string) bool {
 func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelete, oi ObjectInfo, delOpts ObjectOptions, gerr error) (dsc ReplicateDecision) {
 	rcfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil || rcfg == nil {
+		replLogOnceIf(ctx, err, bucket)
 		return
 	}
 	// If incoming request is a replication request, it does not need to be re-replicated.
@@ -423,7 +428,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 
 	rcfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil || rcfg == nil {
-		logger.LogOnceIf(ctx, fmt.Errorf("unable to obtain replication config for bucket: %s: err: %s", bucket, err), bucket)
+		replLogOnceIf(ctx, fmt.Errorf("unable to obtain replication config for bucket: %s: err: %s", bucket, err), bucket)
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object: ObjectInfo{
@@ -440,7 +445,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	}
 	dsc, err := parseReplicateDecision(ctx, bucket, dobj.ReplicationState.ReplicateDecisionStr)
 	if err != nil {
-		logger.LogOnceIf(ctx, fmt.Errorf("unable to parse replication decision parameters for bucket: %s, err: %s, decision: %s",
+		replLogOnceIf(ctx, fmt.Errorf("unable to parse replication decision parameters for bucket: %s, err: %s, decision: %s",
 			bucket, err, dobj.ReplicationState.ReplicateDecisionStr), dobj.ReplicationState.ReplicateDecisionStr)
 		sendEvent(eventArgs{
 			BucketName: bucket,
@@ -494,7 +499,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		tgtClnt := globalBucketTargetSys.GetRemoteTargetClient(bucket, tgtEntry.Arn)
 		if tgtClnt == nil {
 			// Skip stale targets if any and log them to be missing at least once.
-			logger.LogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtEntry.Arn), tgtEntry.Arn)
+			replLogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtEntry.Arn), tgtEntry.Arn)
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
@@ -606,7 +611,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		return
 	}
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete-"+tgt.ARN)
+		replLogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete-"+tgt.ARN)
 		sendEvent(eventArgs{
 			BucketName: dobj.Bucket,
 			Object: ObjectInfo{
@@ -681,7 +686,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		} else {
 			rinfo.VersionPurgeStatus = Failed
 		}
-		logger.LogIf(ctx, fmt.Errorf("unable to replicate delete marker to %s: %s/%s(%s): %w", tgt.EndpointURL(), tgt.Bucket, dobj.ObjectName, versionID, rmErr))
+		replLogIf(ctx, fmt.Errorf("unable to replicate delete marker to %s: %s/%s(%s): %w", tgt.EndpointURL(), tgt.Bucket, dobj.ObjectName, versionID, rmErr))
 		if rmErr != nil && minio.IsNetworkOrHostDown(rmErr, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 			globalBucketTargetSys.markOffline(tgt.EndpointURL())
 		}
@@ -993,8 +998,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 	object := ri.Name
 
 	cfg, err := getReplicationConfig(ctx, bucket)
-	if err != nil {
-		logger.LogOnceIf(ctx, err, "get-replication-config-"+bucket)
+	if err != nil || cfg == nil {
+		replLogOnceIf(ctx, err, "get-replication-config-"+bucket)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1033,7 +1038,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 	for _, tgtArn := range tgtArns {
 		tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, tgtArn)
 		if tgt == nil {
-			logger.LogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtArn), tgtArn)
+			replLogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtArn), tgtArn)
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
@@ -1155,7 +1160,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	}
 
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s retry:%d", bucket, tgt.ARN, ri.RetryCount), "replication-target-offline"+tgt.ARN)
+		replLogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s retry:%d", bucket, tgt.ARN, ri.RetryCount), "replication-target-offline"+tgt.ARN)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1185,7 +1190,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 				UserAgent:  "Internal: [Replication]",
 				Host:       globalLocalNodeName,
 			})
-			logger.LogOnceIf(ctx, fmt.Errorf("unable to read source object %s/%s(%s): %w", bucket, object, objInfo.VersionID, err), object+":"+objInfo.VersionID)
+			replLogOnceIf(ctx, fmt.Errorf("unable to read source object %s/%s(%s): %w", bucket, object, objInfo.VersionID, err), object+":"+objInfo.VersionID)
 		}
 		return
 	}
@@ -1198,7 +1203,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 
 	size, err := objInfo.GetActualSize()
 	if err != nil {
-		logger.LogIf(ctx, err)
+		replLogIf(ctx, err)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1210,7 +1215,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	}
 
 	if tgt.Bucket == "" {
-		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty for target %s", objInfo.Name, objInfo.VersionID, tgt.EndpointURL()))
+		replLogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty for target %s", objInfo.Name, objInfo.VersionID, tgt.EndpointURL()))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1236,7 +1241,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 
 	putOpts, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err))
+		replLogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1267,23 +1272,19 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	}
 	r := bandwidth.NewMonitoredReader(newCtx, globalBucketMonitor, gr, opts)
 	if objInfo.isMultipart() {
-		if rinfo.Err = replicateObjectWithMultipart(ctx, c, tgt.Bucket, object,
-			r, objInfo, putOpts); rinfo.Err != nil {
-			if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
-				rinfo.ReplicationStatus = replication.Failed
-				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s (target: %s)", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err, tgt.EndpointURL()))
-			}
-		}
+		rinfo.Err = replicateObjectWithMultipart(ctx, c, tgt.Bucket, object, r, objInfo, putOpts)
 	} else {
-		if _, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); rinfo.Err != nil {
-			if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
-				rinfo.ReplicationStatus = replication.Failed
-				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s (target: %s)", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err, tgt.EndpointURL()))
-			}
-		}
+		_, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts)
 	}
-	if rinfo.Err != nil && minio.IsNetworkOrHostDown(rinfo.Err, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		globalBucketTargetSys.markOffline(tgt.EndpointURL())
+	if rinfo.Err != nil {
+		if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
+			rinfo.ReplicationStatus = replication.Failed
+			replLogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): to (target: %s): %w",
+				bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
+		}
+		if minio.IsNetworkOrHostDown(rinfo.Err, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
+			globalBucketTargetSys.markOffline(tgt.EndpointURL())
+		}
 	}
 	return
 }
@@ -1313,7 +1314,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	}
 
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s retry:%d", bucket, tgt.ARN, ri.RetryCount), "replication-target-offline-heal"+tgt.ARN)
+		replLogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s retry:%d", bucket, tgt.ARN, ri.RetryCount), "replication-target-offline-heal"+tgt.ARN)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1344,7 +1345,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				UserAgent:  "Internal: [Replication]",
 				Host:       globalLocalNodeName,
 			})
-			logger.LogIf(ctx, fmt.Errorf("unable to replicate to target %s for %s/%s(%s): %w", tgt.EndpointURL(), bucket, object, objInfo.VersionID, err))
+			replLogIf(ctx, fmt.Errorf("unable to replicate to target %s for %s/%s(%s): %w", tgt.EndpointURL(), bucket, object, objInfo.VersionID, err))
 		}
 		return
 	}
@@ -1364,7 +1365,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 
 	size, err := objInfo.GetActualSize()
 	if err != nil {
-		logger.LogIf(ctx, err)
+		replLogIf(ctx, err)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1381,7 +1382,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	}
 
 	if tgt.Bucket == "" {
-		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s) to %s, target bucket is missing", objInfo.Name, objInfo.VersionID, tgt.EndpointURL()))
+		replLogIf(ctx, fmt.Errorf("unable to replicate object %s(%s) to %s, target bucket is missing", objInfo.Name, objInfo.VersionID, tgt.EndpointURL()))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1411,7 +1412,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		if rAction == replicateNone {
 			if ri.OpType == replication.ExistingObjectReplicationType &&
 				objInfo.ModTime.Unix() > oi.LastModified.Unix() && objInfo.VersionID == nullVersionID {
-				logger.LogIf(ctx, fmt.Errorf("unable to replicate %s/%s (null). Newer version exists on target %s", bucket, object, tgt.EndpointURL()))
+				replLogIf(ctx, fmt.Errorf("unable to replicate %s/%s (null). Newer version exists on target %s", bucket, object, tgt.EndpointURL()))
 				sendEvent(eventArgs{
 					EventName:  event.ObjectReplicationNotTracked,
 					BucketName: bucket,
@@ -1451,7 +1452,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			rAction = replicateAll
 		default:
 			rinfo.Err = cerr
-			logger.LogIf(ctx, fmt.Errorf("unable to replicate %s/%s (%s). Target (%s) returned %s error on HEAD",
+			replLogIf(ctx, fmt.Errorf("unable to replicate %s/%s (%s). Target (%s) returned %s error on HEAD",
 				bucket, object, objInfo.VersionID, tgt.EndpointURL(), cerr))
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
@@ -1501,13 +1502,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		}
 		if _, rinfo.Err = c.CopyObject(ctx, tgt.Bucket, object, tgt.Bucket, object, getCopyObjMetadata(objInfo, tgt.StorageClass), srcOpts, dstOpts); rinfo.Err != nil {
 			rinfo.ReplicationStatus = replication.Failed
-			logger.LogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
+			replLogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
 		}
 	} else {
 		var putOpts minio.PutObjectOptions
 		putOpts, err = putReplicationOpts(ctx, tgt.StorageClass, objInfo)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
+			replLogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
@@ -1537,24 +1538,14 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		}
 		r := bandwidth.NewMonitoredReader(newCtx, globalBucketMonitor, gr, opts)
 		if objInfo.isMultipart() {
-			if rinfo.Err = replicateObjectWithMultipart(ctx, c, tgt.Bucket, object,
-				r, objInfo, putOpts); rinfo.Err != nil {
-				if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
-					rinfo.ReplicationStatus = replication.Failed
-					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
-				} else {
-					rinfo.ReplicationStatus = replication.Completed
-				}
-			}
+			rinfo.Err = replicateObjectWithMultipart(ctx, c, tgt.Bucket, object, r, objInfo, putOpts)
 		} else {
-			if _, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); rinfo.Err != nil {
-				if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
-					rinfo.ReplicationStatus = replication.Failed
-					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
-				} else {
-					rinfo.ReplicationStatus = replication.Completed
-				}
-			}
+			_, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts)
+		}
+		if rinfo.Err != nil && minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
+			rinfo.ReplicationStatus = replication.Failed
+			replLogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s) to target %s: %w",
+				bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
 		}
 		if rinfo.Err != nil && minio.IsNetworkOrHostDown(rinfo.Err, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 			globalBucketTargetSys.markOffline(tgt.EndpointURL())
@@ -1593,14 +1584,10 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 			for attempts <= 3 {
 				actx, acancel := context.WithTimeout(ctx, time.Minute)
 				aerr := c.AbortMultipartUpload(actx, bucket, object, uploadID)
+				acancel()
 				if aerr == nil {
-					acancel()
 					return
 				}
-				acancel()
-				logger.LogIf(actx,
-					fmt.Errorf("trying %s: Unable to cleanup failed multipart replication %s on remote %s/%s: %w - this may consume space on remote cluster",
-						humanize.Ordinal(attempts), uploadID, bucket, object, aerr))
 				attempts++
 				time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
 			}
@@ -1648,6 +1635,8 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 			ETag:       pInfo.ETag,
 		})
 	}
+
+	// really big value but its okay on heavily loaded systems. This is just tail end timeout.
 	cctx, ccancel := context.WithTimeout(ctx, 10*time.Minute)
 	defer ccancel()
 	_, err = c.CompleteMultipartUpload(cctx, bucket, object, uploadID, uploadedParts, minio.PutObjectOptions{
@@ -1866,7 +1855,7 @@ func (p *ReplicationPool) AddMRFWorker() {
 				globalReplicationStats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 
 			default:
-				logger.LogOnceIf(p.ctx, fmt.Errorf("unknown mrf replication type: %T", oi), "unknown-mrf-replicate-type")
+				bugLogIf(p.ctx, fmt.Errorf("unknown mrf replication type: %T", oi), "unknown-mrf-replicate-type")
 			}
 		case <-p.mrfWorkerKillCh:
 			return
@@ -1910,7 +1899,7 @@ func (p *ReplicationPool) AddWorker(input <-chan ReplicationWorkerOperation, opT
 					atomic.AddInt32(opTracker, -1)
 				}
 			default:
-				logger.LogOnceIf(p.ctx, fmt.Errorf("unknown replication type: %T", oi), "unknown-replicate-type")
+				bugLogIf(p.ctx, fmt.Errorf("unknown replication type: %T", oi), "unknown-replicate-type")
 			}
 		}
 	}
@@ -1949,7 +1938,7 @@ func (p *ReplicationPool) AddLargeWorker(input <-chan ReplicationWorkerOperation
 			case DeletedObjectReplicationInfo:
 				replicateDelete(p.ctx, v, p.objLayer)
 			default:
-				logger.LogOnceIf(p.ctx, fmt.Errorf("unknown replication type: %T", oi), "unknown-replicate-type")
+				bugLogIf(p.ctx, fmt.Errorf("unknown replication type: %T", oi), "unknown-replicate-type")
 			}
 		}
 	}
@@ -2096,9 +2085,9 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		p.mu.RUnlock()
 		switch prio {
 		case "fast":
-			logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic"), string(replicationSubsystem))
+			replLogOnceIf(GlobalContext, fmt.Errorf("Unable to keep up with incoming traffic"), string(replicationSubsystem), logger.WarningKind)
 		case "slow":
-			logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic - we recommend increasing replication priority with `mc admin config set api replication_priority=auto`"), string(replicationSubsystem))
+			replLogOnceIf(GlobalContext, fmt.Errorf("Unable to keep up with incoming traffic - we recommend increasing replication priority with `mc admin config set api replication_priority=auto`"), string(replicationSubsystem), logger.WarningKind)
 		default:
 			maxWorkers = min(maxWorkers, WorkerMaxLimit)
 			if p.ActiveWorkers() < maxWorkers {
@@ -2153,9 +2142,9 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 		p.mu.RUnlock()
 		switch prio {
 		case "fast":
-			logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming deletes"), string(replicationSubsystem))
+			replLogOnceIf(GlobalContext, fmt.Errorf("Unable to keep up with incoming deletes"), string(replicationSubsystem), logger.WarningKind)
 		case "slow":
-			logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming deletes - we recommend increasing replication priority with `mc admin config set api replication_priority=auto`"), string(replicationSubsystem))
+			replLogOnceIf(GlobalContext, fmt.Errorf("Unable to keep up with incoming deletes - we recommend increasing replication priority with `mc admin config set api replication_priority=auto`"), string(replicationSubsystem), logger.WarningKind)
 		default:
 			maxWorkers = min(maxWorkers, WorkerMaxLimit)
 			if p.ActiveWorkers() < maxWorkers {
@@ -2247,6 +2236,8 @@ func getProxyTargets(ctx context.Context, bucket, object string, opts ObjectOpti
 	}
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil || cfg == nil {
+		replLogOnceIf(ctx, err, bucket)
+
 		return &madmin.BucketTargets{}
 	}
 	topts := replication.ObjectOpts{Name: object}
@@ -2288,7 +2279,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
 		if rs != nil {
 			h, err := rs.ToHeader()
 			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("invalid range header for %s/%s(%s) - %w", bucket, object, opts.VersionID, err))
+				replLogIf(ctx, fmt.Errorf("invalid range header for %s/%s(%s) - %w", bucket, object, opts.VersionID, err))
 				continue
 			}
 			gopts.Set(xhttp.Range, h)
@@ -2656,7 +2647,7 @@ func (s *replicationResyncer) PersistToDisk(ctx context.Context, objectAPI Objec
 				}
 				if updt {
 					if err := saveResyncStatus(ctx, bucket, brs, objectAPI); err != nil {
-						logger.LogIf(ctx, fmt.Errorf("could not save resync metadata to drive for %s - %w", bucket, err))
+						replLogIf(ctx, fmt.Errorf("could not save resync metadata to drive for %s - %w", bucket, err))
 					} else {
 						lastResyncStatusSave[bucket] = brs.LastUpdate
 					}
@@ -2744,12 +2735,12 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	objInfoCh := make(chan ObjectInfo)
 	cfg, err := getReplicationConfig(ctx, opts.bucket)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("replication resync of %s for arn %s failed with %w", opts.bucket, opts.arn, err))
+		replLogIf(ctx, fmt.Errorf("replication resync of %s for arn %s failed with %w", opts.bucket, opts.arn, err))
 		return
 	}
 	tgts, err := globalBucketTargetSys.ListBucketTargets(ctx, opts.bucket)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("replication resync of %s for arn %s failed  %w", opts.bucket, opts.arn, err))
+		replLogIf(ctx, fmt.Errorf("replication resync of %s for arn %s failed  %w", opts.bucket, opts.arn, err))
 		return
 	}
 	rcfg := replicationConfig{
@@ -2762,12 +2753,12 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 			TargetArn: opts.arn,
 		})
 	if len(tgtArns) != 1 {
-		logger.LogIf(ctx, fmt.Errorf("replication resync failed for %s - arn specified %s is missing in the replication config", opts.bucket, opts.arn))
+		replLogIf(ctx, fmt.Errorf("replication resync failed for %s - arn specified %s is missing in the replication config", opts.bucket, opts.arn))
 		return
 	}
 	tgt := globalBucketTargetSys.GetRemoteTargetClient(opts.bucket, opts.arn)
 	if tgt == nil {
-		logger.LogIf(ctx, fmt.Errorf("replication resync failed for %s - target could not be created for arn %s", opts.bucket, opts.arn))
+		replLogIf(ctx, fmt.Errorf("replication resync failed for %s - target could not be created for arn %s", opts.bucket, opts.arn))
 		return
 	}
 	// mark resync status as resync started
@@ -2778,7 +2769,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	// Walk through all object versions - Walk() is always in ascending order needed to ensure
 	// delete marker replicated to target after object version is first created.
 	if err := objectAPI.Walk(ctx, opts.bucket, "", objInfoCh, WalkOptions{}); err != nil {
-		logger.LogIf(ctx, err)
+		replLogIf(ctx, err)
 		return
 	}
 
@@ -3053,7 +3044,7 @@ func (p *ReplicationPool) loadResync(ctx context.Context, buckets []BucketInfo,
 		meta, err := loadBucketResyncMetadata(ctx, bucket, objAPI)
 		if err != nil {
 			if !errors.Is(err, errVolumeNotFound) {
-				logger.LogIf(ctx, err)
+				replLogIf(ctx, err)
 			}
 			continue
 		}
@@ -3140,18 +3131,18 @@ func saveResyncStatus(ctx context.Context, bucket string, brs BucketReplicationR
 func getReplicationDiff(ctx context.Context, objAPI ObjectLayer, bucket string, opts madmin.ReplDiffOpts) (chan madmin.DiffInfo, error) {
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		replLogOnceIf(ctx, err, bucket)
 		return nil, err
 	}
 	tgts, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		replLogIf(ctx, err)
 		return nil, err
 	}
 
 	objInfoCh := make(chan ObjectInfo, 10)
 	if err := objAPI.Walk(ctx, bucket, opts.Prefix, objInfoCh, WalkOptions{}); err != nil {
-		logger.LogIf(ctx, err)
+		replLogIf(ctx, err)
 		return nil, err
 	}
 	rcfg := replicationConfig{
@@ -3233,7 +3224,11 @@ func QueueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, ret
 	if oi.ModTime.IsZero() {
 		return
 	}
-	rcfg, _ := getReplicationConfig(ctx, bucket)
+	rcfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil {
+		replLogOnceIf(ctx, err, bucket)
+		return
+	}
 	tgts, _ := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
 	queueReplicationHeal(ctx, bucket, oi, replicationConfig{
 		Config:  rcfg,
@@ -3535,7 +3530,7 @@ func (p *ReplicationPool) processMRF() {
 				continue
 			}
 			if err := p.queueMRFHeal(); err != nil && !osIsNotExist(err) {
-				logger.LogIf(p.ctx, err)
+				replLogIf(p.ctx, err)
 			}
 			pTimer.Reset(mrfQueueInterval)
 		case <-p.ctx.Done():

cmd/bucket-targets.go

@@ -20,7 +20,6 @@ package cmd
 import (
 	"context"
 	"errors"
-	"fmt"
 	"net/url"
 	"sync"
 	"time"
@@ -32,7 +31,6 @@ import (
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
 )
 
 const (
@@ -131,7 +129,7 @@ func (sys *BucketTargetSys) initHC(ep *url.URL) {
 func newHCClient() *madmin.AnonymousClient {
 	clnt, e := madmin.NewAnonymousClientNoEndpoint()
 	if e != nil {
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to initialize health check client"), string(replicationSubsystem))
+		bugLogIf(GlobalContext, errors.New("Unable to initialize health check client"))
 		return nil
 	}
 	clnt.SetCustomTransport(globalRemoteTargetTransport)
@@ -430,7 +428,7 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 	if arn.Type == madmin.ReplicationService {
 		// reject removal of remote target if replication configuration is present
 		rcfg, err := getReplicationConfig(ctx, bucket)
-		if err == nil {
+		if err == nil && rcfg != nil {
 			for _, tgtArn := range rcfg.FilterTargetArns(replication.ObjectOpts{OpType: replication.AllReplicationType}) {
 				if err == nil && (tgtArn == arnStr || rcfg.RoleArn == arnStr) {
 					sys.RLock()
@@ -624,7 +622,7 @@ func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
 	for _, tgt := range cfg.Targets {
 		tgtClient, err := sys.getRemoteTargetClient(&tgt)
 		if err != nil {
-			logger.LogIf(GlobalContext, err)
+			replLogIf(GlobalContext, err)
 			continue
 		}
 		sys.arnRemotesMap[tgt.Arn] = arnTarget{Client: tgtClient}

cmd/bucket-versioning-handler.go

@@ -82,7 +82,7 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		}, r.URL)
 		return
 	}
-	if _, err := getReplicationConfig(ctx, bucket); err == nil && v.Suspended() {
+	if rc, _ := getReplicationConfig(ctx, bucket); rc != nil && v.Suspended() {
 		writeErrorResponse(ctx, w, APIError{
 			Code:           "InvalidBucketState",
 			Description:    "A replication configuration is present on this bucket, bucket wide versioning cannot be suspended.",
@@ -108,7 +108,7 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:       madmin.SRBucketMetaTypeVersionConfig,
 		Bucket:     bucket,
 		Versioning: &cfgStr,

cmd/callhome.go

@@ -29,7 +29,6 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go/v3"
-	"github.com/minio/minio/internal/logger"
 )
 
 var callhomeLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
@@ -112,7 +111,7 @@ func performCallhome(ctx context.Context) {
 	deadline := 10 * time.Second // Default deadline is 10secs for callhome
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
-		logger.LogIf(ctx, errors.New("Callhome: object layer not ready"))
+		internalLogIf(ctx, errors.New("Callhome: object layer not ready"))
 		return
 	}
 
@@ -145,7 +144,7 @@ func performCallhome(ctx context.Context) {
 				// Received all data. Send to SUBNET and return
 				err := sendHealthInfo(ctx, healthInfo)
 				if err != nil {
-					logger.LogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err))
+					internalLogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err))
 				}
 				return
 			}
@@ -180,12 +179,12 @@ func createHealthJSONGzip(ctx context.Context, healthInfo madmin.HealthInfo) []b
 
 	enc := json.NewEncoder(gzWriter)
 	if e := enc.Encode(header); e != nil {
-		logger.LogIf(ctx, fmt.Errorf("Could not encode health info header: %w", e))
+		internalLogIf(ctx, fmt.Errorf("Could not encode health info header: %w", e))
 		return nil
 	}
 
 	if e := enc.Encode(healthInfo); e != nil {
-		logger.LogIf(ctx, fmt.Errorf("Could not encode health info: %w", e))
+		internalLogIf(ctx, fmt.Errorf("Could not encode health info: %w", e))
 		return nil
 	}
 

cmd/common-main.go

@@ -401,6 +401,8 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 	ctxt.ConnWriteDeadline = ctx.Duration("conn-write-deadline")
 	ctxt.ConnClientReadDeadline = ctx.Duration("conn-client-read-deadline")
 	ctxt.ConnClientWriteDeadline = ctx.Duration("conn-client-write-deadline")
+	ctxt.SendBufSize = ctx.Int("send-buf-size")
+	ctxt.RecvBufSize = ctx.Int("recv-buf-size")
 
 	ctxt.ShutdownTimeout = ctx.Duration("shutdown-timeout")
 	ctxt.IdleTimeout = ctx.Duration("idle-timeout")
@@ -958,16 +960,18 @@ func handleKMSConfig() {
 			}
 		}
 
-		KMS, err := kms.NewWithConfig(kmsConf)
+		KMS, err := kms.NewWithConfig(kmsConf, KMSLogger{})
 		if err != nil {
 			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
 		}
-		// We check that the default key ID exists or try to create it otherwise.
-		// This implicitly checks that we can communicate to KES. We don't treat
-		// a policy error as failure condition since MinIO may not have the permission
+		// Try to generate a data encryption key. Only try to create key if this fails.
+		// This implicitly checks that we can communicate to KES.
+		// We don't treat a policy error as failure condition since MinIO may not have the permission
 		// to create keys - just to generate/decrypt data encryption keys.
-		if err = KMS.CreateKey(context.Background(), env.Get(kms.EnvKESKeyName, "")); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
-			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
+		if _, err = KMS.GenerateKey(GlobalContext, env.Get(kms.EnvKESKeyName, ""), kms.Context{}); err != nil && errors.Is(err, kes.ErrKeyNotFound) {
+			if err = KMS.CreateKey(GlobalContext, env.Get(kms.EnvKESKeyName, "")); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
+				logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
+			}
 		}
 		GlobalKMS = KMS
 	}
@@ -1044,7 +1048,7 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 		}
 		if err = manager.AddCertificate(certFile, keyFile); err != nil {
 			err = fmt.Errorf("Unable to load TLS certificate '%s,%s': %w", certFile, keyFile, err)
-			logger.LogIf(GlobalContext, err, logger.ErrorKind)
+			bootLogIf(GlobalContext, err, logger.ErrorKind)
 		}
 	}
 	secureConn = true

cmd/config-current.go

@@ -479,7 +479,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 	dnsURL, dnsUser, dnsPass, err := env.LookupEnv(config.EnvDNSWebhook)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
+		configLogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
 	}
 	if err == nil && dnsURL != "" {
 		bootstrapTraceMsg("initialize remote bucket DNS store")
@@ -487,27 +487,27 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 			dns.Authentication(dnsUser, dnsPass),
 			dns.RootCAs(globalRootCAs))
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
 		}
 	}
 
 	etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+		configLogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 	}
 
 	if etcdCfg.Enabled {
 		bootstrapTraceMsg("initialize etcd store")
 		globalEtcdClient, err = etcd.New(etcdCfg)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 		}
 
 		if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
 			if globalDNSConfig != nil {
 				// if global DNS is already configured, indicate with a warning, in case
 				// users are confused.
-				logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, etcd is not used for DNS store", globalDNSConfig))
+				configLogIf(ctx, fmt.Errorf("DNS store is already configured with %s, etcd is not used for DNS store", globalDNSConfig))
 			} else {
 				globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
 					dns.DomainNames(globalDomainNames),
@@ -516,7 +516,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 					dns.CoreDNSPath(etcdCfg.CoreDNSPath),
 				)
 				if err != nil {
-					logger.LogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
+					configLogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w",
 						globalDomainNames, err))
 				}
 			}
@@ -532,7 +532,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 	globalSite, err = config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default])
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err))
+		configLogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err))
 	}
 
 	globalAutoEncryption = crypto.LookupAutoEncryption() // Enable auto-encryption if enabled
@@ -545,19 +545,19 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	bootstrapTraceMsg("initialize the event notification targets")
 	globalNotifyTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
+		configLogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
 
 	bootstrapTraceMsg("initialize the lambda targets")
 	globalLambdaTargetList, err = lambda.FetchEnabledTargets(GlobalContext, s, transport)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize lambda target(s): %w", err))
+		configLogIf(ctx, fmt.Errorf("Unable to initialize lambda target(s): %w", err))
 	}
 
 	bootstrapTraceMsg("applying the dynamic configuration")
 	// Apply dynamic config values
 	if err := applyDynamicConfig(ctx, objAPI, s); err != nil {
-		logger.LogIf(ctx, err)
+		configLogIf(ctx, err)
 	}
 }
 
@@ -566,21 +566,18 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		return errServerNotInitialized
 	}
 
+	var errs []error
 	setDriveCounts := objAPI.SetDriveCounts()
 	switch subSys {
 	case config.APISubSys:
 		apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
+			configLogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
 		}
 
 		globalAPIConfig.init(apiConfig, setDriveCounts)
 		autoGenerateRootCredentials() // Generate the KMS root credentials here since we don't know whether API root access is disabled until now.
-
-		// Initialize remote instance transport once.
-		getRemoteInstanceTransportOnce.Do(func() {
-			getRemoteInstanceTransport = NewHTTPTransportWithTimeout(apiConfig.RemoteTransportDeadline)
-		})
+		setRemoteInstanceTransport(NewHTTPTransportWithTimeout(apiConfig.RemoteTransportDeadline))
 	case config.CompressionSubSys:
 		cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
 		if err != nil {
@@ -592,52 +589,55 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 	case config.HealSubSys:
 		healCfg, err := heal.LookupConfig(s[config.HealSubSys][config.Default])
 		if err != nil {
-			return fmt.Errorf("Unable to apply heal config: %w", err)
+			errs = append(errs, fmt.Errorf("Unable to apply heal config: %w", err))
+		} else {
+			globalHealConfig.Update(healCfg)
 		}
-		globalHealConfig.Update(healCfg)
 	case config.BatchSubSys:
 		batchCfg, err := batch.LookupConfig(s[config.BatchSubSys][config.Default])
 		if err != nil {
-			return fmt.Errorf("Unable to apply batch config: %w", err)
+			errs = append(errs, fmt.Errorf("Unable to apply batch config: %w", err))
+		} else {
+			globalBatchConfig.Update(batchCfg)
 		}
-		globalBatchConfig.Update(batchCfg)
 	case config.ScannerSubSys:
 		scannerCfg, err := scanner.LookupConfig(s[config.ScannerSubSys][config.Default])
 		if err != nil {
-			return fmt.Errorf("Unable to apply scanner config: %w", err)
+			errs = append(errs, fmt.Errorf("Unable to apply scanner config: %w", err))
+		} else {
+			// update dynamic scanner values.
+			scannerIdleMode.Store(scannerCfg.IdleMode)
+			scannerCycle.Store(scannerCfg.Cycle)
+			scannerExcessObjectVersions.Store(scannerCfg.ExcessVersions)
+			scannerExcessFolders.Store(scannerCfg.ExcessFolders)
+			configLogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait))
 		}
-		// update dynamic scanner values.
-		scannerIdleMode.Store(scannerCfg.IdleMode)
-		scannerCycle.Store(scannerCfg.Cycle)
-		scannerExcessObjectVersions.Store(scannerCfg.ExcessVersions)
-		scannerExcessFolders.Store(scannerCfg.ExcessFolders)
-		logger.LogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait))
 	case config.LoggerWebhookSubSys:
 		loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.LoggerWebhookSubSys)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to load logger webhook config: %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to load logger webhook config: %w", err))
 		}
 		userAgent := getUserAgent(getMinioMode())
 		for n, l := range loggerCfg.HTTP {
 			if l.Enabled {
-				l.LogOnceIf = logger.LogOnceConsoleIf
+				l.LogOnceIf = configLogOnceConsoleIf
 				l.UserAgent = userAgent
 				l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			}
 			loggerCfg.HTTP[n] = l
 		}
 		if errs := logger.UpdateHTTPWebhooks(ctx, loggerCfg.HTTP); len(errs) > 0 {
-			logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %v", errs))
+			configLogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %v", errs))
 		}
 	case config.AuditWebhookSubSys:
 		loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.AuditWebhookSubSys)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to load audit webhook config: %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to load audit webhook config: %w", err))
 		}
 		userAgent := getUserAgent(getMinioMode())
 		for n, l := range loggerCfg.AuditWebhook {
 			if l.Enabled {
-				l.LogOnceIf = logger.LogOnceConsoleIf
+				l.LogOnceIf = configLogOnceConsoleIf
 				l.UserAgent = userAgent
 				l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			}
@@ -645,42 +645,40 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		}
 
 		if errs := logger.UpdateAuditWebhooks(ctx, loggerCfg.AuditWebhook); len(errs) > 0 {
-			logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %v", errs))
+			configLogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %v", errs))
 		}
 	case config.AuditKafkaSubSys:
 		loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.AuditKafkaSubSys)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to load audit kafka config: %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to load audit kafka config: %w", err))
 		}
 		for n, l := range loggerCfg.AuditKafka {
 			if l.Enabled {
 				if l.TLS.Enable {
 					l.TLS.RootCAs = globalRootCAs
 				}
-				l.LogOnce = logger.LogOnceIf
+				l.LogOnce = configLogOnceIf
 				loggerCfg.AuditKafka[n] = l
 			}
 		}
 		if errs := logger.UpdateAuditKafkaTargets(ctx, loggerCfg); len(errs) > 0 {
-			logger.LogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %v", errs))
+			configLogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %v", errs))
 		}
 	case config.StorageClassSubSys:
 		for i, setDriveCount := range setDriveCounts {
 			sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount)
 			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
+				configLogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err))
 				break
 			}
-			// if we validated all setDriveCounts and it was successful
-			// proceed to store the correct storage class globally.
-			if i == len(setDriveCounts)-1 {
+			if i == 0 {
 				globalStorageClass.Update(sc)
 			}
 		}
 	case config.SubnetSubSys:
-		subnetConfig, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
+		subnetConfig, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalRemoteTargetTransport)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
 		} else {
 			globalSubnetConfig.Update(subnetConfig, globalIsCICD)
 			globalSubnetConfig.ApplyEnv() // update environment settings for Console UI
@@ -688,7 +686,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 	case config.CallhomeSubSys:
 		callhomeCfg, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default])
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to load callhome config: %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to load callhome config: %w", err))
 		} else {
 			enable := callhomeCfg.Enable && !globalCallhomeConfig.Enabled()
 			globalCallhomeConfig.Update(callhomeCfg)
@@ -697,45 +695,50 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			}
 		}
 	case config.DriveSubSys:
-		if driveConfig, err := drive.LookupConfig(s[config.DriveSubSys][config.Default]); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to load drive config: %w", err))
+		driveConfig, err := drive.LookupConfig(s[config.DriveSubSys][config.Default])
+		if err != nil {
+			configLogIf(ctx, fmt.Errorf("Unable to load drive config: %w", err))
 		} else {
-			err := globalDriveConfig.Update(driveConfig)
-			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to update drive config: %v", err))
+			if err = globalDriveConfig.Update(driveConfig); err != nil {
+				configLogIf(ctx, fmt.Errorf("Unable to update drive config: %v", err))
 			}
 		}
 	case config.CacheSubSys:
 		cacheCfg, err := cache.LookupConfig(s[config.CacheSubSys][config.Default], globalRemoteTargetTransport)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to load cache config: %w", err))
+			configLogIf(ctx, fmt.Errorf("Unable to load cache config: %w", err))
 		} else {
 			globalCacheConfig.Update(cacheCfg)
 		}
 	case config.BrowserSubSys:
 		browserCfg, err := browser.LookupConfig(s[config.BrowserSubSys][config.Default])
 		if err != nil {
-			return fmt.Errorf("Unable to apply browser config: %w", err)
+			errs = append(errs, fmt.Errorf("Unable to apply browser config: %w", err))
+		} else {
+			globalBrowserConfig.Update(browserCfg)
 		}
-		globalBrowserConfig.Update(browserCfg)
 	case config.ILMSubSys:
 		ilmCfg, err := ilm.LookupConfig(s[config.ILMSubSys][config.Default])
 		if err != nil {
-			return fmt.Errorf("Unable to apply ilm config: %w", err)
+			errs = append(errs, fmt.Errorf("Unable to apply ilm config: %w", err))
+		} else {
+			if globalTransitionState != nil {
+				globalTransitionState.UpdateWorkers(ilmCfg.TransitionWorkers)
+			}
+			if globalExpiryState != nil {
+				globalExpiryState.ResizeWorkers(ilmCfg.ExpirationWorkers)
+			}
+			globalILMConfig.update(ilmCfg)
 		}
-		if globalTransitionState != nil {
-			globalTransitionState.UpdateWorkers(ilmCfg.TransitionWorkers)
-		}
-		if globalExpiryState != nil {
-			globalExpiryState.ResizeWorkers(ilmCfg.ExpirationWorkers)
-		}
-		globalILMConfig.update(ilmCfg)
 	}
 	globalServerConfigMu.Lock()
 	defer globalServerConfigMu.Unlock()
 	if globalServerConfig != nil {
 		globalServerConfig[subSys] = s[subSys]
 	}
+	if len(errs) > 0 {
+		return errors.Join(errs...)
+	}
 	return nil
 }
 
@@ -753,7 +756,7 @@ func autoGenerateRootCredentials() {
 	if manager, ok := GlobalKMS.(kms.KeyManager); ok {
 		stat, err := GlobalKMS.Stat(GlobalContext)
 		if err != nil {
-			logger.LogIf(GlobalContext, err, "Unable to generate root credentials using KMS")
+			kmsLogIf(GlobalContext, err, "Unable to generate root credentials using KMS")
 			return
 		}
 

cmd/data-scanner.go

@@ -36,12 +36,13 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/object/lock"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config/heal"
 	"github.com/minio/minio/internal/event"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 	uatomic "go.uber.org/atomic"
 )
@@ -63,11 +64,12 @@ var (
 	globalHealConfig heal.Config
 
 	// Sleeper values are updated when config is loaded.
-	scannerSleeper              = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults
-	scannerCycle                = uatomic.NewDuration(dataScannerStartDelay)
-	scannerIdleMode             = uatomic.NewInt32(0) // default is throttled when idle
-	scannerExcessObjectVersions = uatomic.NewInt64(100)
-	scannerExcessFolders        = uatomic.NewInt64(50000)
+	scannerSleeper                       = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults
+	scannerCycle                         = uatomic.NewDuration(dataScannerStartDelay)
+	scannerIdleMode                      = uatomic.NewInt32(0) // default is throttled when idle
+	scannerExcessObjectVersions          = uatomic.NewInt64(100)
+	scannerExcessObjectVersionsTotalSize = uatomic.NewInt64(1024 * 1024 * 1024 * 1024) // 1 TB
+	scannerExcessFolders                 = uatomic.NewInt64(50000)
 )
 
 // initDataScanner will start the scanner in the background.
@@ -122,13 +124,13 @@ func readBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer) backgroundH
 	buf, err := readConfig(ctx, objAPI, backgroundHealInfoPath)
 	if err != nil {
 		if !errors.Is(err, errConfigNotFound) {
-			logger.LogOnceIf(ctx, err, backgroundHealInfoPath)
+			internalLogOnceIf(ctx, err, backgroundHealInfoPath)
 		}
 		return backgroundHealInfo{}
 	}
 	var info backgroundHealInfo
 	if err = json.Unmarshal(buf, &info); err != nil {
-		logger.LogOnceIf(ctx, err, backgroundHealInfoPath)
+		bugLogIf(ctx, err, backgroundHealInfoPath)
 	}
 	return info
 }
@@ -140,13 +142,13 @@ func saveBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer, info backgr
 
 	b, err := json.Marshal(info)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		bugLogIf(ctx, err)
 		return
 	}
 	// Get last healing information
 	err = saveConfig(ctx, objAPI, backgroundHealInfoPath, b)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		internalLogIf(ctx, err)
 	}
 }
 
@@ -167,7 +169,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 		cycleInfo.next = binary.LittleEndian.Uint64(buf[:8])
 		buf = buf[8:]
 		_, err := cycleInfo.UnmarshalMsg(buf)
-		logger.LogIf(ctx, err)
+		bugLogIf(ctx, err)
 	}
 
 	scannerTimer := time.NewTimer(scannerCycle.Load())
@@ -204,7 +206,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
 			err := objAPI.NSScanner(ctx, results, uint32(cycleInfo.current), scanMode)
-			logger.LogOnceIf(ctx, err, "ns-scanner")
+			scannerLogIf(ctx, err)
 			res := map[string]string{"cycle": strconv.FormatUint(cycleInfo.current, 10)}
 			if err != nil {
 				res["error"] = err.Error()
@@ -224,7 +226,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 				binary.LittleEndian.PutUint64(tmp, cycleInfo.next)
 				tmp, _ = cycleInfo.MarshalMsg(tmp)
 				err = saveConfig(ctx, objAPI, dataUsageBloomNamePath, tmp)
-				logger.LogOnceIf(ctx, err, dataUsageBloomNamePath)
+				scannerLogIf(ctx, err, dataUsageBloomNamePath)
 			}
 		}
 	}
@@ -752,7 +754,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 							versionID: "",
 						}, madmin.HealItemObject)
 						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-							logger.LogOnceIf(ctx, err, entry.name)
+							scannerLogIf(ctx, err)
 						}
 						foundObjs = foundObjs || err == nil
 						return
@@ -769,7 +771,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						}, madmin.HealItemObject)
 						stopFn(int(ver.Size))
 						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-							logger.LogOnceIf(ctx, err, fiv.Name)
+							scannerLogIf(ctx, err, fiv.Name)
 						}
 						if err == nil {
 							successVersions++
@@ -945,17 +947,39 @@ func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi Object
 func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi ObjectInfo) (action lifecycle.Action, size int64) {
 	size, err := oi.GetActualSize()
 	if i.debug {
-		logger.LogIf(ctx, err)
+		scannerLogIf(ctx, err)
 	}
 	if i.lifeCycle == nil {
 		return action, size
 	}
 
 	versionID := oi.VersionID
-	vcfg, _ := globalBucketVersioningSys.Get(i.bucket)
-	rCfg, _ := globalBucketObjectLockSys.Get(i.bucket)
-	replcfg, _ := getReplicationConfig(ctx, i.bucket)
-	lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, replcfg, oi)
+
+	var vc *versioning.Versioning
+	var lr objectlock.Retention
+	var rcfg *replication.Config
+	if !isMinioMetaBucketName(i.bucket) {
+		vc, err = globalBucketVersioningSys.Get(i.bucket)
+		if err != nil {
+			scannerLogOnceIf(ctx, err, i.bucket)
+			return
+		}
+
+		// Check if bucket is object locked.
+		lr, err = globalBucketObjectLockSys.Get(i.bucket)
+		if err != nil {
+			scannerLogOnceIf(ctx, err, i.bucket)
+			return
+		}
+
+		rcfg, err = getReplicationConfig(ctx, i.bucket)
+		if err != nil {
+			scannerLogOnceIf(ctx, err, i.bucket)
+			return
+		}
+	}
+
+	lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, lr, rcfg, oi)
 	if i.debug {
 		if versionID != "" {
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, lcEvt.Action)
@@ -969,11 +993,11 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 	// This can happen when,
 	// - ExpireObjectAllVersions flag is enabled
 	// - NoncurrentVersionExpiration is applicable
-	case lifecycle.DeleteVersionAction, lifecycle.DeleteAllVersionsAction:
+	case lifecycle.DeleteVersionAction, lifecycle.DeleteAllVersionsAction, lifecycle.DelMarkerDeleteAllVersionsAction:
 		size = 0
 	case lifecycle.DeleteAction:
 		// On a non-versioned bucket, DeleteObject removes the only version permanently.
-		if !vcfg.PrefixEnabled(oi.Name) {
+		if !vc.PrefixEnabled(oi.Name) {
 			size = 0
 		}
 	}
@@ -1066,7 +1090,7 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 	}
 
 	// Check if we have many versions after applyNewerNoncurrentVersionLimit.
-	if len(objInfos) > int(scannerExcessObjectVersions.Load()) {
+	if len(objInfos) >= int(scannerExcessObjectVersions.Load()) {
 		// Notify object accessed via a GET request.
 		sendEvent(eventArgs{
 			EventName:  event.ObjectManyVersions,
@@ -1090,6 +1114,39 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 		})
 	}
 
+	cumulativeSize := int64(0)
+	for _, objInfo := range objInfos {
+		cumulativeSize += objInfo.Size
+	}
+	// Check if the cumulative size of all versions of this object is high.
+	if cumulativeSize >= scannerExcessObjectVersionsTotalSize.Load() {
+		// Notify object accessed via a GET request.
+		sendEvent(eventArgs{
+			EventName:  event.ObjectLargeVersions,
+			BucketName: i.bucket,
+			Object: ObjectInfo{
+				Name: i.objectPath(),
+			},
+			UserAgent: "Scanner",
+			Host:      globalLocalNodeName,
+			RespElements: map[string]string{
+				"x-minio-versions-count": strconv.Itoa(len(objInfos)),
+				"x-minio-versions-size":  strconv.FormatInt(cumulativeSize, 10),
+			},
+		})
+
+		auditLogInternal(context.Background(), AuditLogOptions{
+			Event:   "scanner:largeversions",
+			APIName: "Scanner",
+			Bucket:  i.bucket,
+			Object:  i.objectPath(),
+			Tags: map[string]interface{}{
+				"x-minio-versions-count": strconv.Itoa(len(objInfos)),
+				"x-minio-versions-size":  strconv.FormatInt(cumulativeSize, 10),
+			},
+		})
+	}
+
 	return objInfos, nil
 }
 
@@ -1105,7 +1162,7 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 
 	// Note: objDeleted is true if and only if action ==
 	// lifecycle.DeleteAllVersionsAction
-	if action == lifecycle.DeleteAllVersionsAction {
+	if action.DeleteAll() {
 		return true, 0
 	}
 
@@ -1123,7 +1180,7 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 				err := o.CheckAbandonedParts(ctx, i.bucket, i.objectPath(), madmin.HealOpts{Remove: healDeleteDangling})
 				done()
 				if err != nil {
-					logger.LogOnceIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err), i.objectPath())
+					healingLogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err), i.objectPath())
 				}
 			}
 		}
@@ -1199,7 +1256,7 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return false
 		}
-		logger.LogOnceIf(ctx, err, obj.Name)
+		ilmLogIf(ctx, err)
 		return false
 	}
 	// Notification already sent in *expireTransitionedObject*, just return 'true' here.
@@ -1235,7 +1292,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 
 		if lcEvent.Action != lifecycle.NoneAction {
 			numVersions := uint64(1)
-			if lcEvent.Action == lifecycle.DeleteAllVersionsAction {
+			if lcEvent.Action.DeleteAll() {
 				numVersions = uint64(obj.NumVersions)
 			}
 			globalScannerMetrics.timeILM(lcEvent.Action)(numVersions)
@@ -1248,7 +1305,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 			return false
 		}
 		// Assume it is still there.
-		logger.LogOnceIf(ctx, err, "non-transition-expiry")
+		ilmLogOnceIf(ctx, err, "non-transition-expiry")
 		return false
 	}
 	if dobj.Name == "" {
@@ -1263,8 +1320,11 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	if obj.DeleteMarker {
 		eventName = event.ObjectRemovedDeleteMarkerCreated
 	}
-	if lcEvent.Action.DeleteAll() {
+	switch lcEvent.Action {
+	case lifecycle.DeleteAllVersionsAction:
 		eventName = event.ObjectRemovedDeleteAllVersions
+	case lifecycle.DelMarkerDeleteAllVersionsAction:
+		eventName = event.ILMDelMarkerExpirationDelete
 	}
 	// Notify object deleted event.
 	sendEvent(eventArgs{
@@ -1289,7 +1349,7 @@ func applyLifecycleAction(event lifecycle.Event, src lcEventSrc, obj ObjectInfo)
 	switch action := event.Action; action {
 	case lifecycle.DeleteVersionAction, lifecycle.DeleteAction,
 		lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction,
-		lifecycle.DeleteAllVersionsAction:
+		lifecycle.DeleteAllVersionsAction, lifecycle.DelMarkerDeleteAllVersionsAction:
 		success = applyExpiryRule(event, src, obj)
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
 		success = applyTransitionRule(event, src, obj)
@@ -1389,48 +1449,7 @@ func (d *dynamicSleeper) Timer(ctx context.Context) func() {
 	t := time.Now()
 	return func() {
 		doneAt := time.Now()
-		for {
-			// Grab current values
-			d.mu.RLock()
-			minWait, maxWait := d.minSleep, d.maxSleep
-			factor := d.factor
-			cycle := d.cycle
-			d.mu.RUnlock()
-			elapsed := doneAt.Sub(t)
-			// Don't sleep for really small amount of time
-			wantSleep := time.Duration(float64(elapsed) * factor)
-			if wantSleep <= minWait {
-				return
-			}
-			if maxWait > 0 && wantSleep > maxWait {
-				wantSleep = maxWait
-			}
-			timer := time.NewTimer(wantSleep)
-			select {
-			case <-ctx.Done():
-				if !timer.Stop() {
-					<-timer.C
-				}
-				if d.isScanner {
-					globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
-				}
-				return
-			case <-timer.C:
-				if d.isScanner {
-					globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
-				}
-				return
-			case <-cycle:
-				if !timer.Stop() {
-					// We expired.
-					<-timer.C
-					if d.isScanner {
-						globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
-					}
-					return
-				}
-			}
-		}
+		d.Sleep(ctx, doneAt.Sub(t))
 	}
 }
 

cmd/data-usage-cache.go

@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -36,8 +35,6 @@ import (
 	"github.com/klauspost/compress/zstd"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/lifecycle"
-	"github.com/minio/minio/internal/hash"
-	"github.com/minio/minio/internal/logger"
 	"github.com/tinylib/msgp/msgp"
 	"github.com/valyala/bytebufferpool"
 )
@@ -635,7 +632,7 @@ func (d *dataUsageCache) copyWithChildren(src *dataUsageCache, hash dataUsageHas
 	d.Cache[hash.Key()] = e
 	for ch := range e.Children {
 		if ch == hash.Key() {
-			logger.LogIf(GlobalContext, errors.New("dataUsageCache.copyWithChildren: Circular reference"))
+			scannerLogIf(GlobalContext, errors.New("dataUsageCache.copyWithChildren: Circular reference"))
 			return
 		}
 		d.copyWithChildren(src, dataUsageHash(ch), &hash)
@@ -1006,11 +1003,23 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string)
 		ctx, cancel := context.WithTimeout(ctx, timeout)
 		defer cancel()
 
-		r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, ObjectOptions{NoLock: true})
+		r, err := store.GetObjectNInfo(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, name), nil, http.Header{}, ObjectOptions{NoLock: true})
 		if err != nil {
 			switch err.(type) {
 			case ObjectNotFound, BucketNotFound:
-				return false, nil
+				r, err = store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, ObjectOptions{NoLock: true})
+				if err != nil {
+					switch err.(type) {
+					case ObjectNotFound, BucketNotFound:
+						return false, nil
+					case InsufficientReadQuorum, StorageErr:
+						return true, nil
+					}
+					return false, err
+				}
+				err = d.deserialize(r)
+				r.Close()
+				return err != nil, nil
 			case InsufficientReadQuorum, StorageErr:
 				return true, nil
 			}
@@ -1041,7 +1050,7 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string)
 	}
 
 	if retries == 5 {
-		logger.LogOnceIf(ctx, fmt.Errorf("maximum retry reached to load the data usage cache `%s`", name), "retry-loading-data-usage-cache")
+		scannerLogOnceIf(ctx, fmt.Errorf("maximum retry reached to load the data usage cache `%s`", name), "retry-loading-data-usage-cache")
 	}
 
 	return nil
@@ -1071,24 +1080,11 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 	}
 
 	save := func(name string, timeout time.Duration) error {
-		hr, err := hash.NewReader(ctx, bytes.NewReader(buf.Bytes()), int64(buf.Len()), "", "", int64(buf.Len()))
-		if err != nil {
-			return err
-		}
-
 		// Abandon if more than a minute, so we don't hold up scanner.
 		ctx, cancel := context.WithTimeout(ctx, timeout)
 		defer cancel()
 
-		_, err = store.PutObject(ctx,
-			dataUsageBucket,
-			name,
-			NewPutObjReader(hr),
-			ObjectOptions{NoLock: true})
-		if isErrBucketNotFound(err) {
-			return nil
-		}
-		return err
+		return saveConfig(ctx, store, pathJoin(bucketMetaPrefix, name), buf.Bytes())
 	}
 	defer save(name+".bkp", 5*time.Second) // Keep a backup as well
 

cmd/data-usage.go

@@ -25,7 +25,6 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/internal/cachevalue"
-	"github.com/minio/minio/internal/logger"
 )
 
 const (
@@ -49,7 +48,7 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
 		json := jsoniter.ConfigCompatibleWithStandardLibrary
 		dataUsageJSON, err := json.Marshal(dataUsageInfo)
 		if err != nil {
-			logger.LogIf(ctx, err)
+			scannerLogIf(ctx, err)
 			continue
 		}
 		if attempts > 10 {
@@ -57,7 +56,7 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
 			attempts = 1
 		}
 		if err = saveConfig(ctx, objAPI, dataUsageObjNamePath, dataUsageJSON); err != nil {
-			logger.LogOnceIf(ctx, err, dataUsageObjNamePath)
+			scannerLogOnceIf(ctx, err, dataUsageObjNamePath)
 		}
 		attempts++
 	}

cmd/encryption-v1.go

@@ -1089,7 +1089,7 @@ func (o *ObjectInfo) decryptPartsChecksums() {
 	if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted {
 		decrypted, err := o.metadataDecrypter()("object-checksum", data)
 		if err != nil {
-			logger.LogIf(GlobalContext, err)
+			encLogIf(GlobalContext, err)
 			return
 		}
 		data = decrypted
@@ -1151,7 +1151,7 @@ func (o *ObjectInfo) decryptChecksums(part int) map[string]string {
 	if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted {
 		decrypted, err := o.metadataDecrypter()("object-checksum", data)
 		if err != nil {
-			logger.LogIf(GlobalContext, err)
+			encLogIf(GlobalContext, err)
 			return nil
 		}
 		data = decrypted

cmd/endpoint-ellipses.go

@@ -477,7 +477,7 @@ func mergeDisksLayoutFromArgs(args []string, ctxt *serverCtxt) (err error) {
 		}
 		ctxt.Layout = disksLayout{
 			legacy: true,
-			pools:  []poolDisksLayout{{layout: setArgs}},
+			pools:  []poolDisksLayout{{layout: setArgs, cmdline: strings.Join(args, " ")}},
 		}
 		return
 	}

cmd/endpoint.go

@@ -514,7 +514,7 @@ func (l EndpointServerPools) hostsSorted() []*xnet.Host {
 		}
 		host, err := xnet.ParseHost(hostStr)
 		if err != nil {
-			logger.LogIf(GlobalContext, err)
+			internalLogIf(GlobalContext, err)
 			continue
 		}
 		hosts[i] = host
@@ -645,7 +645,7 @@ func (endpoints Endpoints) UpdateIsLocal() error {
 							))
 						ctx := logger.SetReqInfo(GlobalContext,
 							reqInfo)
-						logger.LogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
+						bootLogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
 							endpoints[i].Hostname()), endpoints[i].Hostname(), logger.ErrorKind)
 					}
 
@@ -675,7 +675,7 @@ func (endpoints Endpoints) UpdateIsLocal() error {
 							))
 						ctx := logger.SetReqInfo(GlobalContext,
 							reqInfo)
-						logger.LogOnceIf(ctx, err, endpoints[i].Hostname(), logger.ErrorKind)
+						bootLogOnceIf(ctx, err, endpoints[i].Hostname(), logger.ErrorKind)
 					}
 				} else {
 					resolvedList[i] = true
@@ -807,7 +807,7 @@ func (p PoolEndpointList) UpdateIsLocal() error {
 						continue
 					}
 
-					if endpoint.Host == "" {
+					if endpoint.Host == "" || (orchestrated && env.Get("_MINIO_SERVER_LOCAL", "") == endpoint.Host) {
 						if !foundLocal {
 							foundLocal = true
 						}
@@ -837,7 +837,7 @@ func (p PoolEndpointList) UpdateIsLocal() error {
 								))
 							ctx := logger.SetReqInfo(GlobalContext,
 								reqInfo)
-							logger.LogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
+							bootLogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
 								endpoint.Hostname()), endpoint.Hostname(), logger.ErrorKind)
 						}
 						continue
@@ -866,7 +866,7 @@ func (p PoolEndpointList) UpdateIsLocal() error {
 								))
 							ctx := logger.SetReqInfo(GlobalContext,
 								reqInfo)
-							logger.LogOnceIf(ctx, fmt.Errorf("Unable to resolve DNS for %s: %w", endpoint, err), endpoint.Hostname(), logger.ErrorKind)
+							bootLogOnceIf(ctx, fmt.Errorf("Unable to resolve DNS for %s: %w", endpoint, err), endpoint.Hostname(), logger.ErrorKind)
 						}
 					} else {
 						resolvedList[endpoint] = true
@@ -1205,7 +1205,7 @@ func GetProxyEndpoints(endpointServerPools EndpointServerPools) []ProxyEndpoint
 
 			proxyEps = append(proxyEps, ProxyEndpoint{
 				Endpoint:  endpoint,
-				Transport: globalProxyTransport,
+				Transport: globalRemoteTargetTransport,
 			})
 		}
 	}

cmd/erasure-common.go

@@ -25,7 +25,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/sync/errgroup"
 )
 
@@ -154,6 +153,8 @@ func readMultipleFiles(ctx context.Context, disks []StorageAPI, req ReadMultiple
 		errFileVersionNotFound,
 		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
 		io.EOF,              // some times we would read without locks, ignore these errors
+		context.DeadlineExceeded,
+		context.Canceled,
 	}
 	ignoredErrs = append(ignoredErrs, objectOpIgnoredErrs...)
 
@@ -163,7 +164,7 @@ func readMultipleFiles(ctx context.Context, disks []StorageAPI, req ReadMultiple
 			continue
 		}
 		if !IsErr(err, ignoredErrs...) {
-			logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
+			storageLogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
 				disks[index], req.Bucket, req.Prefix, err),
 				disks[index].String())
 		}

cmd/erasure-decode.go

@@ -38,6 +38,7 @@ type parallelReader struct {
 	shardFileSize int64
 	buf           [][]byte
 	readerToBuf   []int
+	stashBuffer   []byte
 }
 
 // newParallelReader returns parallelReader.
@@ -46,6 +47,21 @@ func newParallelReader(readers []io.ReaderAt, e Erasure, offset, totalLength int
 	for i := range r2b {
 		r2b[i] = i
 	}
+	bufs := make([][]byte, len(readers))
+	// Fill buffers
+	b := globalBytePoolCap.Load().Get()
+	shardSize := int(e.ShardSize())
+	if cap(b) < len(readers)*shardSize {
+		// We should always have enough capacity, but older objects may be bigger.
+		globalBytePoolCap.Load().Put(b)
+		b = nil
+	} else {
+		// Seed the buffers.
+		for i := range bufs {
+			bufs[i] = b[i*shardSize : (i+1)*shardSize]
+		}
+	}
+
 	return &parallelReader{
 		readers:       readers,
 		orgReaders:    readers,
@@ -55,6 +71,15 @@ func newParallelReader(readers []io.ReaderAt, e Erasure, offset, totalLength int
 		shardFileSize: e.ShardFileSize(totalLength),
 		buf:           make([][]byte, len(readers)),
 		readerToBuf:   r2b,
+		stashBuffer:   b,
+	}
+}
+
+// Done will release any resources used by the parallelReader.
+func (p *parallelReader) Done() {
+	if p.stashBuffer != nil {
+		globalBytePoolCap.Load().Put(p.stashBuffer)
+		p.stashBuffer = nil
 	}
 }
 
@@ -224,6 +249,7 @@ func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.Read
 	if len(prefer) == len(readers) {
 		reader.preferReaders(prefer)
 	}
+	defer reader.Done()
 
 	startBlock := offset / e.blockSize
 	endBlock := (offset + length) / e.blockSize
@@ -294,6 +320,7 @@ func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.Rea
 	if len(readers) == len(prefer) {
 		reader.preferReaders(prefer)
 	}
+	defer reader.Done()
 
 	startBlock := int64(0)
 	endBlock := totalLength / e.blockSize

cmd/erasure-healing.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/grid"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/sync/errgroup"
 )
@@ -43,8 +44,8 @@ const (
 	healingMetricCheckAbandonedParts
 )
 
-func (er erasureObjects) listAndHeal(bucket, prefix string, scanMode madmin.HealScanMode, healEntry func(string, metaCacheEntry, madmin.HealScanMode) error) error {
-	ctx, cancel := context.WithCancel(context.Background())
+func (er erasureObjects) listAndHeal(ctx context.Context, bucket, prefix string, scanMode madmin.HealScanMode, healEntry func(string, metaCacheEntry, madmin.HealScanMode) error) error {
+	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	disks, _ := er.getOnlineDisksWithHealing(false)
@@ -52,6 +53,10 @@ func (er erasureObjects) listAndHeal(bucket, prefix string, scanMode madmin.Heal
 		return errors.New("listAndHeal: No non-healing drives found")
 	}
 
+	expectedDisks := len(disks)/2 + 1
+	fallbackDisks := disks[expectedDisks:]
+	disks = disks[:expectedDisks]
+
 	// How to resolve partial results.
 	resolver := metadataResolutionParams{
 		dirQuorum: 1,
@@ -68,6 +73,7 @@ func (er erasureObjects) listAndHeal(bucket, prefix string, scanMode madmin.Heal
 
 	lopts := listPathRawOptions{
 		disks:          disks,
+		fallbackDisks:  fallbackDisks,
 		bucket:         bucket,
 		path:           path,
 		filterPrefix:   filterPrefix,
@@ -210,7 +216,7 @@ func (fi FileInfo) DataMov() bool {
 	return ok
 }
 
-func auditHealObject(ctx context.Context, bucket, object, versionID string, err error) {
+func auditHealObject(ctx context.Context, bucket, object, versionID string, result madmin.HealResultItem, err error) {
 	if len(logger.AuditTargets()) == 0 {
 		return
 	}
@@ -225,6 +231,10 @@ func auditHealObject(ctx context.Context, bucket, object, versionID string, err
 		opts.Error = err.Error()
 	}
 
+	if result.After.Drives != nil {
+		opts.Tags = map[string]interface{}{"drives-result": result.After.Drives}
+	}
+
 	auditLogInternal(ctx, opts)
 }
 
@@ -236,7 +246,9 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	storageDisks := er.getDisks()
 	storageEndpoints := er.getEndpoints()
 
-	defer auditHealObject(ctx, bucket, object, versionID, err)
+	defer func() {
+		auditHealObject(ctx, bucket, object, versionID, result, err)
+	}()
 
 	if globalTrace.NumSubscribers(madmin.TraceHealing) > 0 {
 		startTime := time.Now()
@@ -359,21 +371,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			driveState = madmin.DriveStateCorrupt
 		}
 
-		if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta) {
-			outDatedDisks[i] = storageDisks[i]
-			disksToHealCount++
-			result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
-				UUID:     "",
-				Endpoint: storageEndpoints[i].String(),
-				State:    driveState,
-			})
-			result.After.Drives = append(result.After.Drives, madmin.HealDriveInfo{
-				UUID:     "",
-				Endpoint: storageEndpoints[i].String(),
-				State:    driveState,
-			})
-			continue
-		}
 		result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
 			UUID:     "",
 			Endpoint: storageEndpoints[i].String(),
@@ -384,6 +381,12 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			Endpoint: storageEndpoints[i].String(),
 			State:    driveState,
 		})
+
+		if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta) {
+			outDatedDisks[i] = storageDisks[i]
+			disksToHealCount++
+			continue
+		}
 	}
 
 	if isAllNotFound(errs) {
@@ -449,7 +452,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(availableDisks) {
 		err := fmt.Errorf("unexpected file distribution (%v) from available disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
 			latestMeta.Erasure.Distribution, availableDisks, bucket, object, versionID)
-		logger.LogOnceIf(ctx, err, "heal-object-available-disks")
+		healingLogOnceIf(ctx, err, "heal-object-available-disks")
 		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
 			bucket, object, versionID), err
 	}
@@ -459,7 +462,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(outDatedDisks) {
 		err := fmt.Errorf("unexpected file distribution (%v) from outdated disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
 			latestMeta.Erasure.Distribution, outDatedDisks, bucket, object, versionID)
-		logger.LogOnceIf(ctx, err, "heal-object-outdated-disks")
+		healingLogOnceIf(ctx, err, "heal-object-outdated-disks")
 		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
 			bucket, object, versionID), err
 	}
@@ -469,7 +472,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(partsMetadata) {
 		err := fmt.Errorf("unexpected file distribution (%v) from metadata entries (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
 			latestMeta.Erasure.Distribution, len(partsMetadata), bucket, object, versionID)
-		logger.LogOnceIf(ctx, err, "heal-object-metadata-entries")
+		healingLogOnceIf(ctx, err, "heal-object-metadata-entries")
 		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
 			bucket, object, versionID), err
 	}
@@ -539,7 +542,10 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 				}
 				partPath := pathJoin(tmpID, dstDataDir, fmt.Sprintf("part.%d", partNumber))
 				if len(inlineBuffers) > 0 {
-					inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, erasure.ShardFileSize(latestMeta.Size)+32))
+					buf := grid.GetByteBufferCap(int(erasure.ShardFileSize(latestMeta.Size)) + 64)
+					inlineBuffers[i] = bytes.NewBuffer(buf[:0])
+					defer grid.PutByteBuffer(buf)
+
 					writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
 				} else {
 					writers[i] = newBitrotWriter(disk, bucket, minioMetaTmpBucket, partPath,

cmd/erasure-metadata-utils.go

@@ -19,22 +19,25 @@ package cmd
 
 import (
 	"context"
+	"encoding/binary"
 	"errors"
 	"hash/crc32"
 
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/sync/errgroup"
 )
 
 // figure out the most commonVersions across disk that satisfies
-// the 'writeQuorum' this function returns '0' if quorum cannot
+// the 'writeQuorum' this function returns "" if quorum cannot
 // be achieved and disks have too many inconsistent versions.
-func reduceCommonVersions(diskVersions []uint64, writeQuorum int) (commonVersions uint64) {
+func reduceCommonVersions(diskVersions [][]byte, writeQuorum int) (versions []byte) {
 	diskVersionsCount := make(map[uint64]int)
 	for _, versions := range diskVersions {
-		diskVersionsCount[versions]++
+		if len(versions) > 0 {
+			diskVersionsCount[binary.BigEndian.Uint64(versions)]++
+		}
 	}
 
+	var commonVersions uint64
 	max := 0
 	for versions, count := range diskVersionsCount {
 		if max < count {
@@ -44,10 +47,38 @@ func reduceCommonVersions(diskVersions []uint64, writeQuorum int) (commonVersion
 	}
 
 	if max >= writeQuorum {
-		return commonVersions
+		for _, versions := range diskVersions {
+			if binary.BigEndian.Uint64(versions) == commonVersions {
+				return versions
+			}
+		}
 	}
 
-	return 0
+	return []byte{}
+}
+
+// figure out the most commonVersions across disk that satisfies
+// the 'writeQuorum' this function returns '0' if quorum cannot
+// be achieved and disks have too many inconsistent versions.
+func reduceCommonDataDir(dataDirs []string, writeQuorum int) (dataDir string) {
+	dataDirsCount := make(map[string]int)
+	for _, ddir := range dataDirs {
+		dataDirsCount[ddir]++
+	}
+
+	max := 0
+	for ddir, count := range dataDirsCount {
+		if max < count {
+			max = count
+			dataDir = ddir
+		}
+	}
+
+	if max >= writeQuorum {
+		return dataDir
+	}
+
+	return ""
 }
 
 // Returns number of errors that occurred the most (incl. nil) and the
@@ -284,7 +315,7 @@ func shuffleDisks(disks []StorageAPI, distribution []int) (shuffledDisks []Stora
 // the corresponding error in errs slice is not nil
 func evalDisks(disks []StorageAPI, errs []error) []StorageAPI {
 	if len(errs) != len(disks) {
-		logger.LogIf(GlobalContext, errors.New("unexpected drives/errors slice length"))
+		bugLogIf(GlobalContext, errors.New("unexpected drives/errors slice length"))
 		return nil
 	}
 	newDisks := make([]StorageAPI, len(disks))

cmd/erasure-metadata.go

@@ -30,7 +30,6 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/sync/errgroup"
 )
 
@@ -268,7 +267,7 @@ func (fi FileInfo) ObjectToPartOffset(ctx context.Context, offset int64) (partIn
 		// Continue to towards the next part.
 		partOffset -= part.Size
 	}
-	logger.LogIf(ctx, InvalidRange{})
+	internalLogIf(ctx, InvalidRange{})
 	// Offset beyond the size of the object return InvalidRange.
 	return 0, 0, InvalidRange{}
 }
@@ -276,7 +275,7 @@ func (fi FileInfo) ObjectToPartOffset(ctx context.Context, offset int64) (partIn
 func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.Time, etag string, quorum int) (FileInfo, error) {
 	// with less quorum return error.
 	if quorum < 1 {
-		return FileInfo{}, errErasureReadQuorum
+		return FileInfo{}, InsufficientReadQuorum{Err: errErasureReadQuorum, Type: RQInsufficientOnlineDrives}
 	}
 	metaHashes := make([]string, len(metaArr))
 	h := sha256.New()
@@ -342,7 +341,7 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 	}
 
 	if maxCount < quorum {
-		return FileInfo{}, errErasureReadQuorum
+		return FileInfo{}, InsufficientReadQuorum{Err: errErasureReadQuorum, Type: RQInconsistentMeta}
 	}
 
 	// Find the successor mod time in quorum, otherwise leave the
@@ -378,7 +377,7 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 		}
 		return candidate, nil
 	}
-	return FileInfo{}, errErasureReadQuorum
+	return FileInfo{}, InsufficientReadQuorum{Err: errErasureReadQuorum, Type: RQInconsistentMeta}
 }
 
 // pickValidFileInfo - picks one valid FileInfo content and returns from a
@@ -404,7 +403,7 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, origbucket, bu
 			if fi.IsValid() {
 				return disks[index].WriteMetadata(ctx, origbucket, bucket, prefix, fi)
 			}
-			return errCorruptedFormat
+			return errFileCorrupt
 		}, index)
 	}
 
@@ -499,7 +498,7 @@ func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []
 	parities := listObjectParities(partsMetaData, errs)
 	parityBlocks := commonParity(parities, defaultParityCount)
 	if parityBlocks < 0 {
-		return -1, -1, errErasureReadQuorum
+		return -1, -1, InsufficientReadQuorum{Err: errErasureReadQuorum, Type: RQInsufficientOnlineDrives}
 	}
 
 	dataBlocks := len(partsMetaData) - parityBlocks

cmd/erasure-metadata_test.go

@@ -210,13 +210,13 @@ func TestFindFileInfoInQuorum(t *testing.T) {
 		{
 			fis:            getNFInfo(16, 7, 1603863445, "36a21454-a2ca-11eb-bbaa-93a81c686f21", nil),
 			modTime:        time.Unix(1603863445, 0),
-			expectedErr:    errErasureReadQuorum,
+			expectedErr:    InsufficientReadQuorum{},
 			expectedQuorum: 8,
 		},
 		{
 			fis:            getNFInfo(16, 16, 1603863445, "36a21454-a2ca-11eb-bbaa-93a81c686f21", nil),
 			modTime:        time.Unix(1603863445, 0),
-			expectedErr:    errErasureReadQuorum,
+			expectedErr:    InsufficientReadQuorum{},
 			expectedQuorum: 0,
 		},
 		{
@@ -241,7 +241,9 @@ func TestFindFileInfoInQuorum(t *testing.T) {
 		test := test
 		t.Run("", func(t *testing.T) {
 			fi, err := findFileInfoInQuorum(context.Background(), test.fis, test.modTime, "", test.expectedQuorum)
-			if err != test.expectedErr {
+			_, ok1 := err.(InsufficientReadQuorum)
+			_, ok2 := test.expectedErr.(InsufficientReadQuorum)
+			if ok1 != ok2 {
 				t.Errorf("Expected %s, got %s", test.expectedErr, err)
 			}
 			if test.succmodTimes != nil {

cmd/erasure-multipart.go

@@ -194,11 +194,10 @@ func (er erasureObjects) deleteAll(ctx context.Context, bucket, prefix string) {
 
 // Remove the old multipart uploads on the given disk.
 func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk StorageAPI, expiry time.Duration) {
-	now := time.Now()
-	diskPath := disk.Endpoint().Path
+	drivePath := disk.Endpoint().Path
 
-	readDirFn(pathJoin(diskPath, minioMetaMultipartBucket), func(shaDir string, typ os.FileMode) error {
-		readDirFn(pathJoin(diskPath, minioMetaMultipartBucket, shaDir), func(uploadIDDir string, typ os.FileMode) error {
+	readDirFn(pathJoin(drivePath, minioMetaMultipartBucket), func(shaDir string, typ os.FileMode) error {
+		readDirFn(pathJoin(drivePath, minioMetaMultipartBucket, shaDir), func(uploadIDDir string, typ os.FileMode) error {
 			uploadIDPath := pathJoin(shaDir, uploadIDDir)
 			fi, err := disk.ReadVersion(ctx, "", minioMetaMultipartBucket, uploadIDPath, "", ReadOptions{})
 			if err != nil {
@@ -206,9 +205,12 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 			}
 			w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 			return w.Run(func() error {
-				wait := deletedCleanupSleeper.Timer(ctx)
-				if now.Sub(fi.ModTime) > expiry {
-					removeAll(pathJoin(diskPath, minioMetaMultipartBucket, uploadIDPath))
+				wait := deleteMultipartCleanupSleeper.Timer(ctx)
+				if time.Since(fi.ModTime) > expiry {
+					pathUUID := mustGetUUID()
+					targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID)
+
+					renameAll(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath), targetPath, pathJoin(drivePath, minioMetaBucket))
 				}
 				wait()
 				return nil
@@ -220,19 +222,23 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		}
 		w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 		return w.Run(func() error {
-			wait := deletedCleanupSleeper.Timer(ctx)
-			if now.Sub(vi.Created) > expiry {
+			wait := deleteMultipartCleanupSleeper.Timer(ctx)
+			if time.Since(vi.Created) > expiry {
+				pathUUID := mustGetUUID()
+				targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID)
+
 				// We are not deleting shaDir recursively here, if shaDir is empty
 				// and its older then we can happily delete it.
-				Remove(pathJoin(diskPath, minioMetaMultipartBucket, shaDir))
+				Rename(pathJoin(drivePath, minioMetaMultipartBucket, shaDir), targetPath)
 			}
 			wait()
 			return nil
 		})
 	})
 
-	readDirFn(pathJoin(diskPath, minioMetaTmpBucket), func(tmpDir string, typ os.FileMode) error {
-		if tmpDir == ".trash/" { // do not remove .trash/ here, it has its own routines
+	readDirFn(pathJoin(drivePath, minioMetaTmpBucket), func(tmpDir string, typ os.FileMode) error {
+		if strings.HasPrefix(tmpDir, ".trash") {
+			// do not remove .trash/ here, it has its own routines
 			return nil
 		}
 		vi, err := disk.StatVol(ctx, pathJoin(minioMetaTmpBucket, tmpDir))
@@ -241,9 +247,12 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		}
 		w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 		return w.Run(func() error {
-			wait := deletedCleanupSleeper.Timer(ctx)
-			if now.Sub(vi.Created) > expiry {
-				removeAll(pathJoin(diskPath, minioMetaTmpBucket, tmpDir))
+			wait := deleteMultipartCleanupSleeper.Timer(ctx)
+			if time.Since(vi.Created) > expiry {
+				pathUUID := mustGetUUID()
+				targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID)
+
+				renameAll(pathJoin(drivePath, minioMetaTmpBucket, tmpDir), targetPath, pathJoin(drivePath, minioMetaBucket))
 			}
 			wait()
 			return nil
@@ -590,7 +599,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	data := r.Reader
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < -1 {
-		logger.LogIf(ctx, errInvalidArgument, logger.ErrorKind)
+		bugLogIf(ctx, errInvalidArgument, logger.ErrorKind)
 		return pi, toObjectErr(errInvalidArgument)
 	}
 
@@ -670,12 +679,12 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 			// Account for padding and forced compression overhead and encryption.
 			buffer = make([]byte, data.ActualSize()+256+32+32, data.ActualSize()*2+512)
 		} else {
-			buffer = globalBytePoolCap.Get()
-			defer globalBytePoolCap.Put(buffer)
+			buffer = globalBytePoolCap.Load().Get()
+			defer globalBytePoolCap.Load().Put(buffer)
 		}
 	case size >= fi.Erasure.BlockSize:
-		buffer = globalBytePoolCap.Get()
-		defer globalBytePoolCap.Put(buffer)
+		buffer = globalBytePoolCap.Load().Get()
+		defer globalBytePoolCap.Load().Put(buffer)
 	case size < fi.Erasure.BlockSize:
 		// No need to allocate fully fi.Erasure.BlockSize buffer if the incoming data is smaller.
 		buffer = make([]byte, size, 2*size+int64(fi.Erasure.ParityBlocks+fi.Erasure.DataBlocks-1))
@@ -696,10 +705,11 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	if data.Size() > bigFileThreshold {
 		// Add input readahead.
 		// We use 2 buffers, so we always have a full buffer of input.
-		bufA := globalBytePoolCap.Get()
-		bufB := globalBytePoolCap.Get()
-		defer globalBytePoolCap.Put(bufA)
-		defer globalBytePoolCap.Put(bufB)
+		pool := globalBytePoolCap.Load()
+		bufA := pool.Get()
+		bufB := pool.Get()
+		defer pool.Put(bufA)
+		defer pool.Put(bufB)
 		ra, err := readahead.NewReaderBuffer(data, [][]byte{bufA[:fi.Erasure.BlockSize], bufB[:fi.Erasure.BlockSize]})
 		if err == nil {
 			toEncode = ra
@@ -729,6 +739,15 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	partPath := pathJoin(uploadIDPath, fi.DataDir, partSuffix)
 	onlineDisks, err = renamePart(ctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, writeQuorum)
 	if err != nil {
+		if errors.Is(err, errFileNotFound) {
+			// An in-quorum errFileNotFound means that client stream
+			// prematurely closed and we do not find any xl.meta or
+			// part.1's - in such a scenario we must return as if client
+			// disconnected. This means that erasure.Encode() CreateFile()
+			// did not do anything.
+			return pi, IncompleteBody{Bucket: bucket, Object: object}
+		}
+
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 
@@ -1026,7 +1045,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	if len(partInfoFiles) != len(parts) {
 		// Should only happen through internal error
 		err := fmt.Errorf("unexpected part result count: %d, want %d", len(partInfoFiles), len(parts))
-		logger.LogIf(ctx, err)
+		bugLogIf(ctx, err)
 		return oi, toObjectErr(err, bucket, object)
 	}
 
@@ -1096,7 +1115,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		_, err := pfi.UnmarshalMsg(part.Data)
 		if err != nil {
 			// Maybe crash or similar.
-			logger.LogIf(ctx, err)
+			bugLogIf(ctx, err)
 			return oi, InvalidPart{
 				PartNumber: partID,
 			}
@@ -1105,7 +1124,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		partI := pfi.Parts[0]
 		partNumber := partI.Number
 		if partID != partNumber {
-			logger.LogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", partID, partID, partI.Number))
+			internalLogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", partID, partID, partI.Number))
 			return oi, InvalidPart{
 				PartNumber: partID,
 			}
@@ -1301,24 +1320,28 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	}()
 
 	// Rename the multipart object to final location.
-	onlineDisks, versionsDisparity, err := renameData(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath,
+	onlineDisks, versions, oldDataDir, err := renameData(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath,
 		partsMetadata, bucket, object, writeQuorum)
 	if err != nil {
-		return oi, toObjectErr(err, bucket, object)
+		return oi, toObjectErr(err, bucket, object, uploadID)
 	}
 
-	if !opts.Speedtest && versionsDisparity {
+	if err = er.commitRenameDataDir(ctx, bucket, object, oldDataDir, onlineDisks); err != nil {
+		return ObjectInfo{}, toObjectErr(err, bucket, object, uploadID)
+	}
+
+	if !opts.Speedtest && len(versions) > 0 {
 		globalMRFState.addPartialOp(partialOperation{
-			bucket:      bucket,
-			object:      object,
-			queued:      time.Now(),
-			allVersions: true,
-			setIndex:    er.setIndex,
-			poolIndex:   er.poolIndex,
+			bucket:    bucket,
+			object:    object,
+			queued:    time.Now(),
+			versions:  versions,
+			setIndex:  er.setIndex,
+			poolIndex: er.poolIndex,
 		})
 	}
 
-	if !opts.Speedtest && !versionsDisparity {
+	if !opts.Speedtest && len(versions) == 0 {
 		// Check if there is any offline disk and add it to the MRF list
 		for _, disk := range onlineDisks {
 			if disk != nil && disk.IsOnline() {

cmd/erasure-object.go

@@ -41,13 +41,13 @@ import (
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/grid"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/mimedb"
 	"github.com/minio/pkg/v2/sync/errgroup"
-	"github.com/minio/pkg/v2/wildcard"
 )
 
 // list all errors which can be ignored in object operations.
@@ -146,11 +146,12 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		modTime = dstOpts.MTime
 		fi.ModTime = dstOpts.MTime
 	}
+	// check inline before overwriting metadata.
+	inlineData := fi.InlineData()
 
 	fi.Metadata = srcInfo.UserDefined
 	srcInfo.UserDefined["etag"] = srcInfo.ETag
 
-	inlineData := fi.InlineData()
 	freeVersionID := fi.TierFreeVersionID()
 	freeVersionMarker := fi.TierFreeVersion()
 
@@ -561,7 +562,7 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 func fileInfoFromRaw(ri RawFileInfo, bucket, object string, readData, inclFreeVers, allParts bool) (FileInfo, error) {
 	var xl xlMetaV2
 	if err := xl.LoadOrConvert(ri.Buf); err != nil {
-		return FileInfo{}, err
+		return FileInfo{}, errFileCorrupt
 	}
 
 	fi, err := xl.ToFileInfo(bucket, object, "", inclFreeVers, allParts)
@@ -570,7 +571,7 @@ func fileInfoFromRaw(ri RawFileInfo, bucket, object string, readData, inclFreeVe
 	}
 
 	if !fi.IsValid() {
-		return FileInfo{}, errCorruptedFormat
+		return FileInfo{}, errFileCorrupt
 	}
 
 	versionID := fi.VersionID
@@ -660,7 +661,7 @@ func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo,
 	if !lfi.IsValid() {
 		for i := range errs {
 			if errs[i] == nil {
-				errs[i] = errCorruptedFormat
+				errs[i] = errFileCorrupt
 			}
 		}
 		return metaFileInfos, errs
@@ -927,7 +928,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	if !fi.Deleted && len(fi.Erasure.Distribution) != len(onlineDisks) {
 		err := fmt.Errorf("unexpected file distribution (%v) from online disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)",
 			fi.Erasure.Distribution, onlineDisks, bucket, object, opts.VersionID)
-		logger.LogOnceIf(ctx, err, "get-object-file-info-manually-modified")
+		storageLogOnceIf(ctx, err, "get-object-file-info-manually-modified")
 		return fi, nil, nil, toObjectErr(err, bucket, object, opts.VersionID)
 	}
 
@@ -1004,7 +1005,7 @@ func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, obj
 }
 
 // Similar to rename but renames data from srcEntry to dstEntry at dataDir
-func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry string, metadata []FileInfo, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, bool, error) {
+func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry string, metadata []FileInfo, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, []byte, string, error) {
 	g := errgroup.WithNErrs(len(disks))
 
 	fvID := mustGetUUID()
@@ -1012,7 +1013,8 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str
 		metadata[index].SetTierFreeVersionID(fvID)
 	}
 
-	diskVersions := make([]uint64, len(disks))
+	diskVersions := make([][]byte, len(disks))
+	dataDirs := make([]string, len(disks))
 	// Rename file on all underlying storage disks.
 	for index := range disks {
 		index := index
@@ -1031,11 +1033,12 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str
 			if !fi.IsValid() {
 				return errFileCorrupt
 			}
-			sign, err := disks[index].RenameData(ctx, srcBucket, srcEntry, fi, dstBucket, dstEntry, RenameOptions{})
+			resp, err := disks[index].RenameData(ctx, srcBucket, srcEntry, fi, dstBucket, dstEntry, RenameOptions{})
 			if err != nil {
 				return err
 			}
-			diskVersions[index] = sign
+			diskVersions[index] = resp.Sign
+			dataDirs[index] = resp.OldDataDir
 			return nil
 		}, index)
 	}
@@ -1043,8 +1046,6 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str
 	// Wait for all renames to finish.
 	errs := g.Wait()
 
-	var versionsDisparity bool
-
 	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
 	if err != nil {
 		dg := errgroup.WithNErrs(len(disks))
@@ -1058,27 +1059,35 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str
 			// caller this dangling object will be now scheduled to be removed
 			// via active healing.
 			dg.Go(func() error {
-				return disks[index].DeleteVersion(context.Background(), dstBucket, dstEntry, metadata[index], false, DeleteOptions{UndoWrite: true})
+				return disks[index].DeleteVersion(context.Background(), dstBucket, dstEntry, metadata[index], false, DeleteOptions{
+					UndoWrite:  true,
+					OldDataDir: dataDirs[index],
+				})
 			}, index)
 		}
 		dg.Wait()
 	}
+	var dataDir string
+	var versions []byte
 	if err == nil {
-		versions := reduceCommonVersions(diskVersions, writeQuorum)
+		versions = reduceCommonVersions(diskVersions, writeQuorum)
 		for index, dversions := range diskVersions {
 			if errs[index] != nil {
 				continue
 			}
-			if versions != dversions {
-				versionsDisparity = true
+			if !bytes.Equal(dversions, versions) {
+				if len(dversions) > len(versions) {
+					versions = dversions
+				}
 				break
 			}
 		}
+		dataDir = reduceCommonDataDir(dataDirs, writeQuorum)
 	}
 
 	// We can safely allow RenameData errors up to len(er.getDisks()) - writeQuorum
 	// otherwise return failure.
-	return evalDisks(disks, errs), versionsDisparity, err
+	return evalDisks(disks, errs), versions, dataDir, err
 }
 
 func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
@@ -1106,7 +1115,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < -1 {
-		logger.LogIf(ctx, errInvalidArgument, logger.ErrorKind)
+		bugLogIf(ctx, errInvalidArgument, logger.ErrorKind)
 		return ObjectInfo{}, toObjectErr(errInvalidArgument)
 	}
 
@@ -1136,8 +1145,8 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 	case size == 0:
 		buffer = make([]byte, 1) // Allocate at least a byte to reach EOF
 	case size >= fi.Erasure.BlockSize:
-		buffer = globalBytePoolCap.Get()
-		defer globalBytePoolCap.Put(buffer)
+		buffer = globalBytePoolCap.Load().Get()
+		defer globalBytePoolCap.Load().Put(buffer)
 	case size < fi.Erasure.BlockSize:
 		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
 		buffer = make([]byte, size, 2*size+int64(fi.Erasure.ParityBlocks+fi.Erasure.DataBlocks-1))
@@ -1147,7 +1156,6 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 		buffer = buffer[:fi.Erasure.BlockSize]
 	}
 
-	shardFileSize := erasure.ShardFileSize(data.Size())
 	writers := make([]io.Writer, len(onlineDisks))
 	inlineBuffers := make([]*bytes.Buffer, len(onlineDisks))
 	for i, disk := range onlineDisks {
@@ -1155,7 +1163,9 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 			continue
 		}
 		if disk.IsOnline() {
-			inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, shardFileSize))
+			buf := grid.GetByteBufferCap(int(erasure.ShardFileSize(data.Size())) + 64)
+			inlineBuffers[i] = bytes.NewBuffer(buf[:0])
+			defer grid.PutByteBuffer(buf)
 			writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 	}
@@ -1226,44 +1236,6 @@ func (er erasureObjects) PutObject(ctx context.Context, bucket string, object st
 	return er.putObject(ctx, bucket, object, data, opts)
 }
 
-// Heal up to two versions of one object when there is disparity between disks
-func healObjectVersionsDisparity(bucket string, entry metaCacheEntry, scanMode madmin.HealScanMode) error {
-	if entry.isDir() {
-		return nil
-	}
-	// We might land at .metacache, .trash, .multipart
-	// no need to heal them skip, only when bucket
-	// is '.minio.sys'
-	if bucket == minioMetaBucket {
-		if wildcard.Match("buckets/*/.metacache/*", entry.name) {
-			return nil
-		}
-		if wildcard.Match("tmp/*", entry.name) {
-			return nil
-		}
-		if wildcard.Match("multipart/*", entry.name) {
-			return nil
-		}
-		if wildcard.Match("tmp-old/*", entry.name) {
-			return nil
-		}
-	}
-
-	fivs, err := entry.fileInfoVersions(bucket)
-	if err != nil {
-		healObject(bucket, entry.name, "", madmin.HealDeepScan)
-		return err
-	}
-
-	if len(fivs.Versions) <= 2 {
-		for _, version := range fivs.Versions {
-			healObject(bucket, entry.name, version.VersionID, scanMode)
-		}
-	}
-
-	return nil
-}
-
 // putObject wrapper for erasureObjects PutObject
 func (er erasureObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	if !opts.NoAuditLog {
@@ -1295,7 +1267,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	// Validate input data size and it can never be less than -1.
 	if data.Size() < -1 {
-		logger.LogIf(ctx, errInvalidArgument, logger.ErrorKind)
+		bugLogIf(ctx, errInvalidArgument, logger.ErrorKind)
 		return ObjectInfo{}, toObjectErr(errInvalidArgument)
 	}
 
@@ -1385,8 +1357,8 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	case size == 0:
 		buffer = make([]byte, 1) // Allocate at least a byte to reach EOF
 	case size >= fi.Erasure.BlockSize || size == -1:
-		buffer = globalBytePoolCap.Get()
-		defer globalBytePoolCap.Put(buffer)
+		buffer = globalBytePoolCap.Load().Get()
+		defer globalBytePoolCap.Load().Put(buffer)
 	case size < fi.Erasure.BlockSize:
 		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
 		buffer = make([]byte, size, 2*size+int64(fi.Erasure.ParityBlocks+fi.Erasure.DataBlocks-1))
@@ -1435,11 +1407,9 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		}
 
 		if len(inlineBuffers) > 0 {
-			sz := shardFileSize
-			if sz < 0 {
-				sz = data.ActualSize()
-			}
-			inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, sz))
+			buf := grid.GetByteBufferCap(int(shardFileSize) + 64)
+			inlineBuffers[i] = bytes.NewBuffer(buf[:0])
+			defer grid.PutByteBuffer(buf)
 			writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
 			continue
 		}
@@ -1448,18 +1418,19 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	toEncode := io.Reader(data)
-	if data.Size() > bigFileThreshold {
+	if data.Size() >= bigFileThreshold {
 		// We use 2 buffers, so we always have a full buffer of input.
-		bufA := globalBytePoolCap.Get()
-		bufB := globalBytePoolCap.Get()
-		defer globalBytePoolCap.Put(bufA)
-		defer globalBytePoolCap.Put(bufB)
+		pool := globalBytePoolCap.Load()
+		bufA := pool.Get()
+		bufB := pool.Get()
+		defer pool.Put(bufA)
+		defer pool.Put(bufB)
 		ra, err := readahead.NewReaderBuffer(data, [][]byte{bufA[:fi.Erasure.BlockSize], bufB[:fi.Erasure.BlockSize]})
 		if err == nil {
 			toEncode = ra
 			defer ra.Close()
 		}
-		logger.LogIf(ctx, err)
+		bugLogIf(ctx, err)
 	}
 	n, erasureErr := erasure.Encode(ctx, toEncode, writers, buffer, writeQuorum)
 	closeBitrotWriters(writers)
@@ -1545,14 +1516,23 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	// Rename the successfully written temporary object to final location.
-	onlineDisks, versionsDisparity, err := renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum)
+	onlineDisks, versions, oldDataDir, err := renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum)
 	if err != nil {
 		if errors.Is(err, errFileNotFound) {
-			return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object)
+			// An in-quorum errFileNotFound means that client stream
+			// prematurely closed and we do not find any xl.meta or
+			// part.1's - in such a scenario we must return as if client
+			// disconnected. This means that erasure.Encode() CreateFile()
+			// did not do anything.
+			return ObjectInfo{}, IncompleteBody{Bucket: bucket, Object: object}
 		}
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
+	if err = er.commitRenameDataDir(ctx, bucket, object, oldDataDir, onlineDisks); err != nil {
+		return ObjectInfo{}, toObjectErr(err, bucket, object)
+	}
+
 	for i := 0; i < len(onlineDisks); i++ {
 		if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
 			// Object info is the same in all disks, so we can pick
@@ -1567,7 +1547,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		// When there is versions disparity we are healing
 		// the content implicitly for all versions, we can
 		// avoid triggering another MRF heal for offline drives.
-		if !versionsDisparity {
+		if len(versions) == 0 {
 			// Whether a disk was initially or becomes offline
 			// during this upload, send it to the MRF list.
 			for i := 0; i < len(onlineDisks); i++ {
@@ -1580,12 +1560,12 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			}
 		} else {
 			globalMRFState.addPartialOp(partialOperation{
-				bucket:      bucket,
-				object:      object,
-				queued:      time.Now(),
-				allVersions: true,
-				setIndex:    er.setIndex,
-				poolIndex:   er.poolIndex,
+				bucket:    bucket,
+				object:    object,
+				queued:    time.Now(),
+				versions:  versions,
+				setIndex:  er.setIndex,
+				poolIndex: er.poolIndex,
 			})
 		}
 	}
@@ -1795,6 +1775,30 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 	return dobjects, errs
 }
 
+func (er erasureObjects) commitRenameDataDir(ctx context.Context, bucket, object, dataDir string, onlineDisks []StorageAPI) error {
+	if dataDir == "" {
+		return nil
+	}
+	g := errgroup.WithNErrs(len(onlineDisks))
+	for index := range onlineDisks {
+		index := index
+		g.Go(func() error {
+			if onlineDisks[index] == nil {
+				return nil
+			}
+			return onlineDisks[index].Delete(ctx, bucket, pathJoin(object, dataDir), DeleteOptions{
+				Recursive: true,
+			})
+		}, index)
+	}
+	for _, err := range g.Wait() {
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string) error {
 	disks := er.getDisks()
 	g := errgroup.WithNErrs(len(disks))
@@ -1831,9 +1835,18 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	var replcfg *replication.Config
 	if opts.Expiration.Expire {
 		// Check if the current bucket has a configured lifecycle policy
-		lc, _ = globalLifecycleSys.Get(bucket)
-		rcfg, _ = globalBucketObjectLockSys.Get(bucket)
-		replcfg, _ = getReplicationConfig(ctx, bucket)
+		lc, err = globalLifecycleSys.Get(bucket)
+		if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) {
+			return objInfo, err
+		}
+		rcfg, err = globalBucketObjectLockSys.Get(bucket)
+		if err != nil {
+			return objInfo, err
+		}
+		replcfg, err = getReplicationConfig(ctx, bucket)
+		if err != nil {
+			return objInfo, err
+		}
 	}
 
 	// expiration attempted on a bucket with no lifecycle
@@ -1873,12 +1886,13 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 			// based on the latest objectInfo and see if the object still
 			// qualifies for deletion.
 			if gerr == nil {
-				evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi)
 				var isErr bool
+				evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi)
 				switch evt.Action {
-				case lifecycle.NoneAction:
-					isErr = true
-				case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+				case lifecycle.DeleteAllVersionsAction, lifecycle.DelMarkerDeleteAllVersionsAction:
+					// opts.DeletePrefix is used only in the above lifecycle Expiration actions.
+				default:
+					// object has been modified since lifecycle action was previously evaluated
 					isErr = true
 				}
 				if isErr {
@@ -1904,23 +1918,26 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	versionFound := true
 	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
 	goi, _, gerr := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
+	tryDel := false
 	if gerr != nil && goi.Name == "" {
 		if _, ok := gerr.(InsufficientReadQuorum); ok {
-			// Add an MRF heal for next time.
-			er.addPartial(bucket, object, opts.VersionID)
-
-			return objInfo, InsufficientWriteQuorum{}
+			if opts.Versioned || opts.VersionSuspended || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 {
+				// Add an MRF heal for next time.
+				er.addPartial(bucket, object, opts.VersionID)
+				return objInfo, InsufficientWriteQuorum{}
+			}
+			tryDel = true // only for unversioned objects if there is write quorum
 		}
 		// For delete marker replication, versionID being replicated will not exist on disk
 		if opts.DeleteMarker {
 			versionFound = false
-		} else {
+		} else if !tryDel {
 			return objInfo, gerr
 		}
 	}
 
 	if opts.EvalMetadataFn != nil {
-		dsc, err := opts.EvalMetadataFn(&goi, err)
+		dsc, err := opts.EvalMetadataFn(&goi, gerr)
 		if err != nil {
 			return ObjectInfo{}, err
 		}
@@ -2269,7 +2286,7 @@ func (er erasureObjects) GetObjectTags(ctx context.Context, bucket, object strin
 
 // TransitionObject - transition object content to target tier.
 func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
-	tgtClient, err := globalTierConfigMgr.getDriver(opts.Transition.Tier)
+	tgtClient, err := globalTierConfigMgr.getDriver(ctx, opts.Transition.Tier)
 	if err != nil {
 		return err
 	}
@@ -2389,7 +2406,7 @@ func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, obje
 	}, ObjectOptions{
 		VersionID: oi.VersionID,
 	}); err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err))
+		storageLogIf(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err))
 		return err
 	}
 	return nil

cmd/erasure-server-pool-decom.go

@@ -31,6 +31,10 @@ import (
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
@@ -467,7 +471,7 @@ func (p poolMeta) save(ctx context.Context, pools []*erasureSets) error {
 	for i, eset := range pools {
 		if err = saveConfig(ctx, eset, poolMetaName, buf); err != nil {
 			if !errors.Is(err, context.Canceled) {
-				logger.LogIf(ctx, fmt.Errorf("saving pool.bin for pool index %d failed with: %v", i, err))
+				storageLogIf(ctx, fmt.Errorf("saving pool.bin for pool index %d failed with: %v", i, err))
 			}
 			return err
 		}
@@ -542,11 +546,11 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 						return
 					}
 					if configRetriableErrors(err) {
-						logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pools %v: %w: retrying..", pools, err))
+						decomLogIf(ctx, fmt.Errorf("Unable to resume decommission of pools %v: %w: retrying..", pools, err))
 						time.Sleep(time.Second + time.Duration(r.Float64()*float64(5*time.Second)))
 						continue
 					}
-					logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pools, err))
+					decomLogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pools, err))
 					return
 				}
 			}
@@ -741,7 +745,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 	const envDecomWorkers = "_MINIO_DECOMMISSION_WORKERS"
 	workerSize, err := env.GetInt(envDecomWorkers, len(pool.sets))
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("invalid workers value err: %v, defaulting to %d", err, len(pool.sets)))
+		decomLogIf(ctx, fmt.Errorf("invalid workers value err: %v, defaulting to %d", err, len(pool.sets)))
 		workerSize = len(pool.sets)
 	}
 
@@ -754,14 +758,33 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 		return err
 	}
 
-	vc, _ := globalBucketVersioningSys.Get(bi.Name)
+	var vc *versioning.Versioning
+	var lc *lifecycle.Lifecycle
+	var lr objectlock.Retention
+	var rcfg *replication.Config
+	if bi.Name != minioMetaBucket {
+		vc, err = globalBucketVersioningSys.Get(bi.Name)
+		if err != nil {
+			return err
+		}
 
-	// Check if the current bucket has a configured lifecycle policy
-	lc, _ := globalLifecycleSys.Get(bi.Name)
+		// Check if the current bucket has a configured lifecycle policy
+		lc, err = globalLifecycleSys.Get(bi.Name)
+		if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bi.Name}) {
+			return err
+		}
 
-	// Check if bucket is object locked.
-	lr, _ := globalBucketObjectLockSys.Get(bi.Name)
-	rcfg, _ := getReplicationConfig(ctx, bi.Name)
+		// Check if bucket is object locked.
+		lr, err = globalBucketObjectLockSys.Get(bi.Name)
+		if err != nil {
+			return err
+		}
+
+		rcfg, err = getReplicationConfig(ctx, bi.Name)
+		if err != nil {
+			return err
+		}
+	}
 
 	for setIdx, set := range pool.sets {
 		set := set
@@ -852,7 +875,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					}
 					stopFn(err)
 					if err != nil {
-						logger.LogIf(ctx, err)
+						decomLogIf(ctx, err)
 						failure = true
 					}
 					z.poolMetaMutex.Lock()
@@ -877,7 +900,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						}); err != nil {
 							stopFn(err)
 							failure = true
-							logger.LogIf(ctx, err)
+							decomLogIf(ctx, err)
 							continue
 						}
 						stopFn(nil)
@@ -906,20 +929,20 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						if bi.Name == minioMetaBucket && strings.Contains(version.Name, dataUsageCacheName) {
 							ignore = true
 							stopFn(err)
-							logger.LogIf(ctx, err)
+							decomLogIf(ctx, err)
 							break
 						}
 					}
 					if err != nil {
 						failure = true
-						logger.LogIf(ctx, err)
+						decomLogIf(ctx, err)
 						stopFn(err)
 						continue
 					}
 					if err = z.decommissionObject(ctx, bi.Name, gr); err != nil {
 						stopFn(err)
 						failure = true
-						logger.LogIf(ctx, err)
+						decomLogIf(ctx, err)
 						continue
 					}
 					stopFn(nil)
@@ -953,13 +976,13 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				stopFn(err)
 				auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err)
 				if err != nil {
-					logger.LogIf(ctx, err)
+					decomLogIf(ctx, err)
 				}
 			}
 			z.poolMetaMutex.Lock()
 			z.poolMeta.TrackCurrentBucketObject(idx, bi.Name, entry.name)
 			ok, err := z.poolMeta.updateAfter(ctx, idx, z.serverPools, 30*time.Second)
-			logger.LogIf(ctx, err)
+			decomLogIf(ctx, err)
 			if ok {
 				globalNotificationSys.ReloadPoolMeta(ctx)
 			}
@@ -987,7 +1010,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				}
 				setN := humanize.Ordinal(setIdx + 1)
 				retryDur := time.Duration(rand.Float64() * float64(5*time.Second))
-				logger.LogOnceIf(ctx, fmt.Errorf("listing objects from %s set failed with %v, retrying in %v", setN, err, retryDur), "decom-listing-failed"+setN)
+				decomLogOnceIf(ctx, fmt.Errorf("listing objects from %s set failed with %v, retrying in %v", setN, err, retryDur), "decom-listing-failed"+setN)
 				time.Sleep(retryDur)
 			}
 		}(setIdx)
@@ -1055,7 +1078,7 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 			z.poolMetaMutex.Lock()
 			if z.poolMeta.BucketDone(idx, bucket) {
 				// remove from pendingBuckets and persist.
-				logger.LogIf(ctx, z.poolMeta.save(ctx, z.serverPools))
+				decomLogIf(ctx, z.poolMeta.save(ctx, z.serverPools))
 			}
 			z.poolMetaMutex.Unlock()
 			continue
@@ -1072,7 +1095,7 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 
 		z.poolMetaMutex.Lock()
 		if z.poolMeta.BucketDone(idx, bucket) {
-			logger.LogIf(ctx, z.poolMeta.save(ctx, z.serverPools))
+			decomLogIf(ctx, z.poolMeta.save(ctx, z.serverPools))
 		}
 		z.poolMetaMutex.Unlock()
 	}
@@ -1088,14 +1111,33 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error
 	pool := z.serverPools[idx]
 	for _, set := range pool.sets {
 		for _, bi := range buckets {
-			vc, _ := globalBucketVersioningSys.Get(bi.Name)
+			var vc *versioning.Versioning
+			var lc *lifecycle.Lifecycle
+			var lr objectlock.Retention
+			var rcfg *replication.Config
+			if bi.Name != minioMetaBucket {
+				vc, err = globalBucketVersioningSys.Get(bi.Name)
+				if err != nil {
+					return err
+				}
 
-			// Check if the current bucket has a configured lifecycle policy
-			lc, _ := globalLifecycleSys.Get(bi.Name)
+				// Check if the current bucket has a configured lifecycle policy
+				lc, err = globalLifecycleSys.Get(bi.Name)
+				if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bi.Name}) {
+					return err
+				}
 
-			// Check if bucket is object locked.
-			lr, _ := globalBucketObjectLockSys.Get(bi.Name)
-			rcfg, _ := getReplicationConfig(ctx, bi.Name)
+				// Check if bucket is object locked.
+				lr, err = globalBucketObjectLockSys.Get(bi.Name)
+				if err != nil {
+					return err
+				}
+
+				rcfg, err = getReplicationConfig(ctx, bi.Name)
+				if err != nil {
+					return err
+				}
+			}
 
 			filterLifecycle := func(bucket, object string, fi FileInfo) bool {
 				if lc == nil {
@@ -1118,7 +1160,7 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error
 			}
 
 			var versionsFound int
-			err := set.listObjectsToDecommission(ctx, bi, func(entry metaCacheEntry) {
+			if err = set.listObjectsToDecommission(ctx, bi, func(entry metaCacheEntry) {
 				if !entry.isObject() {
 					return
 				}
@@ -1146,8 +1188,7 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error
 
 					versionsFound++
 				}
-			})
-			if err != nil {
+			}); err != nil {
 				return err
 			}
 
@@ -1170,8 +1211,8 @@ func (z *erasureServerPools) doDecommissionInRoutine(ctx context.Context, idx in
 	dctx = logger.SetReqInfo(dctx, &logger.ReqInfo{})
 
 	if err := z.decommissionInBackground(dctx, idx); err != nil {
-		logger.LogIf(GlobalContext, err)
-		logger.LogIf(GlobalContext, z.DecommissionFailed(dctx, idx))
+		decomLogIf(GlobalContext, err)
+		decomLogIf(GlobalContext, z.DecommissionFailed(dctx, idx))
 		return
 	}
 
@@ -1181,20 +1222,20 @@ func (z *erasureServerPools) doDecommissionInRoutine(ctx context.Context, idx in
 	z.poolMetaMutex.Unlock()
 
 	if !failed {
-		logger.Event(dctx, "Decommissioning complete for pool '%s', verifying for any pending objects", poolCmdLine)
+		decomLogEvent(dctx, "Decommissioning complete for pool '%s', verifying for any pending objects", poolCmdLine)
 		err := z.checkAfterDecom(dctx, idx)
 		if err != nil {
-			logger.LogIf(ctx, err)
+			decomLogIf(ctx, err)
 			failed = true
 		}
 	}
 
 	if failed {
 		// Decommission failed indicate as such.
-		logger.LogIf(GlobalContext, z.DecommissionFailed(dctx, idx))
+		decomLogIf(GlobalContext, z.DecommissionFailed(dctx, idx))
 	} else {
 		// Complete the decommission..
-		logger.LogIf(GlobalContext, z.CompleteDecommission(dctx, idx))
+		decomLogIf(GlobalContext, z.CompleteDecommission(dctx, idx))
 	}
 }
 
@@ -1404,16 +1445,18 @@ func (z *erasureServerPools) getBucketsToDecommission(ctx context.Context) ([]de
 
 	// Buckets data are dispersed in multiple zones/sets, make
 	// sure to decommission the necessary metadata.
-	decomBuckets = append(decomBuckets, decomBucketInfo{
-		Name:   minioMetaBucket,
-		Prefix: minioConfigPrefix,
-	})
-	decomBuckets = append(decomBuckets, decomBucketInfo{
-		Name:   minioMetaBucket,
-		Prefix: bucketMetaPrefix,
-	})
+	decomMetaBuckets := []decomBucketInfo{
+		{
+			Name:   minioMetaBucket,
+			Prefix: minioConfigPrefix,
+		},
+		{
+			Name:   minioMetaBucket,
+			Prefix: bucketMetaPrefix,
+		},
+	}
 
-	return decomBuckets, nil
+	return append(decomMetaBuckets, decomBuckets...), nil
 }
 
 func (z *erasureServerPools) StartDecommission(ctx context.Context, indices ...int) (err error) {

cmd/erasure-server-pool-rebalance.go

@@ -32,6 +32,10 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/lithammer/shortuuid/v4"
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/hash"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
@@ -146,7 +150,7 @@ func (z *erasureServerPools) updateRebalanceStats(ctx context.Context) error {
 		lock := z.serverPools[0].NewNSLock(minioMetaBucket, rebalMetaName)
 		lkCtx, err := lock.GetLock(ctx, globalOperationTimeout)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("failed to acquire write lock on %s/%s: %w", minioMetaBucket, rebalMetaName, err))
+			rebalanceLogIf(ctx, fmt.Errorf("failed to acquire write lock on %s/%s: %w", minioMetaBucket, rebalMetaName, err))
 			return err
 		}
 		defer lock.Unlock(lkCtx)
@@ -372,7 +376,7 @@ func (z *erasureServerPools) IsPoolRebalancing(poolIndex int) bool {
 }
 
 func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int) (err error) {
-	doneCh := make(chan struct{})
+	doneCh := make(chan error, 1)
 	defer xioutil.SafeClose(doneCh)
 
 	// Save rebalance.bin periodically.
@@ -387,56 +391,59 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
 
 		timer := time.NewTimer(randSleepFor())
 		defer timer.Stop()
-		var rebalDone bool
-		var traceMsg string
+
+		var (
+			quit     bool
+			traceMsg string
+		)
 
 		for {
 			select {
-			case <-doneCh:
-				// rebalance completed for poolIdx
+			case rebalErr := <-doneCh:
+				quit = true
 				now := time.Now()
+				var status rebalStatus
+
+				switch {
+				case errors.Is(rebalErr, context.Canceled):
+					status = rebalStopped
+					traceMsg = fmt.Sprintf("stopped at %s", now)
+				case rebalErr == nil:
+					status = rebalCompleted
+					traceMsg = fmt.Sprintf("completed at %s", now)
+				default:
+					status = rebalFailed
+					traceMsg = fmt.Sprintf("stopped at %s with err: %v", now, rebalErr)
+				}
+
 				z.rebalMu.Lock()
-				z.rebalMeta.PoolStats[poolIdx].Info.Status = rebalCompleted
+				z.rebalMeta.PoolStats[poolIdx].Info.Status = status
 				z.rebalMeta.PoolStats[poolIdx].Info.EndTime = now
 				z.rebalMu.Unlock()
 
-				rebalDone = true
-				traceMsg = fmt.Sprintf("completed at %s", now)
-
-			case <-ctx.Done():
-
-				// rebalance stopped for poolIdx
-				now := time.Now()
-				z.rebalMu.Lock()
-				z.rebalMeta.PoolStats[poolIdx].Info.Status = rebalStopped
-				z.rebalMeta.PoolStats[poolIdx].Info.EndTime = now
-				z.rebalMeta.cancel = nil // remove the already used context.CancelFunc
-				z.rebalMu.Unlock()
-
-				rebalDone = true
-				traceMsg = fmt.Sprintf("stopped at %s", now)
-
 			case <-timer.C:
 				traceMsg = fmt.Sprintf("saved at %s", time.Now())
 			}
 
 			stopFn := globalRebalanceMetrics.log(rebalanceMetricSaveMetadata, poolIdx, traceMsg)
-			err := z.saveRebalanceStats(ctx, poolIdx, rebalSaveStats)
+			err := z.saveRebalanceStats(GlobalContext, poolIdx, rebalSaveStats)
 			stopFn(err)
-			logger.LogIf(ctx, err)
-			timer.Reset(randSleepFor())
+			rebalanceLogIf(GlobalContext, err)
 
-			if rebalDone {
+			if quit {
 				return
 			}
+
+			timer.Reset(randSleepFor())
 		}
 	}()
 
-	logger.Event(ctx, "Pool %d rebalancing is started", poolIdx+1)
+	rebalanceLogEvent(ctx, "Pool %d rebalancing is started", poolIdx+1)
 
 	for {
 		select {
 		case <-ctx.Done():
+			doneCh <- ctx.Err()
 			return
 		default:
 		}
@@ -448,17 +455,20 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
 		}
 
 		stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceBucket, poolIdx, bucket)
-		err = z.rebalanceBucket(ctx, bucket, poolIdx)
-		if err != nil {
+		if err = z.rebalanceBucket(ctx, bucket, poolIdx); err != nil {
 			stopFn(err)
-			logger.LogIf(ctx, err)
+			if errors.Is(err, errServerNotInitialized) || errors.Is(err, errBucketMetadataNotInitialized) {
+				continue
+			}
+			rebalanceLogIf(GlobalContext, err)
+			doneCh <- err
 			return
 		}
 		stopFn(nil)
 		z.bucketRebalanceDone(bucket, poolIdx)
 	}
 
-	logger.Event(ctx, "Pool %d rebalancing is done", poolIdx+1)
+	rebalanceLogEvent(GlobalContext, "Pool %d rebalancing is done", poolIdx+1)
 
 	return err
 }
@@ -521,21 +531,43 @@ func (set *erasureObjects) listObjectsToRebalance(ctx context.Context, bucketNam
 }
 
 // rebalanceBucket rebalances objects under bucket in poolIdx pool
-func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string, poolIdx int) error {
+func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string, poolIdx int) (err error) {
 	ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})
-	vc, _ := globalBucketVersioningSys.Get(bucket)
-	// Check if the current bucket has a configured lifecycle policy
-	lc, _ := globalLifecycleSys.Get(bucket)
-	// Check if bucket is object locked.
-	lr, _ := globalBucketObjectLockSys.Get(bucket)
-	rcfg, _ := getReplicationConfig(ctx, bucket)
+
+	var vc *versioning.Versioning
+	var lc *lifecycle.Lifecycle
+	var lr objectlock.Retention
+	var rcfg *replication.Config
+	if bucket != minioMetaBucket {
+		vc, err = globalBucketVersioningSys.Get(bucket)
+		if err != nil {
+			return err
+		}
+
+		// Check if the current bucket has a configured lifecycle policy
+		lc, err = globalLifecycleSys.Get(bucket)
+		if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) {
+			return err
+		}
+
+		// Check if bucket is object locked.
+		lr, err = globalBucketObjectLockSys.Get(bucket)
+		if err != nil {
+			return err
+		}
+
+		rcfg, err = getReplicationConfig(ctx, bucket)
+		if err != nil {
+			return err
+		}
+	}
 
 	pool := z.serverPools[poolIdx]
 
 	const envRebalanceWorkers = "_MINIO_REBALANCE_WORKERS"
 	workerSize, err := env.GetInt(envRebalanceWorkers, len(pool.sets))
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("invalid workers value err: %v, defaulting to %d", err, len(pool.sets)))
+		rebalanceLogIf(ctx, fmt.Errorf("invalid workers value err: %v, defaulting to %d", err, len(pool.sets)))
 		workerSize = len(pool.sets)
 	}
 
@@ -630,7 +662,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 						})
 					var failure bool
 					if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-						logger.LogIf(ctx, err)
+						rebalanceLogIf(ctx, err)
 						failure = true
 					}
 
@@ -665,14 +697,14 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 					}
 					if err != nil {
 						failure = true
-						logger.LogIf(ctx, err)
+						rebalanceLogIf(ctx, err)
 						stopFn(err)
 						continue
 					}
 
 					if err = z.rebalanceObject(ctx, bucket, gr); err != nil {
 						failure = true
-						logger.LogIf(ctx, err)
+						rebalanceLogIf(ctx, err)
 						stopFn(err)
 						continue
 					}
@@ -706,7 +738,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 				stopFn(err)
 				auditLogRebalance(ctx, "Rebalance:DeleteObject", bucket, entry.name, "", err)
 				if err != nil {
-					logger.LogIf(ctx, err)
+					rebalanceLogIf(ctx, err)
 				}
 			}
 		}
@@ -724,7 +756,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 				return
 			}
 			setN := humanize.Ordinal(setIdx + 1)
-			logger.LogOnceIf(ctx, fmt.Errorf("listing objects from %s set failed with %v", setN, err), "rebalance-listing-failed"+setN)
+			rebalanceLogIf(ctx, fmt.Errorf("listing objects from %s set failed with %v", setN, err), "rebalance-listing-failed"+setN)
 		}(setIdx)
 	}
 
@@ -743,7 +775,7 @@ func (z *erasureServerPools) saveRebalanceStats(ctx context.Context, poolIdx int
 	lock := z.serverPools[0].NewNSLock(minioMetaBucket, rebalMetaName)
 	lkCtx, err := lock.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("failed to acquire write lock on %s/%s: %w", minioMetaBucket, rebalMetaName, err))
+		rebalanceLogIf(ctx, fmt.Errorf("failed to acquire write lock on %s/%s: %w", minioMetaBucket, rebalMetaName, err))
 		return err
 	}
 	defer lock.Unlock(lkCtx)

cmd/erasure-server-pool.go

@@ -103,8 +103,9 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 
 	// Initialize byte pool once for all sets, bpool size is set to
 	// setCount * setDriveCount with each memory upto blockSizeV2.
-	globalBytePoolCap = bpool.NewBytePoolCap(n, blockSizeV2, blockSizeV2*2)
-	globalBytePoolCap.Populate()
+	buffers := bpool.NewBytePoolCap(n, blockSizeV2, blockSizeV2*2)
+	buffers.Populate()
+	globalBytePoolCap.Store(buffers)
 
 	var localDrives []StorageAPI
 	local := endpointServerPools.FirstLocal()
@@ -207,7 +208,7 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 				logger.Fatal(err, "Unable to initialize backend")
 			}
 			retry := time.Duration(r.Float64() * float64(5*time.Second))
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize backend: %w, retrying in %s", err, retry))
+			storageLogIf(ctx, fmt.Errorf("Unable to initialize backend: %w, retrying in %s", err, retry))
 			time.Sleep(retry)
 			attempt++
 			continue
@@ -376,7 +377,7 @@ func (z *erasureServerPools) getAvailablePoolIdx(ctx context.Context, bucket, ob
 		}
 	}
 	// Should not happen, but print values just in case.
-	logger.LogIf(ctx, fmt.Errorf("reached end of serverPools (total: %v, atTotal: %v, choose: %v)", total, atTotal, choose))
+	storageLogIf(ctx, fmt.Errorf("reached end of serverPools (total: %v, atTotal: %v, choose: %v)", total, atTotal, choose))
 	return -1
 }
 
@@ -457,7 +458,13 @@ type PoolObjInfo struct {
 	Err     error
 }
 
-func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bucket, object string, opts ObjectOptions) (PoolObjInfo, error) {
+type poolErrs struct {
+	Index int
+	Err   error
+}
+
+func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bucket, object string, opts ObjectOptions) (PoolObjInfo, []poolErrs, error) {
+	var noReadQuorumPools []poolErrs
 	poolObjInfos := make([]PoolObjInfo, len(z.serverPools))
 	poolOpts := make([]ObjectOptions, len(z.serverPools))
 	for i := range z.serverPools {
@@ -507,8 +514,9 @@ func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bu
 		}
 		if pinfo.Err == nil {
 			// found a pool
-			return pinfo, nil
+			return pinfo, z.poolsWithObject(poolObjInfos, opts), nil
 		}
+
 		if isErrReadQuorum(pinfo.Err) && !opts.MetadataChg {
 			// read quorum is returned when the object is visibly
 			// present but its unreadable, we simply ask the writes to
@@ -517,30 +525,49 @@ func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bu
 			// with enough disks online but sufficiently inconsistent to
 			// break parity threshold, allow them to be overwritten
 			// or allow new versions to be added.
-			return pinfo, nil
+
+			return pinfo, z.poolsWithObject(poolObjInfos, opts), nil
 		}
 		defPool = pinfo
 		if !isErrObjectNotFound(pinfo.Err) {
-			return pinfo, pinfo.Err
+			return pinfo, noReadQuorumPools, pinfo.Err
 		}
 
 		// No object exists or its a delete marker,
 		// check objInfo to confirm.
 		if pinfo.ObjInfo.DeleteMarker && pinfo.ObjInfo.Name != "" {
-			return pinfo, nil
+			return pinfo, noReadQuorumPools, nil
 		}
 	}
 	if opts.ReplicationRequest && opts.DeleteMarker && defPool.Index >= 0 {
 		// If the request is a delete marker replication request, return a default pool
 		// in cases where the object does not exist.
 		// This is to ensure that the delete marker is replicated to the destination.
-		return defPool, nil
+		return defPool, noReadQuorumPools, nil
 	}
-	return PoolObjInfo{}, toObjectErr(errFileNotFound, bucket, object)
+	return PoolObjInfo{}, noReadQuorumPools, toObjectErr(errFileNotFound, bucket, object)
+}
+
+// return all pools with read quorum error or no error for an object with given opts.Note that this array is
+// returned in the order of recency of object ModTime.
+func (z *erasureServerPools) poolsWithObject(pools []PoolObjInfo, opts ObjectOptions) (errs []poolErrs) {
+	for _, pool := range pools {
+		if opts.SkipDecommissioned && z.IsSuspended(pool.Index) {
+			continue
+		}
+		// Skip object if it's from pools participating in a rebalance operation.
+		if opts.SkipRebalancing && z.IsPoolRebalancing(pool.Index) {
+			continue
+		}
+		if isErrReadQuorum(pool.Err) || pool.Err == nil {
+			errs = append(errs, poolErrs{Err: pool.Err, Index: pool.Index})
+		}
+	}
+	return errs
 }
 
 func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, bucket, object string, opts ObjectOptions) (idx int, err error) {
-	pinfo, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, opts)
+	pinfo, _, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, opts)
 	if err != nil {
 		return -1, err
 	}
@@ -610,7 +637,7 @@ func (z *erasureServerPools) Shutdown(ctx context.Context) error {
 
 	for _, err := range g.Wait() {
 		if err != nil {
-			logger.LogIf(ctx, err)
+			storageLogIf(ctx, err)
 		}
 		// let's the rest shutdown
 	}
@@ -714,7 +741,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataU
 				// Start scanner. Blocks until done.
 				err := erObj.nsScanner(ctx, allBuckets, wantCycle, updates, healScanMode)
 				if err != nil {
-					logger.LogIf(ctx, err)
+					scannerLogIf(ctx, err)
 					mu.Lock()
 					if firstErr == nil {
 						firstErr = err
@@ -1081,7 +1108,8 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 
 	gopts := opts
 	gopts.NoLock = true
-	pinfo, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, gopts)
+
+	pinfo, noReadQuorumPools, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, gopts)
 	if err != nil {
 		if _, ok := err.(InsufficientReadQuorum); ok {
 			return objInfo, InsufficientWriteQuorum{}
@@ -1095,11 +1123,44 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 		return pinfo.ObjInfo, nil
 	}
 
+	// Delete concurrently in all server pools with read quorum error for unversioned objects.
+	if len(noReadQuorumPools) > 0 && !opts.Versioned && !opts.VersionSuspended {
+		return z.deleteObjectFromAllPools(ctx, bucket, object, opts, noReadQuorumPools)
+	}
 	objInfo, err = z.serverPools[pinfo.Index].DeleteObject(ctx, bucket, object, opts)
 	objInfo.Name = decodeDirObject(object)
 	return objInfo, err
 }
 
+func (z *erasureServerPools) deleteObjectFromAllPools(ctx context.Context, bucket string, object string, opts ObjectOptions, poolIndices []poolErrs) (objInfo ObjectInfo, err error) {
+	derrs := make([]error, len(poolIndices))
+	dobjects := make([]ObjectInfo, len(poolIndices))
+
+	// Delete concurrently in all server pools that reported no error or read quorum error
+	// where the read quorum issue is from metadata inconsistency.
+	var wg sync.WaitGroup
+	for idx, pe := range poolIndices {
+		if v, ok := pe.Err.(InsufficientReadQuorum); ok && v.Type != RQInconsistentMeta {
+			derrs[idx] = InsufficientWriteQuorum{}
+			continue
+		}
+		wg.Add(1)
+		pool := z.serverPools[pe.Index]
+		go func(idx int, pool *erasureSets) {
+			defer wg.Done()
+			dobjects[idx], derrs[idx] = pool.DeleteObject(ctx, bucket, object, opts)
+		}(idx, pool)
+	}
+	wg.Wait()
+
+	// the poolIndices array is pre-sorted in order of latest ModTime, we care only about pool with latest object though
+	// the delete call tries to clean up other pools during DeleteObject call.
+	objInfo = dobjects[0]
+	objInfo.Name = decodeDirObject(object)
+	err = derrs[0]
+	return objInfo, err
+}
+
 func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
 	derrs := make([]error, len(objects))
 	dobjects := make([]DeletedObject, len(objects))
@@ -1141,7 +1202,7 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 		j := j
 		obj := obj
 		eg.Go(func() error {
-			pinfo, err := z.getPoolInfoExistingWithOpts(ctx, bucket, obj.ObjectName, ObjectOptions{
+			pinfo, _, err := z.getPoolInfoExistingWithOpts(ctx, bucket, obj.ObjectName, ObjectOptions{
 				NoLock: true,
 			})
 			if err != nil {
@@ -1329,7 +1390,7 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 	merged, err := z.listPath(ctx, &opts)
 	if err != nil && err != io.EOF {
 		if !isErrBucketNotFound(err) {
-			logger.LogOnceIf(ctx, err, "erasure-list-objects-path-"+bucket)
+			storageLogOnceIf(ctx, err, "erasure-list-objects-path-"+bucket)
 		}
 		return loi, toObjectErr(err, bucket)
 	}
@@ -1523,7 +1584,7 @@ func (z *erasureServerPools) listObjectsGeneric(ctx context.Context, bucket, pre
 	merged, err := z.listPath(ctx, &opts)
 	if err != nil && err != io.EOF {
 		if !isErrBucketNotFound(err) {
-			logger.LogOnceIf(ctx, err, "erasure-list-objects-path-"+bucket)
+			storageLogOnceIf(ctx, err, "erasure-list-objects-path-"+bucket)
 		}
 		return loi, toObjectErr(err, bucket)
 	}
@@ -1945,7 +2006,7 @@ func (z *erasureServerPools) HealFormat(ctx context.Context, dryRun bool) (madmi
 	for _, pool := range z.serverPools {
 		result, err := pool.HealFormat(ctx, dryRun)
 		if err != nil && !errors.Is(err, errNoHealRequired) {
-			logger.LogOnceIf(ctx, err, "erasure-heal-format")
+			healingLogOnceIf(ctx, err, "erasure-heal-format")
 			continue
 		}
 		// Count errNoHealRequired across all serverPools,
@@ -2136,7 +2197,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 					}
 
 					if err := listPathRaw(ctx, lopts); err != nil {
-						logger.LogIf(ctx, fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts))
+						storageLogIf(ctx, fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts))
 						cancel()
 						return
 					}
@@ -2181,7 +2242,7 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 		if opts.Remove && !opts.DryRun {
 			err := z.CheckAbandonedParts(ctx, bucket, entry.name, opts)
 			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", bucket, entry.name, err))
+				healingLogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", bucket, entry.name, err))
 			}
 		}
 		for _, version := range fivs.Versions {
@@ -2209,7 +2270,7 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 			go func(idx int, set *erasureObjects) {
 				defer wg.Done()
 
-				errs[idx] = set.listAndHeal(bucket, prefix, opts.ScanMode, healEntry)
+				errs[idx] = set.listAndHeal(ctx, bucket, prefix, opts.ScanMode, healEntry)
 			}(idx, set)
 		}
 		wg.Wait()
@@ -2385,7 +2446,7 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 	// Check if disks are healing on in-case of VMware vsphere deployments.
 	if opts.Maintenance && opts.DeploymentType == vmware {
 		if drivesHealing > 0 {
-			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Total drives to be healed %d", drivesHealing))
+			healingLogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Total drives to be healed %d", drivesHealing))
 		}
 	}
 
@@ -2445,15 +2506,15 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 
 			healthy := erasureSetUpCount[poolIdx][setIdx].online >= poolWriteQuorums[poolIdx]
 			if !healthy {
-				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
+				storageLogIf(logger.SetReqInfo(ctx, reqInfo),
 					fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
-						poolIdx, setIdx, poolWriteQuorums[poolIdx]))
+						poolIdx, setIdx, poolWriteQuorums[poolIdx]), logger.FatalKind)
 			}
 			result.Healthy = result.Healthy && healthy
 
 			healthyRead := erasureSetUpCount[poolIdx][setIdx].online >= poolReadQuorums[poolIdx]
 			if !healthyRead {
-				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
+				storageLogIf(logger.SetReqInfo(ctx, reqInfo),
 					fmt.Errorf("Read quorum may be lost on pool: %d, set: %d, expected read quorum: %d",
 						poolIdx, setIdx, poolReadQuorums[poolIdx]))
 			}

cmd/erasure-sets.go

@@ -31,7 +31,6 @@ import (
 	"time"
 
 	"github.com/dchest/siphash"
-	"github.com/dustin/go-humanize"
 	"github.com/google/uuid"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -115,26 +114,26 @@ func (s *erasureSets) getDiskMap() map[Endpoint]StorageAPI {
 
 // Initializes a new StorageAPI from the endpoint argument, returns
 // StorageAPI and also `format` which exists on the disk.
-func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, []byte, error) {
+func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 	disk, err := newStorageAPI(endpoint, storageOpts{
 		cleanUp:     false,
 		healthCheck: false,
 	})
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, err
 	}
 
-	format, formatData, err := loadFormatErasureWithData(disk, false)
+	format, err := loadFormatErasure(disk, false)
 	if err != nil {
 		if errors.Is(err, errUnformattedDisk) {
 			info, derr := disk.DiskInfo(context.TODO(), DiskInfoOptions{})
 			if derr != nil && info.RootDisk {
 				disk.Close()
-				return nil, nil, nil, fmt.Errorf("Drive: %s is a root drive", disk)
+				return nil, nil, fmt.Errorf("Drive: %s is a root drive", disk)
 			}
 		}
 		disk.Close()
-		return nil, nil, nil, fmt.Errorf("Drive: %s returned %w", disk, err) // make sure to '%w' to wrap the error
+		return nil, nil, fmt.Errorf("Drive: %s returned %w", disk, err) // make sure to '%w' to wrap the error
 	}
 
 	disk.Close()
@@ -143,10 +142,10 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, []byte, e
 		healthCheck: true,
 	})
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, err
 	}
 
-	return disk, format, formatData, nil
+	return disk, format, nil
 }
 
 // findDiskIndex - returns the i,j'th position of the input `diskID` against the reference
@@ -227,7 +226,7 @@ func (s *erasureSets) connectDisks() {
 		wg.Add(1)
 		go func(endpoint Endpoint) {
 			defer wg.Done()
-			disk, format, formatData, err := connectEndpoint(endpoint)
+			disk, format, err := connectEndpoint(endpoint)
 			if err != nil {
 				if endpoint.IsLocal && errors.Is(err, errUnformattedDisk) {
 					globalBackgroundHealState.pushHealLocalDisks(endpoint)
@@ -261,8 +260,6 @@ func (s *erasureSets) connectDisks() {
 			}
 
 			disk.SetDiskID(format.Erasure.This)
-			disk.SetDiskLoc(s.poolIndex, setIndex, diskIndex)
-			disk.SetFormatData(formatData)
 			s.erasureDisks[setIndex][diskIndex] = disk
 
 			if disk.IsLocal() {
@@ -448,27 +445,17 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 					diskID, err := disk.GetDiskID()
 					if err != nil {
 						if !errors.Is(err, errUnformattedDisk) {
-							logger.LogIf(ctx, err)
+							bootLogIf(ctx, err)
 						}
 						return
 					}
 					if diskID == "" {
 						return
 					}
-					m, n, err := findDiskIndexByDiskID(format, diskID)
-					if err != nil {
-						logger.LogIf(ctx, err)
-						return
-					}
-					if m != i || n != j {
-						logger.LogIf(ctx, fmt.Errorf("Detected unexpected drive ordering refusing to use the drive - poolID: %s, found drive mounted at (set=%s, drive=%s) expected mount at (set=%s, drive=%s): %s(%s)", humanize.Ordinal(poolIdx+1), humanize.Ordinal(m+1), humanize.Ordinal(n+1), humanize.Ordinal(i+1), humanize.Ordinal(j+1), disk, diskID))
-						s.erasureDisks[i][j] = &unrecognizedDisk{storage: disk}
-						return
-					}
-					disk.SetDiskLoc(s.poolIndex, m, n)
-					s.erasureDisks[m][n] = disk
+					s.erasureDisks[i][j] = disk
 				}(disk, i, j)
 			}
+
 			innerWg.Wait()
 
 			// Initialize erasure objects for a given set.
@@ -1083,7 +1070,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 
 	if !reflect.DeepEqual(s.format, refFormat) {
 		// Format is corrupted and unrecognized by the running instance.
-		logger.LogIf(ctx, fmt.Errorf("Unable to heal the newly replaced drives due to format.json inconsistencies, please engage MinIO support for further assistance: %w",
+		healingLogIf(ctx, fmt.Errorf("Unable to heal the newly replaced drives due to format.json inconsistencies, please engage MinIO support for further assistance: %w",
 			errCorruptedFormat))
 		return res, errCorruptedFormat
 	}
@@ -1112,7 +1099,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 				continue
 			}
 			if err := saveFormatErasure(storageDisks[index], format, formatOpID); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Drive %s failed to write updated 'format.json': %v", storageDisks[index], err))
+				healingLogIf(ctx, fmt.Errorf("Drive %s failed to write updated 'format.json': %v", storageDisks[index], err))
 				storageDisks[index].Close()
 				tmpNewFormats[index] = nil // this disk failed to write new format
 			}
@@ -1127,7 +1114,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 
 			m, n, err := findDiskIndexByDiskID(refFormat, format.Erasure.This)
 			if err != nil {
-				logger.LogIf(ctx, err)
+				healingLogIf(ctx, err)
 				continue
 			}
 
@@ -1137,8 +1124,6 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 
 			if disk := storageDisks[index]; disk != nil {
 				if disk.IsLocal() {
-					disk.SetDiskLoc(s.poolIndex, m, n)
-
 					xldisk, ok := disk.(*xlStorageDiskIDCheck)
 					if ok {
 						_, commonDeletes := calcCommonWritesDeletes(currentDisksInfo[m], (s.setDriveCount+1)/2)
@@ -1155,7 +1140,6 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 					if err != nil {
 						continue
 					}
-					disk.SetDiskLoc(s.poolIndex, m, n)
 				}
 
 				s.erasureDisks[m][n] = disk

cmd/erasure.go

@@ -31,7 +31,6 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/dsync"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/sync/errgroup"
 )
 
@@ -273,11 +272,11 @@ func (er erasureObjects) LocalStorageInfo(ctx context.Context, metrics bool) Sto
 }
 
 // getOnlineDisksWithHealingAndInfo - returns online disks and overall healing status.
-// Disks are randomly ordered, but in the following groups:
+// Disks are ordered in the following groups:
 // - Non-scanning disks
 // - Non-healing disks
 // - Healing disks (if inclHealing is true)
-func (er erasureObjects) getOnlineDisksWithHealingAndInfo(inclHealing bool) (newDisks []StorageAPI, newInfos []DiskInfo, healing bool) {
+func (er erasureObjects) getOnlineDisksWithHealingAndInfo(inclHealing bool) (newDisks []StorageAPI, newInfos []DiskInfo, healing int) {
 	var wg sync.WaitGroup
 	disks := er.getDisks()
 	infos := make([]DiskInfo, len(disks))
@@ -316,7 +315,7 @@ func (er erasureObjects) getOnlineDisksWithHealingAndInfo(inclHealing bool) (new
 			continue
 		}
 		if info.Healing {
-			healing = true
+			healing++
 			if inclHealing {
 				healingDisks = append(healingDisks, disks[i])
 				healingInfos = append(healingInfos, infos[i])
@@ -344,32 +343,32 @@ func (er erasureObjects) getOnlineDisksWithHealingAndInfo(inclHealing bool) (new
 	return newDisks, newInfos, healing
 }
 
-func (er erasureObjects) getOnlineDisksWithHealing(inclHealing bool) (newDisks []StorageAPI, healing bool) {
-	newDisks, _, healing = er.getOnlineDisksWithHealingAndInfo(inclHealing)
-	return
+func (er erasureObjects) getOnlineDisksWithHealing(inclHealing bool) ([]StorageAPI, bool) {
+	newDisks, _, healing := er.getOnlineDisksWithHealingAndInfo(inclHealing)
+	return newDisks, healing > 0
 }
 
 // Clean-up previously deleted objects. from .minio.sys/tmp/.trash/
 func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) {
-	// run multiple cleanup's local to this server.
 	var wg sync.WaitGroup
 	for _, disk := range er.getLocalDisks() {
-		if disk != nil {
-			wg.Add(1)
-			go func(disk StorageAPI) {
-				defer wg.Done()
-				diskPath := disk.Endpoint().Path
-				readDirFn(pathJoin(diskPath, minioMetaTmpDeletedBucket), func(ddir string, typ os.FileMode) error {
-					w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
-					return w.Run(func() error {
-						wait := deletedCleanupSleeper.Timer(ctx)
-						removeAll(pathJoin(diskPath, minioMetaTmpDeletedBucket, ddir))
-						wait()
-						return nil
-					})
-				})
-			}(disk)
+		if disk == nil {
+			continue
 		}
+		wg.Add(1)
+		go func(disk StorageAPI) {
+			defer wg.Done()
+			drivePath := disk.Endpoint().Path
+			readDirFn(pathJoin(drivePath, minioMetaTmpDeletedBucket), func(ddir string, typ os.FileMode) error {
+				w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
+				return w.Run(func() error {
+					wait := deleteCleanupSleeper.Timer(ctx)
+					removeAll(pathJoin(drivePath, minioMetaTmpDeletedBucket, ddir))
+					wait()
+					return nil
+				})
+			})
+		}(disk)
 	}
 	wg.Wait()
 }
@@ -384,7 +383,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 	// Collect disks we can use.
 	disks, healing := er.getOnlineDisksWithHealing(false)
 	if len(disks) == 0 {
-		logger.LogIf(ctx, errors.New("data-scanner: all drives are offline or being healed, skipping scanner cycle"))
+		scannerLogIf(ctx, errors.New("data-scanner: all drives are offline or being healed, skipping scanner cycle"))
 		return nil
 	}
 
@@ -449,7 +448,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				if cache.Info.LastUpdate.Equal(lastSave) {
 					continue
 				}
-				logger.LogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-cache-update")
+				scannerLogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-cache-update")
 				updates <- cache.clone()
 
 				lastSave = cache.Info.LastUpdate
@@ -458,7 +457,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 					// Save final state...
 					cache.Info.NextCycle = wantCycle
 					cache.Info.LastUpdate = time.Now()
-					logger.LogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-channel-closed")
+					scannerLogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-channel-closed")
 					updates <- cache.clone()
 					return
 				}
@@ -494,7 +493,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				// Load cache for bucket
 				cacheName := pathJoin(bucket.Name, dataUsageCacheName)
 				cache := dataUsageCache{}
-				logger.LogIf(ctx, cache.load(ctx, er, cacheName))
+				scannerLogIf(ctx, cache.load(ctx, er, cacheName))
 				if cache.Info.Name == "" {
 					cache.Info.Name = bucket.Name
 				}
@@ -530,9 +529,9 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				cache, err = disk.NSScanner(ctx, cache, updates, healScanMode, nil)
 				if err != nil {
 					if !cache.Info.LastUpdate.IsZero() && cache.Info.LastUpdate.After(before) {
-						logger.LogIf(ctx, cache.save(ctx, er, cacheName))
+						scannerLogIf(ctx, cache.save(ctx, er, cacheName))
 					} else {
-						logger.LogIf(ctx, err)
+						scannerLogIf(ctx, err)
 					}
 					// This ensures that we don't close
 					// bucketResults channel while the
@@ -562,7 +561,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				}
 
 				// Save cache
-				logger.LogIf(ctx, cache.save(ctx, er, cacheName))
+				scannerLogIf(ctx, cache.save(ctx, er, cacheName))
 			}
 		}(i)
 	}

cmd/etcd.go

@@ -22,7 +22,6 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/minio/minio/internal/logger"
 	etcd "go.etcd.io/etcd/client/v3"
 )
 
@@ -48,7 +47,7 @@ func saveKeyEtcdWithTTL(ctx context.Context, client *etcd.Client, key string, da
 		return etcdErrToErr(err, client.Endpoints())
 	}
 	_, err = client.Put(timeoutCtx, key, string(data), etcd.WithLease(lease.ID))
-	logger.LogIf(ctx, err)
+	etcdLogIf(ctx, err)
 	return etcdErrToErr(err, client.Endpoints())
 }
 
@@ -59,7 +58,7 @@ func saveKeyEtcd(ctx context.Context, client *etcd.Client, key string, data []by
 		return saveKeyEtcdWithTTL(ctx, client, key, data, opts[0].ttl)
 	}
 	_, err := client.Put(timeoutCtx, key, string(data))
-	logger.LogIf(ctx, err)
+	etcdLogIf(ctx, err)
 	return etcdErrToErr(err, client.Endpoints())
 }
 
@@ -68,7 +67,7 @@ func deleteKeyEtcd(ctx context.Context, client *etcd.Client, key string) error {
 	defer cancel()
 
 	_, err := client.Delete(timeoutCtx, key)
-	logger.LogIf(ctx, err)
+	etcdLogIf(ctx, err)
 	return etcdErrToErr(err, client.Endpoints())
 }
 
@@ -77,7 +76,7 @@ func readKeyEtcd(ctx context.Context, client *etcd.Client, key string) ([]byte,
 	defer cancel()
 	resp, err := client.Get(timeoutCtx, key)
 	if err != nil {
-		logger.LogOnceIf(ctx, err, "etcd-retrieve-keys")
+		etcdLogOnceIf(ctx, err, "etcd-retrieve-keys")
 		return nil, etcdErrToErr(err, client.Endpoints())
 	}
 	if resp.Count == 0 {

cmd/event-notification.go

@@ -28,7 +28,6 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/pubsub"
 	"github.com/minio/pkg/v2/policy"
 )
@@ -83,7 +82,7 @@ func (evnot *EventNotifier) set(bucket BucketInfo, meta BucketMetadata) {
 	config.SetRegion(globalSite.Region)
 	if err := config.Validate(globalSite.Region, globalEventNotifier.targetList); err != nil {
 		if _, ok := err.(*event.ErrARNNotFound); !ok {
-			logger.LogIf(GlobalContext, err)
+			internalLogIf(GlobalContext, err)
 		}
 	}
 	evnot.AddRulesMap(bucket.Name, config.ToRulesMap())

cmd/format-erasure.go

@@ -24,11 +24,9 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
-	"reflect"
 	"sync"
 
 	"github.com/dustin/go-humanize"
-	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/storageclass"
@@ -278,7 +276,7 @@ func formatErasureMigrateV2ToV3(data []byte, export, version string) ([]byte, er
 	tmpOld := pathJoin(export, minioMetaTmpDeletedBucket, mustGetUUID())
 	if err := renameAll(pathJoin(export, minioMetaMultipartBucket),
 		tmpOld, export); err != nil && err != errFileNotFound {
-		logger.LogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate",
+		bootLogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate",
 			pathJoin(export, minioMetaMultipartBucket),
 			tmpOld,
 			osErrToFileErr(err)))
@@ -331,7 +329,7 @@ func loadFormatErasureAll(storageDisks []StorageAPI, heal bool) ([]*formatErasur
 			if storageDisks[index] == nil {
 				return errDiskNotFound
 			}
-			format, formatData, err := loadFormatErasureWithData(storageDisks[index], heal)
+			format, err := loadFormatErasure(storageDisks[index], heal)
 			if err != nil {
 				return err
 			}
@@ -340,7 +338,6 @@ func loadFormatErasureAll(storageDisks []StorageAPI, heal bool) ([]*formatErasur
 				// If no healing required, make the disks valid and
 				// online.
 				storageDisks[index].SetDiskID(format.Erasure.This)
-				storageDisks[index].SetFormatData(formatData)
 			}
 			return nil
 		}, index)
@@ -380,7 +377,6 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, healID string)
 	}
 
 	disk.SetDiskID(format.Erasure.This)
-	disk.SetFormatData(formatData)
 	if healID != "" {
 		ctx := context.Background()
 		ht := initHealingTracker(disk, healID)
@@ -389,56 +385,32 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, healID string)
 	return nil
 }
 
-// loadFormatErasureWithData - loads format.json from disk.
-func loadFormatErasureWithData(disk StorageAPI, heal bool) (format *formatErasureV3, data []byte, err error) {
-	data, err = disk.ReadAll(context.TODO(), minioMetaBucket, formatConfigFile)
+// loadFormatErasure - loads format.json from disk.
+func loadFormatErasure(disk StorageAPI, heal bool) (format *formatErasureV3, err error) {
+	data, err := disk.ReadAll(context.TODO(), minioMetaBucket, formatConfigFile)
 	if err != nil {
 		// 'file not found' and 'volume not found' as
 		// same. 'volume not found' usually means its a fresh disk.
 		if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) {
-			return nil, nil, errUnformattedDisk
-		}
-		return nil, nil, err
-	}
-
-	// Try to decode format json into formatConfigV1 struct.
-	format = &formatErasureV3{}
-	if err = json.Unmarshal(data, format); err != nil {
-		return nil, nil, err
-	}
-
-	if heal {
-		info, err := disk.DiskInfo(context.Background(), DiskInfoOptions{NoOp: heal})
-		if err != nil {
-			return nil, nil, err
-		}
-		format.Info = info
-	}
-
-	// Success.
-	return format, data, nil
-}
-
-// loadFormatErasure - loads format.json from disk.
-func loadFormatErasure(disk StorageAPI) (format *formatErasureV3, err error) {
-	buf, err := disk.ReadAll(context.TODO(), minioMetaBucket, formatConfigFile)
-	if err != nil {
-		// 'file not found' and 'volume not found' as
-		// same. 'volume not found' usually means its a fresh disk.
-		if err == errFileNotFound || err == errVolumeNotFound {
 			return nil, errUnformattedDisk
 		}
 		return nil, err
 	}
 
-	json := jsoniter.ConfigCompatibleWithStandardLibrary
-
 	// Try to decode format json into formatConfigV1 struct.
 	format = &formatErasureV3{}
-	if err = json.Unmarshal(buf, format); err != nil {
+	if err = json.Unmarshal(data, format); err != nil {
 		return nil, err
 	}
 
+	if heal {
+		info, err := disk.DiskInfo(context.Background(), DiskInfoOptions{NoOp: heal})
+		if err != nil {
+			return nil, err
+		}
+		format.Info = info
+	}
+
 	// Success.
 	return format, nil
 }
@@ -481,110 +453,6 @@ func checkFormatErasureValues(formats []*formatErasureV3, disks []StorageAPI, se
 	return nil
 }
 
-// Get Deployment ID for the Erasure sets from format.json.
-// This need not be in quorum. Even if one of the format.json
-// file has this value, we assume it is valid.
-// If more than one format.json's have different id, it is considered a corrupt
-// backend format.
-func formatErasureGetDeploymentID(refFormat *formatErasureV3, formats []*formatErasureV3) (string, error) {
-	var deploymentID string
-	for _, format := range formats {
-		if format == nil || format.ID == "" {
-			continue
-		}
-		if reflect.DeepEqual(format.Erasure.Sets, refFormat.Erasure.Sets) {
-			// Found an ID in one of the format.json file
-			// Set deploymentID for the first time.
-			if deploymentID == "" {
-				deploymentID = format.ID
-			} else if deploymentID != format.ID {
-				// DeploymentID found earlier doesn't match with the
-				// current format.json's ID.
-				return "", fmt.Errorf("Deployment IDs do not match expected %s, got %s: %w",
-					deploymentID, format.ID, errCorruptedFormat)
-			}
-		}
-	}
-	return deploymentID, nil
-}
-
-// formatErasureFixDeploymentID - Add deployment id if it is not present.
-func formatErasureFixDeploymentID(endpoints Endpoints, storageDisks []StorageAPI, refFormat *formatErasureV3, formats []*formatErasureV3) (err error) {
-	for index := range formats {
-		// If the Erasure sets do not match, set those formats to nil,
-		// We do not have to update the ID on those format.json file.
-		if formats[index] != nil && !reflect.DeepEqual(formats[index].Erasure.Sets, refFormat.Erasure.Sets) {
-			formats[index] = nil
-		}
-	}
-
-	refFormat.ID, err = formatErasureGetDeploymentID(refFormat, formats)
-	if err != nil {
-		return err
-	}
-
-	// If ID is set, then some other node got the lock
-	// before this node could and generated an ID
-	// for the deployment. No need to generate one.
-	if refFormat.ID != "" {
-		return nil
-	}
-
-	// ID is generated for the first time,
-	// We set the ID in all the formats and update.
-	refFormat.ID = mustGetUUID()
-	for _, format := range formats {
-		if format != nil {
-			format.ID = refFormat.ID
-		}
-	}
-	// Deployment ID needs to be set on all the disks.
-	// Save `format.json` across all disks.
-	return saveFormatErasureAll(GlobalContext, storageDisks, formats)
-}
-
-// Update only the valid local disks which have not been updated before.
-func formatErasureFixLocalDeploymentID(endpoints Endpoints, storageDisks []StorageAPI, refFormat *formatErasureV3) error {
-	// If this server was down when the deploymentID was updated
-	// then we make sure that we update the local disks with the deploymentID.
-
-	// Initialize errs to collect errors inside go-routine.
-	g := errgroup.WithNErrs(len(storageDisks))
-
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if endpoints[index].IsLocal && storageDisks[index] != nil && storageDisks[index].IsOnline() {
-				format, err := loadFormatErasure(storageDisks[index])
-				if err != nil {
-					// Disk can be offline etc.
-					// ignore the errors seen here.
-					return nil
-				}
-				if format.ID != "" {
-					return nil
-				}
-				if !reflect.DeepEqual(format.Erasure.Sets, refFormat.Erasure.Sets) {
-					return nil
-				}
-				format.ID = refFormat.ID
-				// Heal the drive if we fixed its deployment ID.
-				if err := saveFormatErasure(storageDisks[index], format, mustGetUUID()); err != nil {
-					logger.LogIf(GlobalContext, err)
-					return fmt.Errorf("Unable to save format.json, %w", err)
-				}
-			}
-			return nil
-		}, index)
-	}
-	for _, err := range g.Wait() {
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // Get backend Erasure format in quorum `format.json`.
 func getFormatErasureInQuorum(formats []*formatErasureV3) (*formatErasureV3, error) {
 	formatCountMap := make(map[int]int, len(formats))

cmd/format-erasure_test.go

@@ -21,7 +21,6 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
-	"errors"
 	"os"
 	"reflect"
 	"testing"
@@ -431,62 +430,6 @@ func BenchmarkGetFormatErasureInQuorum(b *testing.B) {
 	}
 }
 
-// Tests formatErasureGetDeploymentID()
-func TestGetErasureID(t *testing.T) {
-	setCount := 2
-	setDriveCount := 8
-
-	format := newFormatErasureV3(setCount, setDriveCount)
-	format.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1
-	formats := make([]*formatErasureV3, 16)
-
-	for i := 0; i < setCount; i++ {
-		for j := 0; j < setDriveCount; j++ {
-			newFormat := format.Clone()
-			newFormat.Erasure.This = format.Erasure.Sets[i][j]
-			formats[i*setDriveCount+j] = newFormat
-		}
-	}
-
-	// Return a format from list of formats in quorum.
-	quorumFormat, err := getFormatErasureInQuorum(formats)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Check if the reference format and input formats are same.
-	var id string
-	if id, err = formatErasureGetDeploymentID(quorumFormat, formats); err != nil {
-		t.Fatal(err)
-	}
-
-	if id == "" {
-		t.Fatal("ID cannot be empty.")
-	}
-
-	formats[0] = nil
-	if id, err = formatErasureGetDeploymentID(quorumFormat, formats); err != nil {
-		t.Fatal(err)
-	}
-	if id == "" {
-		t.Fatal("ID cannot be empty.")
-	}
-
-	formats[1].Erasure.Sets[0][0] = "bad-uuid"
-	if id, err = formatErasureGetDeploymentID(quorumFormat, formats); err != nil {
-		t.Fatal(err)
-	}
-
-	if id == "" {
-		t.Fatal("ID cannot be empty.")
-	}
-
-	formats[2].ID = "bad-id"
-	if _, err = formatErasureGetDeploymentID(quorumFormat, formats); !errors.Is(err, errCorruptedFormat) {
-		t.Fatalf("Unexpected error %s", err)
-	}
-}
-
 // Initialize new format sets.
 func TestNewFormatSets(t *testing.T) {
 	setCount := 2

cmd/ftp-server-driver.go

@@ -25,6 +25,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"strings"
 	"time"
 
@@ -33,7 +34,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio/internal/auth"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/v2/mimedb"
 	ftp "goftp.io/server/v2"
 )
 
@@ -82,8 +83,13 @@ func (m *minioFileInfo) Mode() os.FileMode {
 	return os.ModePerm
 }
 
+var minFileDate = time.Date(1980, 1, 1, 0, 0, 0, 0, time.UTC) // Workaround for Filezilla
+
 func (m *minioFileInfo) ModTime() time.Time {
-	return m.info.LastModified
+	if !m.info.LastModified.IsZero() {
+		return m.info.LastModified
+	}
+	return minFileDate
 }
 
 func (m *minioFileInfo) IsDir() bool {
@@ -99,7 +105,7 @@ type ftpMetrics struct{}
 
 var globalFtpMetrics ftpMetrics
 
-func ftpTrace(s *ftp.Context, startTime time.Time, source, path string, err error) madmin.TraceInfo {
+func ftpTrace(s *ftp.Context, startTime time.Time, source, objPath string, err error) madmin.TraceInfo {
 	var errStr string
 	if err != nil {
 		errStr = err.Error()
@@ -110,7 +116,7 @@ func ftpTrace(s *ftp.Context, startTime time.Time, source, path string, err erro
 		NodeName:  globalLocalNodeName,
 		FuncName:  fmt.Sprintf("ftp USER=%s COMMAND=%s PARAM=%s ISLOGIN=%t, Source=%s", s.Sess.LoginUser(), s.Cmd, s.Param, s.Sess.IsLogin(), source),
 		Duration:  time.Since(startTime),
-		Path:      path,
+		Path:      objPath,
 		Error:     errStr,
 	}
 }
@@ -124,18 +130,18 @@ func (m *ftpMetrics) log(s *ftp.Context, paths ...string) func(err error) {
 }
 
 // Stat implements ftpDriver
-func (driver *ftpDriver) Stat(ctx *ftp.Context, path string) (fi os.FileInfo, err error) {
-	stopFn := globalFtpMetrics.log(ctx, path)
+func (driver *ftpDriver) Stat(ctx *ftp.Context, objPath string) (fi os.FileInfo, err error) {
+	stopFn := globalFtpMetrics.log(ctx, objPath)
 	defer stopFn(err)
 
-	if path == SlashSeparator {
+	if objPath == SlashSeparator {
 		return &minioFileInfo{
 			p:     SlashSeparator,
 			isDir: true,
 		}, nil
 	}
 
-	bucket, object := path2BucketObject(path)
+	bucket, object := path2BucketObject(objPath)
 	if bucket == "" {
 		return nil, errors.New("bucket name cannot be empty")
 	}
@@ -182,8 +188,8 @@ func (driver *ftpDriver) Stat(ctx *ftp.Context, path string) (fi os.FileInfo, er
 }
 
 // ListDir implements ftpDriver
-func (driver *ftpDriver) ListDir(ctx *ftp.Context, path string, callback func(os.FileInfo) error) (err error) {
-	stopFn := globalFtpMetrics.log(ctx, path)
+func (driver *ftpDriver) ListDir(ctx *ftp.Context, objPath string, callback func(os.FileInfo) error) (err error) {
+	stopFn := globalFtpMetrics.log(ctx, objPath)
 	defer stopFn(err)
 
 	clnt, err := driver.getMinIOClient(ctx)
@@ -194,7 +200,7 @@ func (driver *ftpDriver) ListDir(ctx *ftp.Context, path string, callback func(os
 	cctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	bucket, prefix := path2BucketObject(path)
+	bucket, prefix := path2BucketObject(objPath)
 	if bucket == "" {
 		buckets, err := clnt.ListBuckets(cctx)
 		if err != nil {
@@ -323,7 +329,7 @@ func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error)
 			}
 
 			// Call hook for site replication.
-			logger.LogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{
+			replLogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{
 				Type: madmin.SRIAMItemSTSAcc,
 				STSCredential: &madmin.SRSTSCredential{
 					AccessKey:    cred.AccessKey,
@@ -361,11 +367,11 @@ func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error)
 }
 
 // DeleteDir implements ftpDriver
-func (driver *ftpDriver) DeleteDir(ctx *ftp.Context, path string) (err error) {
-	stopFn := globalFtpMetrics.log(ctx, path)
+func (driver *ftpDriver) DeleteDir(ctx *ftp.Context, objPath string) (err error) {
+	stopFn := globalFtpMetrics.log(ctx, objPath)
 	defer stopFn(err)
 
-	bucket, prefix := path2BucketObject(path)
+	bucket, prefix := path2BucketObject(objPath)
 	if bucket == "" {
 		return errors.New("deleting all buckets not allowed")
 	}
@@ -411,11 +417,11 @@ func (driver *ftpDriver) DeleteDir(ctx *ftp.Context, path string) (err error) {
 }
 
 // DeleteFile implements ftpDriver
-func (driver *ftpDriver) DeleteFile(ctx *ftp.Context, path string) (err error) {
-	stopFn := globalFtpMetrics.log(ctx, path)
+func (driver *ftpDriver) DeleteFile(ctx *ftp.Context, objPath string) (err error) {
+	stopFn := globalFtpMetrics.log(ctx, objPath)
 	defer stopFn(err)
 
-	bucket, object := path2BucketObject(path)
+	bucket, object := path2BucketObject(objPath)
 	if bucket == "" {
 		return errors.New("bucket name cannot be empty")
 	}
@@ -429,19 +435,19 @@ func (driver *ftpDriver) DeleteFile(ctx *ftp.Context, path string) (err error) {
 }
 
 // Rename implements ftpDriver
-func (driver *ftpDriver) Rename(ctx *ftp.Context, fromPath string, toPath string) (err error) {
-	stopFn := globalFtpMetrics.log(ctx, fromPath, toPath)
+func (driver *ftpDriver) Rename(ctx *ftp.Context, fromObjPath string, toObjPath string) (err error) {
+	stopFn := globalFtpMetrics.log(ctx, fromObjPath, toObjPath)
 	defer stopFn(err)
 
 	return NotImplemented{}
 }
 
 // MakeDir implements ftpDriver
-func (driver *ftpDriver) MakeDir(ctx *ftp.Context, path string) (err error) {
-	stopFn := globalFtpMetrics.log(ctx, path)
+func (driver *ftpDriver) MakeDir(ctx *ftp.Context, objPath string) (err error) {
+	stopFn := globalFtpMetrics.log(ctx, objPath)
 	defer stopFn(err)
 
-	bucket, prefix := path2BucketObject(path)
+	bucket, prefix := path2BucketObject(objPath)
 	if bucket == "" {
 		return errors.New("bucket name cannot be empty")
 	}
@@ -457,21 +463,18 @@ func (driver *ftpDriver) MakeDir(ctx *ftp.Context, path string) (err error) {
 
 	dirPath := buildMinioDir(prefix)
 
-	_, err = clnt.PutObject(context.Background(), bucket, dirPath, bytes.NewReader([]byte("")), 0,
-		// Always send Content-MD5 to succeed with bucket with
-		// locking enabled. There is no performance hit since
-		// this is always an empty object
-		minio.PutObjectOptions{SendContentMd5: true},
-	)
+	_, err = clnt.PutObject(context.Background(), bucket, dirPath, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{
+		DisableContentSha256: true,
+	})
 	return err
 }
 
 // GetFile implements ftpDriver
-func (driver *ftpDriver) GetFile(ctx *ftp.Context, path string, offset int64) (n int64, rc io.ReadCloser, err error) {
-	stopFn := globalFtpMetrics.log(ctx, path)
+func (driver *ftpDriver) GetFile(ctx *ftp.Context, objPath string, offset int64) (n int64, rc io.ReadCloser, err error) {
+	stopFn := globalFtpMetrics.log(ctx, objPath)
 	defer stopFn(err)
 
-	bucket, object := path2BucketObject(path)
+	bucket, object := path2BucketObject(objPath)
 	if bucket == "" {
 		return 0, nil, errors.New("bucket name cannot be empty")
 	}
@@ -506,11 +509,11 @@ func (driver *ftpDriver) GetFile(ctx *ftp.Context, path string, offset int64) (n
 }
 
 // PutFile implements ftpDriver
-func (driver *ftpDriver) PutFile(ctx *ftp.Context, path string, data io.Reader, offset int64) (n int64, err error) {
-	stopFn := globalFtpMetrics.log(ctx, path)
+func (driver *ftpDriver) PutFile(ctx *ftp.Context, objPath string, data io.Reader, offset int64) (n int64, err error) {
+	stopFn := globalFtpMetrics.log(ctx, objPath)
 	defer stopFn(err)
 
-	bucket, object := path2BucketObject(path)
+	bucket, object := path2BucketObject(objPath)
 	if bucket == "" {
 		return 0, errors.New("bucket name cannot be empty")
 	}
@@ -526,8 +529,8 @@ func (driver *ftpDriver) PutFile(ctx *ftp.Context, path string, data io.Reader,
 	}
 
 	info, err := clnt.PutObject(context.Background(), bucket, object, data, -1, minio.PutObjectOptions{
-		ContentType:    "application/octet-stream",
-		SendContentMd5: true,
+		ContentType:          mimedb.TypeByExtension(path.Ext(object)),
+		DisableContentSha256: true,
 	})
 	return info.Size, err
 }

cmd/generic-handlers.go

@@ -310,6 +310,13 @@ func hasBadHost(host string) error {
 // Check if the incoming path has bad path components,
 // such as ".." and "."
 func hasBadPathComponent(path string) bool {
+	if len(path) > 4096 {
+		// path cannot be greater than Linux PATH_MAX
+		// this is to avoid a busy loop, that can happen
+		// if the caller sends path of following style
+		// a/a/a/a/a/a/a/a...
+		return true
+	}
 	path = filepath.ToSlash(strings.TrimSpace(path)) // For windows '\' must be converted to '/'
 	for _, p := range strings.Split(path, SlashSeparator) {
 		switch strings.TrimSpace(p) {

cmd/global-heal.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2022 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -19,13 +19,19 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"math/rand"
 	"runtime"
 	"sort"
 	"time"
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config/storageclass"
 	xioutil "github.com/minio/minio/internal/ioutil"
@@ -153,7 +159,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		_, err := objAPI.HealBucket(ctx, bucket, madmin.HealOpts{ScanMode: scanMode})
 		if err != nil {
 			// Log bucket healing error if any, we shall retry again.
-			logger.LogIf(ctx, err)
+			healingLogIf(ctx, err)
 		}
 	}
 
@@ -177,7 +183,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		numHealers = uint64(v)
 	}
 
-	logger.Event(ctx, fmt.Sprintf("Healing drive '%s' - use %d parallel workers.", tracker.disk.String(), numHealers))
+	healingLogEvent(ctx, fmt.Sprintf("Healing drive '%s' - use %d parallel workers.", tracker.disk.String(), numHealers))
 
 	jt, _ := workers.New(int(numHealers))
 
@@ -204,37 +210,76 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		if _, err := objAPI.HealBucket(ctx, bucket, madmin.HealOpts{
 			ScanMode: scanMode,
 		}); err != nil {
-			logger.LogIf(ctx, err)
+			// Set this such that when we return this function
+			// we let the caller retry this disk again for the
+			// buckets that failed healing.
+			retErr = err
+			healingLogIf(ctx, err)
 			continue
 		}
 
-		vc, _ := globalBucketVersioningSys.Get(bucket)
+		var (
+			vc   *versioning.Versioning
+			lc   *lifecycle.Lifecycle
+			lr   objectlock.Retention
+			rcfg *replication.Config
+		)
 
-		// Check if the current bucket has a configured lifecycle policy
-		lc, _ := globalLifecycleSys.Get(bucket)
-
-		// Check if bucket is object locked.
-		lr, _ := globalBucketObjectLockSys.Get(bucket)
-		rcfg, _ := getReplicationConfig(ctx, bucket)
+		if !isMinioMetaBucketName(bucket) {
+			vc, err = globalBucketVersioningSys.Get(bucket)
+			if err != nil {
+				retErr = err
+				healingLogIf(ctx, err)
+				continue
+			}
+			// Check if the current bucket has a configured lifecycle policy
+			lc, err = globalLifecycleSys.Get(bucket)
+			if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) {
+				retErr = err
+				healingLogIf(ctx, err)
+				continue
+			}
+			// Check if bucket is object locked.
+			lr, err = globalBucketObjectLockSys.Get(bucket)
+			if err != nil {
+				retErr = err
+				healingLogIf(ctx, err)
+				continue
+			}
+			rcfg, err = getReplicationConfig(ctx, bucket)
+			if err != nil {
+				retErr = err
+				healingLogIf(ctx, err)
+				continue
+			}
+		}
 
 		if serverDebugLog {
 			console.Debugf(color.Green("healDrive:")+" healing bucket %s content on %s erasure set\n",
 				bucket, humanize.Ordinal(er.setIndex+1))
 		}
 
-		disks, _ := er.getOnlineDisksWithHealing(false)
-		if len(disks) == 0 {
-			// No object healing necessary
-			tracker.bucketDone(bucket)
-			logger.LogIf(ctx, tracker.update(ctx))
-			continue
+		disks, _, healing := er.getOnlineDisksWithHealingAndInfo(true)
+		if len(disks) == healing {
+			// All drives in this erasure set were reformatted for some reasons, abort healing and mark it as successful
+			healingLogIf(ctx, errors.New("all drives are in healing state, aborting.."))
+			return nil
 		}
 
-		// Limit listing to 3 drives.
-		if len(disks) > 3 {
-			disks = disks[:3]
+		disks = disks[:len(disks)-healing] // healing drives are always at the end of the list
+
+		if len(disks) < er.setDriveCount/2 {
+			return fmt.Errorf("not enough drives (found=%d, healing=%d, total=%d) are available to heal `%s`", len(disks), healing, er.setDriveCount, tracker.disk.String())
 		}
 
+		rand.Shuffle(len(disks), func(i, j int) {
+			disks[i], disks[j] = disks[j], disks[i]
+		})
+
+		expectedDisks := len(disks)/2 + 1
+		fallbackDisks := disks[expectedDisks:]
+		disks = disks[:expectedDisks]
+
 		type healEntryResult struct {
 			bytes     uint64
 			success   bool
@@ -293,7 +338,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				if res.entryDone {
 					tracker.setObject(res.name)
 					if time.Since(tracker.getLastUpdate()) > time.Minute {
-						logger.LogIf(ctx, tracker.update(ctx))
+						healingLogIf(ctx, tracker.update(ctx))
 					}
 					continue
 				}
@@ -306,7 +351,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			select {
 			case <-ctx.Done():
 				if !contextCanceled(ctx) {
-					logger.LogIf(ctx, ctx.Err())
+					healingLogIf(ctx, ctx.Err())
 				}
 				return false
 			case results <- result:
@@ -360,7 +405,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 						return
 					}
 					result = healEntryFailure(0)
-					logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, entry.name, err))
+					healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, entry.name, err))
 				} else {
 					result = healEntrySuccess(0)
 				}
@@ -399,9 +444,9 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					// If not deleted, assume they failed.
 					result = healEntryFailure(uint64(version.Size))
 					if version.VersionID != "" {
-						logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
+						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
 					} else {
-						logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, version.Name, err))
+						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, version.Name, err))
 					}
 				} else {
 					result = healEntrySuccess(uint64(version.Size))
@@ -425,26 +470,24 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			waitForLowHTTPReq()
 		}
 
-		actualBucket, prefix := path2BucketObject(bucket)
-
 		// How to resolve partial results.
 		resolver := metadataResolutionParams{
 			dirQuorum: 1,
 			objQuorum: 1,
-			bucket:    actualBucket,
+			bucket:    bucket,
 		}
 
-		err := listPathRaw(ctx, listPathRawOptions{
+		err = listPathRaw(ctx, listPathRawOptions{
 			disks:          disks,
-			bucket:         actualBucket,
-			path:           prefix,
+			fallbackDisks:  fallbackDisks,
+			bucket:         bucket,
 			recursive:      true,
 			forwardTo:      forwardTo,
 			minDisks:       1,
 			reportNotFound: false,
 			agreed: func(entry metaCacheEntry) {
 				jt.Take()
-				go healEntry(actualBucket, entry)
+				go healEntry(bucket, entry)
 			},
 			partial: func(entries metaCacheEntries, _ []error) {
 				entry, ok := entries.resolve(&resolver)
@@ -454,7 +497,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					entry, _ = entries.firstFound()
 				}
 				jt.Take()
-				go healEntry(actualBucket, *entry)
+				go healEntry(bucket, *entry)
 			},
 			finished: nil,
 		})
@@ -465,7 +508,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			// we let the caller retry this disk again for the
 			// buckets it failed to list.
 			retErr = err
-			logger.LogIf(ctx, err)
+			healingLogIf(ctx, err)
 			continue
 		}
 
@@ -475,7 +518,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			return ctx.Err()
 		default:
 			tracker.bucketDone(bucket)
-			logger.LogIf(ctx, tracker.update(ctx))
+			healingLogIf(ctx, tracker.update(ctx))
 		}
 	}
 
@@ -499,16 +542,7 @@ func healObject(bucket, object, versionID string, scan madmin.HealScanMode) erro
 	// Get background heal sequence to send elements to heal
 	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
 	if ok {
-		return bgSeq.queueHealTask(healSource{
-			bucket:    bucket,
-			object:    object,
-			versionID: versionID,
-			noWait:    true, // do not block callers.
-			opts: &madmin.HealOpts{
-				Remove:   healDeleteDangling, // if found dangling purge it.
-				ScanMode: scan,
-			},
-		}, madmin.HealItemObject)
+		return bgSeq.healObject(bucket, object, versionID, scan)
 	}
 	return nil
 }

cmd/globals.go

@@ -171,7 +171,8 @@ type serverCtxt struct {
 	ReadHeaderTimeout   time.Duration
 	MaxIdleConnsPerHost int
 
-	CrossDomainXML string
+	SendBufSize, RecvBufSize int
+	CrossDomainXML           string
 	// The layout of disks as interpreted
 	Layout disksLayout
 }
@@ -241,7 +242,7 @@ var (
 	globalBucketMonitor     *bandwidth.Monitor
 	globalPolicySys         *PolicySys
 	globalIAMSys            *IAMSys
-	globalBytePoolCap       *bpool.BytePoolCap
+	globalBytePoolCap       atomic.Pointer[bpool.BytePoolCap]
 
 	globalLifecycleSys       *LifecycleSys
 	globalBucketSSEConfigSys *BucketSSEConfigSys
@@ -398,12 +399,8 @@ var (
 
 	globalInternodeTransport http.RoundTripper
 
-	globalProxyTransport http.RoundTripper
-
 	globalRemoteTargetTransport http.RoundTripper
 
-	globalHealthChkTransport http.RoundTripper
-
 	globalDNSCache = &dnscache.Resolver{
 		Timeout: 5 * time.Second,
 	}
@@ -450,11 +447,11 @@ var (
 	subnetAdminPublicKey    = []byte("-----BEGIN PUBLIC KEY-----\nMIIBCgKCAQEAyC+ol5v0FP+QcsR6d1KypR/063FInmNEFsFzbEwlHQyEQN3O7kNI\nwVDN1vqp1wDmJYmv4VZGRGzfFw1q+QV7K1TnysrEjrqpVxfxzDQCoUadAp8IxLLc\ns2fjyDNxnZjoC6fTID9C0khKnEa5fPZZc3Ihci9SiCGkPmyUyCGVSxWXIKqL2Lrj\nyDc0pGeEhWeEPqw6q8X2jvTC246tlzqpDeNsPbcv2KblXRcKniQNbBrizT37CKHQ\nM6hc9kugrZbFuo8U5/4RQvZPJnx/DVjLDyoKo2uzuVQs4s+iBrA5sSSLp8rPED/3\n6DgWw3e244Dxtrg972dIT1IOqgn7KUJzVQIDAQAB\n-----END PUBLIC KEY-----")
 	subnetAdminPublicKeyDev = []byte("-----BEGIN PUBLIC KEY-----\nMIIBCgKCAQEArhQYXQd6zI4uagtVfthAPOt6i4AYHnEWCoNeAovM4MNl42I9uQFh\n3VHkbWj9Gpx9ghf6PgRgK+8FcFvy+StmGcXpDCiFywXX24uNhcZjscX1C4Esk0BW\nidfI2eXYkOlymD4lcK70SVgJvC693Qa7Z3FE1KU8Nfv2bkxEE4bzOkojX9t6a3+J\nR8X6Z2U8EMlH1qxJPgiPogELhWP0qf2Lq7GwSAflo1Tj/ytxvD12WrnE0Rrj/8yP\nSnp7TbYm91KocKMExlmvx3l2XPLxeU8nf9U0U+KOmorejD3MDMEPF+tlk9LB3JWP\nZqYYe38rfALVTn4RVJriUcNOoEpEyC0WEwIDAQAB\n-----END PUBLIC KEY-----")
 
-	globalConnReadDeadline  time.Duration
-	globalConnWriteDeadline time.Duration
+	// dynamic sleeper to avoid thundering herd for trash folder expunge routine
+	deleteCleanupSleeper = newDynamicSleeper(5, 25*time.Millisecond, false)
 
-	// Controller for deleted file sweeper.
-	deletedCleanupSleeper = newDynamicSleeper(5, 25*time.Millisecond, false)
+	// dynamic sleeper for multipart expiration routine
+	deleteMultipartCleanupSleeper = newDynamicSleeper(5, 25*time.Millisecond, false)
 
 	// Is _MINIO_DISABLE_API_FREEZE_ON_BOOT set?
 	globalDisableFreezeOnBoot bool

cmd/handler-api.go

@@ -142,7 +142,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 		// total_ram / ram_per_request
 		// ram_per_request is (2MiB+128KiB) * driveCount \
 		//    + 2 * 10MiB (default erasure block size v1) + 2 * 1MiB (default erasure block size v2)
-		blockSize := xioutil.BlockSizeLarge + xioutil.BlockSizeSmall
+		blockSize := xioutil.LargeBlock + xioutil.SmallBlock
 		apiRequestsMaxPerNode = int(maxMem / uint64(maxSetDrives*blockSize+int(blockSizeV1*2+blockSizeV2*2)))
 		if globalIsDistErasure {
 			logger.Info("Automatically configured API requests per node based on available memory on the system: %d", apiRequestsMaxPerNode)

cmd/handler-utils.go

@@ -51,7 +51,7 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError
 	locationConstraint := createBucketLocationConfiguration{}
 	err := xmlDecoder(r.Body, &locationConstraint, r.ContentLength)
 	if err != nil && r.ContentLength != 0 {
-		logger.LogOnceIf(GlobalContext, err, "location-constraint-xml-parsing")
+		internalLogOnceIf(GlobalContext, err, "location-constraint-xml-parsing")
 		// Treat all other failures as XML parsing errors.
 		return "", ErrMalformedXML
 	} // else for both err as nil or io.EOF
@@ -191,7 +191,7 @@ func extractMetadata(ctx context.Context, mimesHeader ...textproto.MIMEHeader) (
 // extractMetadata extracts metadata from map values.
 func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
 	if v == nil {
-		logger.LogIf(ctx, errInvalidArgument)
+		bugLogIf(ctx, errInvalidArgument)
 		return errInvalidArgument
 	}
 
@@ -461,7 +461,7 @@ func proxyRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, e
 		ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
 			success = false
 			if err != nil && !errors.Is(err, context.Canceled) {
-				logger.LogIf(GlobalContext, err)
+				replLogIf(GlobalContext, err)
 			}
 		},
 	})

cmd/healthcheck-handler.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
 )
 
 const unavailable = "offline"
@@ -134,7 +135,7 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 		ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
 		defer cancel()
 
-		if _, err := GlobalKMS.Stat(ctx); err != nil {
+		if _, err := GlobalKMS.GenerateKey(ctx, "", kms.Context{"healthcheck": ""}); err != nil {
 			switch r.Method {
 			case http.MethodHead:
 				apiErr := toAPIError(r.Context(), err)

cmd/iam-etcd-store.go

@@ -30,7 +30,6 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
 	"github.com/puzpuzpuz/xsync/v3"
 	"go.etcd.io/etcd/api/v3/mvccpb"
 	etcd "go.etcd.io/etcd/client/v3"
@@ -460,7 +459,7 @@ func (ies *IAMEtcdStore) watch(ctx context.Context, keyPath string) <-chan iamWa
 						goto outerLoop
 					}
 					if err := watchResp.Err(); err != nil {
-						logger.LogIf(ctx, err)
+						iamLogIf(ctx, err)
 						// log and retry.
 						time.Sleep(1 * time.Second)
 						// Upon an error on watch channel

cmd/iam-object-store.go

@@ -34,7 +34,6 @@ import (
 	"github.com/minio/minio/internal/config"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
 	"github.com/puzpuzpuz/xsync/v3"
 )
 
@@ -400,6 +399,7 @@ var (
 	groupsListKey           = "groups/"
 	policiesListKey         = "policies/"
 	stsListKey              = "sts/"
+	policyDBPrefix          = "policydb/"
 	policyDBUsersListKey    = "policydb/users/"
 	policyDBSTSUsersListKey = "policydb/sts-users/"
 	policyDBGroupsListKey   = "policydb/groups/"
@@ -407,8 +407,13 @@ var (
 
 // splitPath splits a path into a top-level directory and a child item. The
 // parent directory retains the trailing slash.
-func splitPath(s string) (string, string) {
-	i := strings.Index(s, "/")
+func splitPath(s string, lastIndex bool) (string, string) {
+	var i int
+	if lastIndex {
+		i = strings.LastIndex(s, "/")
+	} else {
+		i = strings.Index(s, "/")
+	}
 	if i == -1 {
 		return s, ""
 	}
@@ -425,7 +430,8 @@ func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (map[str
 			return nil, item.Err
 		}
 
-		listKey, trimmedItem := splitPath(item.Item)
+		lastIndex := strings.HasPrefix(item.Item, policyDBPrefix)
+		listKey, trimmedItem := splitPath(item.Item, lastIndex)
 		if listKey == iamFormatFile {
 			continue
 		}
@@ -448,7 +454,7 @@ func (iamOS *IAMObjectStore) PurgeExpiredSTS(ctx context.Context) error {
 	iamListing, ok := iamOS.cachedIAMListing.Load().(map[string][]string)
 	if !ok {
 		// There has been no store yet. This should never happen!
-		logger.LogIf(GlobalContext, errors.New("WARNING: no cached IAM listing found"))
+		iamLogIf(GlobalContext, errors.New("WARNING: no cached IAM listing found"))
 		return nil
 	}
 
@@ -461,7 +467,7 @@ func (iamOS *IAMObjectStore) PurgeExpiredSTS(ctx context.Context) error {
 		// loadUser() will delete expired user during the load.
 		err := iamOS.loadUser(ctx, userName, stsUser, stsAccountsFromStore)
 		if err != nil && !errors.Is(err, errNoSuchUser) {
-			logger.LogIf(GlobalContext,
+			iamLogIf(GlobalContext,
 				fmt.Errorf("unable to load user during STS purge: %w (%s)", err, item))
 		}
 
@@ -472,7 +478,7 @@ func (iamOS *IAMObjectStore) PurgeExpiredSTS(ctx context.Context) error {
 		stsName := strings.TrimSuffix(item, ".json")
 		err := iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, stsAccPoliciesFromStore)
 		if err != nil && !errors.Is(err, errNoSuchPolicy) {
-			logger.LogIf(GlobalContext,
+			iamLogIf(GlobalContext,
 				fmt.Errorf("unable to load policies during STS purge: %w (%s)", err, item))
 		}
 

cmd/iam-store.go

@@ -33,7 +33,6 @@ import (
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/identity/openid"
 	"github.com/minio/minio/internal/jwt"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/policy"
 	"github.com/puzpuzpuz/xsync/v3"
 )
@@ -1635,6 +1634,8 @@ func (store *IAMStoreSys) PolicyMappingNotificationHandler(ctx context.Context,
 	switch {
 	case isGroup:
 		m = cache.iamGroupPolicyMap
+	case userType == stsUser:
+		m = cache.iamSTSPolicyMap
 	default:
 		m = cache.iamUserPolicyMap
 	}
@@ -1882,7 +1883,7 @@ func (store *IAMStoreSys) DeleteUsers(ctx context.Context, users []string) error
 
 			// we are only logging errors, not handling them.
 			err := store.deleteUserIdentity(ctx, user, userType)
-			logger.LogIf(GlobalContext, err)
+			iamLogIf(GlobalContext, err)
 			delete(cache.iamUsersMap, user)
 
 			deleted = true
@@ -2109,6 +2110,32 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 			}
 		}
 	}
+	if iamOS, ok := store.IAMStorageAPI.(*IAMEtcdStore); ok {
+		m := xsync.NewMapOf[string, MappedPolicy]()
+		err := iamOS.loadMappedPolicies(context.Background(), stsUser, false, m)
+		if err == nil {
+			m.Range(func(user string, mappedPolicy MappedPolicy) bool {
+				if userPredicate != nil && !userPredicate(user) {
+					return true
+				}
+
+				commonPolicySet := mappedPolicy.policySet()
+				if !queryPolSet.IsEmpty() {
+					commonPolicySet = commonPolicySet.Intersection(queryPolSet)
+				}
+				for _, policy := range commonPolicySet.ToSlice() {
+					s, ok := policyToUsersMap[policy]
+					if !ok {
+						policyToUsersMap[policy] = set.CreateStringSet(user)
+					} else {
+						s.Add(user)
+						policyToUsersMap[policy] = s
+					}
+				}
+				return true
+			})
+		}
+	}
 
 	policyToGroupsMap := make(map[string]set.StringSet)
 	cache.iamGroupPolicyMap.Range(func(group string, mappedPolicy MappedPolicy) bool {

cmd/iam.go

@@ -32,6 +32,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	libldap "github.com/go-ldap/ldap/v3"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/arn"
@@ -48,6 +49,7 @@ import (
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/v2/ldap"
 	"github.com/minio/pkg/v2/policy"
 	etcd "go.etcd.io/etcd/client/v3"
 )
@@ -230,42 +232,42 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	openidConfig, err := openid.LookupConfig(s,
 		NewHTTPTransport(), xhttp.DrainBody, globalSite.Region)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
+		iamLogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err), logger.WarningKind)
 	}
 
 	// Initialize if LDAP is enabled
 	ldapConfig, err := xldap.Lookup(s, globalRootCAs)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to parse LDAP configuration: %w", err))
+		iamLogIf(ctx, fmt.Errorf("Unable to load LDAP configuration (LDAP configuration will be disabled!): %w", err), logger.WarningKind)
 	}
 
 	stsTLSConfig, err := xtls.Lookup(s[config.IdentityTLSSubSys][config.Default])
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize X.509/TLS STS API: %w", err))
+		iamLogIf(ctx, fmt.Errorf("Unable to initialize X.509/TLS STS API: %w", err), logger.WarningKind)
 	}
 
 	if stsTLSConfig.InsecureSkipVerify {
-		logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
+		iamLogIf(ctx, fmt.Errorf("Enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify), logger.WarningKind)
 	}
 
 	authNPluginCfg, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
 		NewHTTPTransport(), xhttp.DrainBody, globalSite.Region)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthNPlugin: %w", err))
+		iamLogIf(ctx, fmt.Errorf("Unable to initialize AuthNPlugin: %w", err), logger.WarningKind)
 	}
 
 	setGlobalAuthNPlugin(idplugin.New(GlobalContext, authNPluginCfg))
 
 	authZPluginCfg, err := polplugin.LookupConfig(s, GetDefaultConnSettings(), xhttp.DrainBody)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin: %w", err))
+		iamLogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin: %w", err), logger.WarningKind)
 	}
 
 	if authZPluginCfg.URL == nil {
 		opaCfg, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
 			NewHTTPTransport(), xhttp.DrainBody)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin from legacy OPA config: %w", err))
+			iamLogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin from legacy OPA config: %w", err))
 		} else {
 			authZPluginCfg.URL = opaCfg.URL
 			authZPluginCfg.AuthToken = opaCfg.AuthToken
@@ -301,7 +303,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 				time.Sleep(time.Duration(r.Float64() * float64(time.Second)))
 				continue
 			}
-			logger.LogIf(ctx, fmt.Errorf("IAM sub-system is partially initialized, unable to write the IAM format: %w", err))
+			iamLogIf(ctx, fmt.Errorf("IAM sub-system is partially initialized, unable to write the IAM format: %w", err), logger.WarningKind)
 			return
 		}
 
@@ -317,7 +319,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 				continue
 			}
 			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize IAM sub-system, some users may not be available: %w", err))
+				iamLogIf(ctx, fmt.Errorf("Unable to initialize IAM sub-system, some users may not be available: %w", err), logger.WarningKind)
 			}
 		}
 		break
@@ -355,7 +357,7 @@ func (sys *IAMSys) periodicRoutines(ctx context.Context, baseInterval time.Durat
 			for event := range ch {
 				if err := sys.loadWatchedEvent(ctx, event); err != nil {
 					// we simply log errors
-					logger.LogIf(ctx, fmt.Errorf("Failure in loading watch event: %v", err))
+					iamLogIf(ctx, fmt.Errorf("Failure in loading watch event: %v", err), logger.WarningKind)
 				}
 			}
 		}()
@@ -388,7 +390,7 @@ func (sys *IAMSys) periodicRoutines(ctx context.Context, baseInterval time.Durat
 			// Load all IAM items (except STS creds) periodically.
 			refreshStart := time.Now()
 			if err := sys.Load(ctx, false); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Failure in periodic refresh for IAM (took %.2fs): %v", time.Since(refreshStart).Seconds(), err))
+				iamLogIf(ctx, fmt.Errorf("Failure in periodic refresh for IAM (took %.2fs): %v", time.Since(refreshStart).Seconds(), err), logger.WarningKind)
 			} else {
 				took := time.Since(refreshStart).Seconds()
 				if took > maxDurationSecondsForLog {
@@ -400,7 +402,7 @@ func (sys *IAMSys) periodicRoutines(ctx context.Context, baseInterval time.Durat
 			// Purge expired STS credentials.
 			purgeStart := time.Now()
 			if err := sys.store.PurgeExpiredSTS(ctx); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Failure in periodic STS purge for IAM (took %.2fs): %v", time.Since(purgeStart).Seconds(), err))
+				iamLogIf(ctx, fmt.Errorf("Failure in periodic STS purge for IAM (took %.2fs): %v", time.Since(purgeStart).Seconds(), err))
 			} else {
 				took := time.Since(purgeStart).Seconds()
 				if took > maxDurationSecondsForLog {
@@ -450,7 +452,7 @@ func (sys *IAMSys) validateAndAddRolePolicyMappings(ctx context.Context, m map[a
 				errMsg := fmt.Errorf(
 					"The policies \"%s\" mapped to role ARN %s are not defined - this role may not work as expected.",
 					unknownPoliciesSet.ToSlice(), arn.String())
-				logger.LogIf(ctx, errMsg)
+				authZLogIf(ctx, errMsg, logger.WarningKind)
 			}
 		}
 		sys.rolesMap[arn] = rolePolicies
@@ -573,7 +575,7 @@ func (sys *IAMSys) DeletePolicy(ctx context.Context, policyName string, notifyPe
 	for _, nerr := range globalNotificationSys.DeletePolicy(policyName) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
+			iamLogIf(ctx, nerr.Err)
 		}
 	}
 
@@ -638,7 +640,7 @@ func (sys *IAMSys) SetPolicy(ctx context.Context, policyName string, p policy.Po
 		for _, nerr := range globalNotificationSys.LoadPolicy(policyName) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
+				iamLogIf(ctx, nerr.Err)
 			}
 		}
 	}
@@ -660,7 +662,7 @@ func (sys *IAMSys) DeleteUser(ctx context.Context, accessKey string, notifyPeers
 		for _, nerr := range globalNotificationSys.DeleteUser(accessKey) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
+				iamLogIf(ctx, nerr.Err)
 			}
 		}
 	}
@@ -686,7 +688,7 @@ func (sys *IAMSys) notifyForUser(ctx context.Context, accessKey string, isTemp b
 		for _, nerr := range globalNotificationSys.LoadUser(accessKey, isTemp) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
+				iamLogIf(ctx, nerr.Err)
 			}
 		}
 	}
@@ -931,7 +933,7 @@ func (sys *IAMSys) notifyForServiceAccount(ctx context.Context, accessKey string
 		for _, nerr := range globalNotificationSys.LoadServiceAccount(accessKey) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
+				iamLogIf(ctx, nerr.Err)
 			}
 		}
 	}
@@ -1252,7 +1254,7 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string, n
 		for _, nerr := range globalNotificationSys.DeleteServiceAccount(accessKey) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
+				iamLogIf(ctx, nerr.Err)
 			}
 		}
 	}
@@ -1267,10 +1269,6 @@ func (sys *IAMSys) CreateUser(ctx context.Context, accessKey string, ureq madmin
 		return updatedAt, errServerNotInitialized
 	}
 
-	if sys.usersSysType != MinIOUsersSysType {
-		return updatedAt, errIAMActionNotAllowed
-	}
-
 	if !auth.IsAccessKeyValid(accessKey) {
 		return updatedAt, auth.ErrInvalidAccessKeyLength
 	}
@@ -1327,14 +1325,14 @@ func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
 		roleArns := puInfo.roleArns.ToSlice()
 		var roleArn string
 		if len(roleArns) == 0 {
-			logger.LogIf(GlobalContext,
+			iamLogIf(GlobalContext,
 				fmt.Errorf("parentUser: %s had no roleArns mapped!", parentUser))
 			continue
 		}
 		roleArn = roleArns[0]
 		u, err := sys.OpenIDConfig.LookupUser(roleArn, puInfo.subClaimValue)
 		if err != nil {
-			logger.LogIf(GlobalContext, err)
+			iamLogIf(GlobalContext, err)
 			continue
 		}
 		// If user is set to "disabled", we will remove them
@@ -1364,7 +1362,7 @@ func (sys *IAMSys) purgeExpiredCredentialsForLDAP(ctx context.Context) {
 	expiredUsers, err := sys.LDAPConfig.GetNonEligibleUserDistNames(allDistNames)
 	if err != nil {
 		// Log and return on error - perhaps it'll work the next time.
-		logger.LogIf(GlobalContext, err)
+		iamLogIf(GlobalContext, err)
 		return
 	}
 
@@ -1445,7 +1443,7 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 	updatedGroups, err := sys.LDAPConfig.LookupGroupMemberships(parentUsers, parentUserToLDAPUsernameMap)
 	if err != nil {
 		// Log and return on error - perhaps it'll work the next time.
-		logger.LogIf(GlobalContext, err)
+		iamLogIf(GlobalContext, err)
 		return
 	}
 
@@ -1469,12 +1467,239 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 			cred.Groups = currGroups
 			if err := sys.store.UpdateUserIdentity(ctx, cred); err != nil {
 				// Log and continue error - perhaps it'll work the next time.
-				logger.LogIf(GlobalContext, err)
+				iamLogIf(GlobalContext, err)
 			}
 		}
 	}
 }
 
+// NormalizeLDAPAccessKeypairs - normalize the access key pairs (service
+// accounts) for LDAP users. This normalizes the parent user and the group names
+// whenever the parent user parses validly as a DN.
+func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap map[string]madmin.SRSvcAccCreate,
+) (err error) {
+	conn, err := sys.LDAPConfig.LDAP.Connect()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	// Bind to the lookup user account
+	if err = sys.LDAPConfig.LDAP.LookupBind(conn); err != nil {
+		return err
+	}
+
+	var collectedErrors []error
+	updatedKeysMap := make(map[string]madmin.SRSvcAccCreate)
+	for ak, createReq := range accessKeyMap {
+		parent := createReq.Parent
+		groups := createReq.Groups
+
+		_, err := ldap.NormalizeDN(parent)
+		if err != nil {
+			// not a valid DN, ignore.
+			continue
+		}
+
+		hasDiff := false
+
+		// For the parent value, we require that the parent exists in the LDAP
+		// server and is under a configured base DN.
+		validatedParent, isUnderBaseDN, err := sys.LDAPConfig.GetValidatedUserDN(conn, parent)
+		if err != nil {
+			collectedErrors = append(collectedErrors, fmt.Errorf("could not validate `%s` exists in LDAP directory: %w", parent, err))
+			continue
+		}
+		if validatedParent == "" || !isUnderBaseDN {
+			err := fmt.Errorf("DN `%s` was not found in the LDAP directory", parent)
+			collectedErrors = append(collectedErrors, err)
+			continue
+		}
+
+		if validatedParent != parent {
+			hasDiff = true
+		}
+
+		var normalizedGroups []string
+		for _, group := range groups {
+			// For a group, we store the normalized DN even if it not under a
+			// configured base DN.
+			validatedGroup, _, err := sys.LDAPConfig.GetValidatedGroupDN(conn, group)
+			if err != nil {
+				collectedErrors = append(collectedErrors, fmt.Errorf("could not validate `%s` exists in LDAP directory: %w", group, err))
+				continue
+			}
+			if validatedGroup == "" {
+				err := fmt.Errorf("DN `%s` was not found in the LDAP directory", group)
+				collectedErrors = append(collectedErrors, err)
+				continue
+			}
+
+			if validatedGroup != group {
+				hasDiff = true
+			}
+			normalizedGroups = append(normalizedGroups, validatedGroup)
+		}
+
+		if hasDiff {
+			updatedCreateReq := createReq
+			updatedCreateReq.Parent = validatedParent
+			updatedCreateReq.Groups = normalizedGroups
+
+			updatedKeysMap[ak] = updatedCreateReq
+		}
+	}
+
+	// if there are any errors, return a collected error.
+	if len(collectedErrors) > 0 {
+		return fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...))
+	}
+
+	for k, v := range updatedKeysMap {
+		// Replace the map values with the updated ones
+		accessKeyMap[k] = v
+	}
+
+	return nil
+}
+
+func (sys *IAMSys) getStoredLDAPPolicyMappingKeys(ctx context.Context, isGroup bool) set.StringSet {
+	entityKeysInStorage := set.NewStringSet()
+	if iamOS, ok := sys.store.IAMStorageAPI.(*IAMObjectStore); ok {
+		// Load existing mapping keys from the cached listing for
+		// `IAMObjectStore`.
+		iamFilesListing := iamOS.cachedIAMListing.Load().(map[string][]string)
+		listKey := policyDBSTSUsersListKey
+		if isGroup {
+			listKey = policyDBGroupsListKey
+		}
+		for _, item := range iamFilesListing[listKey] {
+			stsUserName := strings.TrimSuffix(item, ".json")
+			entityKeysInStorage.Add(stsUserName)
+		}
+	} else {
+		// For non-iam object store, we copy the mapping keys from the cache.
+		cache := sys.store.rlock()
+		defer sys.store.runlock()
+		cachedPolicyMap := cache.iamSTSPolicyMap
+		if isGroup {
+			cachedPolicyMap = cache.iamGroupPolicyMap
+		}
+		cachedPolicyMap.Range(func(k string, v MappedPolicy) bool {
+			entityKeysInStorage.Add(k)
+			return true
+		})
+	}
+
+	return entityKeysInStorage
+}
+
+// NormalizeLDAPMappingImport - validates the LDAP policy mappings. Keys in the
+// given map may not correspond to LDAP DNs - these keys are ignored.
+//
+// For validated mappings, it updates the key in the given map to be in
+// normalized form.
+func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool,
+	policyMap map[string]MappedPolicy,
+) error {
+	conn, err := sys.LDAPConfig.LDAP.Connect()
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	// Bind to the lookup user account
+	if err = sys.LDAPConfig.LDAP.LookupBind(conn); err != nil {
+		return err
+	}
+
+	// We map keys that correspond to LDAP DNs and validate that they exist in
+	// the LDAP server.
+	var dnValidator func(*libldap.Conn, string) (string, bool, error) = sys.LDAPConfig.GetValidatedUserDN
+	if isGroup {
+		dnValidator = sys.LDAPConfig.GetValidatedGroupDN
+	}
+
+	// map of normalized DN keys to original keys.
+	normalizedDNKeysMap := make(map[string][]string)
+	var collectedErrors []error
+	for k := range policyMap {
+		_, err := ldap.NormalizeDN(k)
+		if err != nil {
+			// not a valid DN, ignore.
+			continue
+		}
+		validatedDN, underBaseDN, err := dnValidator(conn, k)
+		if err != nil {
+			collectedErrors = append(collectedErrors, fmt.Errorf("could not validate `%s` exists in LDAP directory: %w", k, err))
+			continue
+		}
+		if validatedDN == "" || !underBaseDN {
+			err := fmt.Errorf("DN `%s` was not found in the LDAP directory", k)
+			collectedErrors = append(collectedErrors, err)
+			continue
+		}
+
+		if validatedDN != k {
+			normalizedDNKeysMap[validatedDN] = append(normalizedDNKeysMap[validatedDN], k)
+		}
+	}
+
+	// if there are any errors, return a collected error.
+	if len(collectedErrors) > 0 {
+		return fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...))
+	}
+
+	entityKeysInStorage := sys.getStoredLDAPPolicyMappingKeys(ctx, isGroup)
+
+	for normKey, origKeys := range normalizedDNKeysMap {
+		if len(origKeys) > 1 {
+			// If there are multiple DN keys that normalize to the same value,
+			// check if the policy mappings are equal, if they are we don't need
+			// to return an error.
+			policiesDiffer := false
+			firstMappedPolicies := policyMap[origKeys[0]].policySet()
+			for i := 1; i < len(origKeys); i++ {
+				otherMappedPolicies := policyMap[origKeys[i]].policySet()
+				if !firstMappedPolicies.Equals(otherMappedPolicies) {
+					policiesDiffer = true
+					break
+				}
+			}
+
+			if policiesDiffer {
+				return fmt.Errorf("multiple DNs map to the same LDAP DN[%s]: %v; please remove DNs that are not needed",
+					normKey, origKeys)
+			}
+
+			// Policies mapped to the DN's are the same, so we remove the extra
+			// ones from the map.
+			for i := 1; i < len(origKeys); i++ {
+				delete(policyMap, origKeys[i])
+
+				// Remove the mapping from storage by setting the policy to "".
+				if entityKeysInStorage.Contains(origKeys[i]) {
+					// Ignore any deletion error.
+					_, _ = sys.PolicyDBSet(ctx, origKeys[i], "", stsUser, isGroup)
+				}
+			}
+		}
+
+		// Replacing origKeys[0] with normKey in the policyMap
+
+		// len(origKeys) is always > 0, so here len(origKeys) == 1
+		mappingValue := policyMap[origKeys[0]]
+		delete(policyMap, origKeys[0])
+		policyMap[normKey] = mappingValue
+		// Remove the mapping from storage by setting the policy to "".
+		if entityKeysInStorage.Contains(origKeys[0]) {
+			// Ignore any deletion error.
+			_, _ = sys.PolicyDBSet(ctx, origKeys[0], "", stsUser, isGroup)
+		}
+	}
+	return nil
+}
+
 // GetUser - get user credentials
 func (sys *IAMSys) GetUser(ctx context.Context, accessKey string) (u UserIdentity, ok bool) {
 	if !sys.Initialized() {
@@ -1508,7 +1733,7 @@ func (sys *IAMSys) notifyForGroup(ctx context.Context, group string) {
 		for _, nerr := range globalNotificationSys.LoadGroup(group) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
+				iamLogIf(ctx, nerr.Err)
 			}
 		}
 	}
@@ -1521,10 +1746,6 @@ func (sys *IAMSys) AddUsersToGroup(ctx context.Context, group string, members []
 		return updatedAt, errServerNotInitialized
 	}
 
-	if sys.usersSysType != MinIOUsersSysType {
-		return updatedAt, errIAMActionNotAllowed
-	}
-
 	updatedAt, err = sys.store.AddUsersToGroup(ctx, group, members)
 	if err != nil {
 		return updatedAt, err
@@ -1596,7 +1817,9 @@ func (sys *IAMSys) ListGroups(ctx context.Context) (r []string, err error) {
 	}
 }
 
-// PolicyDBSet - sets a policy for a user or group in the PolicyDB - the user doesn't have to exist since sometimes they are virtuals
+// PolicyDBSet - sets a policy for a user or group in the PolicyDB. This does
+// not validate if the user/group exists - that is the responsibility of the
+// caller.
 func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, userType IAMUserType, isGroup bool) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
 		return updatedAt, errServerNotInitialized
@@ -1612,7 +1835,7 @@ func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, userTyp
 		for _, nerr := range globalNotificationSys.LoadPolicyMapping(name, userType, isGroup) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
+				iamLogIf(ctx, nerr.Err)
 			}
 		}
 	}
@@ -1680,12 +1903,12 @@ func (sys *IAMSys) PolicyDBUpdateBuiltin(ctx context.Context, isAttach bool,
 		for _, nerr := range globalNotificationSys.LoadPolicyMapping(userOrGroup, regUser, isGroup) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
+				iamLogIf(ctx, nerr.Err)
 			}
 		}
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemPolicyMapping,
 		PolicyMapping: &madmin.SRPolicyMapping{
 			UserOrGroup: userOrGroup,
@@ -1714,7 +1937,7 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
 	if r.User != "" {
 		dn, err = sys.LDAPConfig.GetValidatedDNForUsername(r.User)
 		if err != nil {
-			logger.LogIf(ctx, err)
+			iamLogIf(ctx, err)
 			return
 		}
 		if dn == "" {
@@ -1730,10 +1953,11 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
 	} else {
 		if isAttach {
 			var foundGroupDN string
-			if foundGroupDN, err = sys.LDAPConfig.GetValidatedGroupDN(r.Group); err != nil {
-				logger.LogIf(ctx, err)
+			var underBaseDN bool
+			if foundGroupDN, underBaseDN, err = sys.LDAPConfig.GetValidatedGroupDN(nil, r.Group); err != nil {
+				iamLogIf(ctx, err)
 				return
-			} else if foundGroupDN == "" {
+			} else if foundGroupDN == "" || !underBaseDN {
 				err = errNoSuchGroup
 				return
 			}
@@ -1746,6 +1970,19 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
 		isGroup = true
 	}
 
+	// Backward compatibility in detaching non-normalized DNs.
+	if !isAttach {
+		var oldDN string
+		if isGroup {
+			oldDN = r.Group
+		} else {
+			oldDN = r.User
+		}
+		if oldDN != dn {
+			sys.store.PolicyDBUpdate(ctx, oldDN, isGroup, stsUser, r.Policies, isAttach)
+		}
+	}
+
 	userType := stsUser
 	updatedAt, addedOrRemoved, effectivePolicies, err = sys.store.PolicyDBUpdate(
 		ctx, dn, isGroup, userType, r.Policies, isAttach)
@@ -1758,12 +1995,12 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
 		for _, nerr := range globalNotificationSys.LoadPolicyMapping(dn, userType, isGroup) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
+				iamLogIf(ctx, nerr.Err)
 			}
 		}
 	}
 
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemPolicyMapping,
 		PolicyMapping: &madmin.SRPolicyMapping{
 			UserOrGroup: dn,
@@ -1826,7 +2063,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args policy.Args, parentUser string)
 	case roleArn != "":
 		arn, err := arn.Parse(roleArn)
 		if err != nil {
-			logger.LogIf(GlobalContext, fmt.Errorf("error parsing role ARN %s: %v", roleArn, err))
+			iamLogIf(GlobalContext, fmt.Errorf("error parsing role ARN %s: %v", roleArn, err))
 			return false
 		}
 		svcPolicies = newMappedPolicy(sys.rolesMap[arn]).toSlice()
@@ -1835,7 +2072,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args policy.Args, parentUser string)
 		// Check policy for parent user of service account.
 		svcPolicies, err = sys.PolicyDBGet(parentUser, args.Groups...)
 		if err != nil {
-			logger.LogIf(GlobalContext, err)
+			iamLogIf(GlobalContext, err)
 			return false
 		}
 
@@ -1910,7 +2147,7 @@ func (sys *IAMSys) IsAllowedSTS(args policy.Args, parentUser string) bool {
 		// If a roleARN is present, the role policy is applied.
 		arn, err := arn.Parse(roleArn)
 		if err != nil {
-			logger.LogIf(GlobalContext, fmt.Errorf("error parsing role ARN %s: %v", roleArn, err))
+			iamLogIf(GlobalContext, fmt.Errorf("error parsing role ARN %s: %v", roleArn, err))
 			return false
 		}
 		policies = newMappedPolicy(sys.rolesMap[arn]).toSlice()
@@ -1918,9 +2155,9 @@ func (sys *IAMSys) IsAllowedSTS(args policy.Args, parentUser string) bool {
 	default:
 		// Otherwise, inherit parent user's policy
 		var err error
-		policies, err = sys.store.PolicyDBGet(parentUser, args.Groups...)
+		policies, err = sys.PolicyDBGet(parentUser, args.Groups...)
 		if err != nil {
-			logger.LogIf(GlobalContext, fmt.Errorf("error fetching policies on %s: %v", parentUser, err))
+			iamLogIf(GlobalContext, fmt.Errorf("error fetching policies on %s: %v", parentUser, err))
 			return false
 		}
 
@@ -1955,11 +2192,11 @@ func (sys *IAMSys) IsAllowedSTS(args policy.Args, parentUser string) bool {
 				_, err := sys.store.GetPolicy(pname)
 				if errors.Is(err, errNoSuchPolicy) {
 					// all policies presented in the claim should exist
-					logger.LogIf(GlobalContext, fmt.Errorf("expected policy (%s) missing from the JWT claim %s, rejecting the request", pname, iamPolicyClaimNameOpenID()))
+					iamLogIf(GlobalContext, fmt.Errorf("expected policy (%s) missing from the JWT claim %s, rejecting the request", pname, iamPolicyClaimNameOpenID()))
 					return false
 				}
 			}
-			logger.LogIf(GlobalContext, fmt.Errorf("all policies were unexpectedly present!"))
+			iamLogIf(GlobalContext, fmt.Errorf("all policies were unexpectedly present!"))
 			return false
 		}
 
@@ -2001,7 +2238,7 @@ func isAllowedBySessionPolicyForServiceAccount(args policy.Args) (hasSessionPoli
 	subPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
 	if err != nil {
 		// Log any error in input session policy config.
-		logger.LogIf(GlobalContext, err)
+		iamLogIf(GlobalContext, err)
 		return
 	}
 
@@ -2062,7 +2299,7 @@ func isAllowedBySessionPolicy(args policy.Args) (hasSessionPolicy bool, isAllowe
 	subPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
 	if err != nil {
 		// Log any error in input session policy config.
-		logger.LogIf(GlobalContext, err)
+		iamLogIf(GlobalContext, err)
 		return
 	}
 
@@ -2100,7 +2337,7 @@ func (sys *IAMSys) IsAllowed(args policy.Args) bool {
 	if authz := newGlobalAuthZPluginFn(); authz != nil {
 		ok, err := authz.IsAllowed(args)
 		if err != nil {
-			logger.LogIf(GlobalContext, err)
+			authZLogIf(GlobalContext, err)
 		}
 		return ok
 	}

cmd/is-dir-empty_linux.go

@@ -25,22 +25,19 @@ import (
 )
 
 // Returns true if no error and there is no object or prefix inside this directory
-func isDirEmpty(dirname string) bool {
-	var stat syscall.Stat_t
-	if err := syscall.Stat(dirname, &stat); err != nil {
-		return false
-	}
-	if stat.Mode&syscall.S_IFMT == syscall.S_IFDIR && stat.Nlink == 2 {
-		return true
-	}
-	// On filesystems such as btrfs, nfs this is not true, so fallback
-	// to performing readdir() instead.
-	if stat.Mode&syscall.S_IFMT == syscall.S_IFDIR && stat.Nlink < 2 {
+func isDirEmpty(dirname string, legacy bool) bool {
+	if legacy {
+		// On filesystems such as btrfs, nfs this is not true, so fallback
+		// to performing readdir() instead.
 		entries, err := readDirN(dirname, 1)
 		if err != nil {
 			return false
 		}
 		return len(entries) == 0
 	}
-	return false
+	var stat syscall.Stat_t
+	if err := syscall.Stat(dirname, &stat); err != nil {
+		return false
+	}
+	return stat.Mode&syscall.S_IFMT == syscall.S_IFDIR && stat.Nlink == 2
 }

cmd/is-dir-empty_other.go

@@ -21,7 +21,7 @@
 package cmd
 
 // isDirEmpty - returns true if there is no error and no object and prefix inside this directory
-func isDirEmpty(dirname string) bool {
+func isDirEmpty(dirname string, _ bool) bool {
 	entries, err := readDirN(dirname, 1)
 	if err != nil {
 		return false

cmd/jwt.go

@@ -24,7 +24,7 @@ import (
 
 	jwtgo "github.com/golang-jwt/jwt/v4"
 	jwtreq "github.com/golang-jwt/jwt/v4/request"
-	lru "github.com/hashicorp/golang-lru"
+	"github.com/hashicorp/golang-lru/v2/expirable"
 	"github.com/minio/minio/internal/auth"
 	xjwt "github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
@@ -50,35 +50,12 @@ var (
 	errMalformedAuth      = errors.New("Malformed authentication input")
 )
 
-// cachedAuthenticateNode will cache authenticateNode results for given values up to ttl.
-func cachedAuthenticateNode(ttl time.Duration) func(accessKey, secretKey, audience string) (string, error) {
-	type key struct {
-		accessKey, secretKey, audience string
-	}
-	type value struct {
-		created time.Time
-		res     string
-		err     error
-	}
-	cache, err := lru.NewARC(100)
-	if err != nil {
-		logger.LogIf(GlobalContext, err)
-		return authenticateNode
-	}
-	return func(accessKey, secretKey, audience string) (string, error) {
-		k := key{accessKey: accessKey, secretKey: secretKey, audience: audience}
-		v, ok := cache.Get(k)
-		if ok {
-			if val, ok := v.(*value); ok && time.Since(val.created) < ttl {
-				return val.res, val.err
-			}
-		}
-		s, err := authenticateNode(accessKey, secretKey, audience)
-		cache.Add(k, &value{created: time.Now(), res: s, err: err})
-		return s, err
-	}
+type cacheKey struct {
+	accessKey, secretKey, audience string
 }
 
+var cacheLRU = expirable.NewLRU[cacheKey, string](1000, nil, 15*time.Second)
+
 func authenticateNode(accessKey, secretKey, audience string) (string, error) {
 	claims := xjwt.NewStandardClaims()
 	claims.SetExpiry(UTCNow().Add(defaultInterNodeJWTExpiry))
@@ -167,7 +144,20 @@ func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, b
 // newCachedAuthToken returns a token that is cached up to 15 seconds.
 // If globalActiveCred is updated it is reflected at once.
 func newCachedAuthToken() func(audience string) string {
-	fn := cachedAuthenticateNode(15 * time.Second)
+	fn := func(accessKey, secretKey, audience string) (s string, err error) {
+		k := cacheKey{accessKey: accessKey, secretKey: secretKey, audience: audience}
+
+		var ok bool
+		s, ok = cacheLRU.Get(k)
+		if !ok {
+			s, err = authenticateNode(accessKey, secretKey, audience)
+			if err != nil {
+				return "", err
+			}
+			cacheLRU.Add(k, s)
+		}
+		return s, nil
+	}
 	return func(audience string) string {
 		cred := globalActiveCred
 		token, err := fn(cred.AccessKey, cred.SecretKey, audience)

cmd/jwt_test.go

@@ -22,7 +22,6 @@ import (
 	"net/http"
 	"os"
 	"testing"
-	"time"
 
 	jwtgo "github.com/golang-jwt/jwt/v4"
 	xjwt "github.com/minio/minio/internal/jwt"
@@ -181,11 +180,11 @@ func BenchmarkAuthenticateNode(b *testing.B) {
 		}
 	})
 	b.Run("cached", func(b *testing.B) {
-		fn := cachedAuthenticateNode(time.Second)
+		fn := newCachedAuthToken()
 		b.ResetTimer()
 		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
-			fn(creds.AccessKey, creds.SecretKey, "aud")
+			fn("aud")
 		}
 	})
 }

cmd/license-update.go

@@ -23,7 +23,6 @@ import (
 	"math/rand"
 	"time"
 
-	"github.com/minio/minio/internal/logger"
 	"github.com/tidwall/gjson"
 )
 
@@ -85,13 +84,13 @@ func performLicenseUpdate(ctx context.Context, objectAPI ObjectLayer) {
 
 	resp, err := globalSubnetConfig.Post(url, nil)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("error from %s: %w", url, err))
+		subnetLogIf(ctx, fmt.Errorf("error from %s: %w", url, err))
 		return
 	}
 
 	r := gjson.Parse(resp).Get("license_v2")
 	if r.Index == 0 {
-		logger.LogIf(ctx, fmt.Errorf("license not found in response from %s", url))
+		internalLogIf(ctx, fmt.Errorf("license not found in response from %s", url))
 		return
 	}
 
@@ -104,13 +103,13 @@ func performLicenseUpdate(ctx context.Context, objectAPI ObjectLayer) {
 	kv := "subnet license=" + lic
 	result, err := setConfigKV(ctx, objectAPI, []byte(kv))
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("error setting subnet license config: %w", err))
+		internalLogIf(ctx, fmt.Errorf("error setting subnet license config: %w", err))
 		return
 	}
 
 	if result.Dynamic {
 		if err := applyDynamicConfigForSubSys(GlobalContext, objectAPI, result.Cfg, result.SubSys); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("error applying subnet dynamic config: %w", err))
+			subnetLogIf(ctx, fmt.Errorf("error applying subnet dynamic config: %w", err))
 			return
 		}
 		globalNotificationSys.SignalConfigReload(result.SubSys)

cmd/listen-notification-handlers.go

@@ -132,7 +132,7 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 				buf.Reset()
 				tmpEvt.Records[0] = ev
 				if err := enc.Encode(tmpEvt); err != nil {
-					logger.LogOnceIf(ctx, err, "event: Encode failed")
+					bugLogIf(ctx, err, "event: Encode failed")
 					continue
 				}
 				mergeCh <- append(grid.GetByteBuffer()[:0], buf.Bytes()...)

cmd/logging.go

@@ -0,0 +1,208 @@
+package cmd
+
+import (
+	"context"
+
+	"github.com/minio/minio/internal/logger"
+)
+
+func replLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "replication", err, errKind...)
+}
+
+func replLogOnceIf(ctx context.Context, err error, id string, errKind ...interface{}) {
+	logger.LogOnceIf(ctx, "replication", err, id, errKind...)
+}
+
+func iamLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "iam", err, errKind...)
+}
+
+func rebalanceLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "rebalance", err, errKind...)
+}
+
+func rebalanceLogEvent(ctx context.Context, msg string, args ...interface{}) {
+	logger.Event(ctx, "rebalance", msg, args...)
+}
+
+func adminLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "admin", err, errKind...)
+}
+
+func authNLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "authN", err, errKind...)
+}
+
+func authZLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "authZ", err, errKind...)
+}
+
+func peersLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "peers", err, errKind...)
+}
+
+func peersLogAlwaysIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogAlwaysIf(ctx, "peers", err, errKind...)
+}
+
+func peersLogOnceIf(ctx context.Context, err error, id string, errKind ...interface{}) {
+	logger.LogOnceIf(ctx, "peers", err, id, errKind...)
+}
+
+func bugLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "internal", err, errKind...)
+}
+
+func healingLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "healing", err, errKind...)
+}
+
+func healingLogEvent(ctx context.Context, msg string, args ...interface{}) {
+	logger.Event(ctx, "healing", msg, args...)
+}
+
+func healingLogOnceIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "healing", err, errKind...)
+}
+
+func batchLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "batch", err, errKind...)
+}
+
+func batchLogOnceIf(ctx context.Context, err error, id string, errKind ...interface{}) {
+	logger.LogOnceIf(ctx, "batch", err, id, errKind...)
+}
+
+func bootLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "bootstrap", err, errKind...)
+}
+
+func bootLogOnceIf(ctx context.Context, err error, id string, errKind ...interface{}) {
+	logger.LogOnceIf(ctx, "bootstrap", err, id, errKind...)
+}
+
+func dnsLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "dns", err, errKind...)
+}
+
+func internalLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "internal", err, errKind...)
+}
+
+func internalLogOnceIf(ctx context.Context, err error, id string, errKind ...interface{}) {
+	logger.LogOnceIf(ctx, "internal", err, id, errKind...)
+}
+
+func transitionLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "transition", err, errKind...)
+}
+
+func configLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "config", err, errKind...)
+}
+
+func configLogOnceIf(ctx context.Context, err error, id string, errKind ...interface{}) {
+	logger.LogOnceIf(ctx, "config", err, id, errKind...)
+}
+
+func configLogOnceConsoleIf(ctx context.Context, err error, id string, errKind ...interface{}) {
+	logger.LogOnceConsoleIf(ctx, "config", err, id, errKind...)
+}
+
+func scannerLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "scanner", err, errKind...)
+}
+
+func scannerLogOnceIf(ctx context.Context, err error, id string, errKind ...interface{}) {
+	logger.LogOnceIf(ctx, "scanner", err, id, errKind...)
+}
+
+func ilmLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "ilm", err, errKind...)
+}
+
+func ilmLogOnceIf(ctx context.Context, err error, id string, errKind ...interface{}) {
+	logger.LogOnceIf(ctx, "ilm", err, id, errKind...)
+}
+
+func encLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "encryption", err, errKind...)
+}
+
+func storageLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "storage", err, errKind...)
+}
+
+func storageLogAlwaysIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogAlwaysIf(ctx, "storage", err, errKind...)
+}
+
+func storageLogOnceIf(ctx context.Context, err error, id string, errKind ...interface{}) {
+	logger.LogOnceIf(ctx, "storage", err, id, errKind...)
+}
+
+func decomLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "decom", err, errKind...)
+}
+
+func decomLogOnceIf(ctx context.Context, err error, id string, errKind ...interface{}) {
+	logger.LogOnceIf(ctx, "decom", err, id, errKind...)
+}
+
+func decomLogEvent(ctx context.Context, msg string, args ...interface{}) {
+	logger.Event(ctx, "decom", msg, args...)
+}
+
+func etcdLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "etcd", err, errKind...)
+}
+
+func etcdLogOnceIf(ctx context.Context, err error, id string, errKind ...interface{}) {
+	logger.LogOnceIf(ctx, "etcd", err, id, errKind...)
+}
+
+func subnetLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "subnet", err, errKind...)
+}
+
+func metricsLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "metrics", err, errKind...)
+}
+
+func s3LogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "s3", err, errKind...)
+}
+
+func sftpLogOnceIf(ctx context.Context, err error, id string, errKind ...interface{}) {
+	logger.LogOnceIf(ctx, "sftp", err, id, errKind...)
+}
+
+func shutdownLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "shutdown", err, errKind...)
+}
+
+func stsLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "sts", err, errKind...)
+}
+
+func tierLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "tier", err, errKind...)
+}
+
+func kmsLogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "kms", err, errKind...)
+}
+
+// KMSLogger permits access to kms module specific logging
+type KMSLogger struct{}
+
+// LogOnceIf is the implementation of LogOnceIf, accessible using the Logger interface
+func (l KMSLogger) LogOnceIf(ctx context.Context, err error, id string, errKind ...interface{}) {
+	logger.LogOnceIf(ctx, "kms", err, id, errKind...)
+}
+
+// LogIf is the implementation of LogIf, accessible using the Logger interface
+func (l KMSLogger) LogIf(ctx context.Context, err error, errKind ...interface{}) {
+	logger.LogIf(ctx, "kms", err, errKind...)
+}

cmd/metacache-bucket.go

@@ -221,7 +221,7 @@ func (b *bucketMetacache) deleteAll() {
 
 	ez, ok := objAPI.(deleteAllStorager)
 	if !ok {
-		logger.LogIf(ctx, errors.New("bucketMetacache: expected objAPI to be 'deleteAllStorager'"))
+		bugLogIf(ctx, errors.New("bucketMetacache: expected objAPI to be 'deleteAllStorager'"))
 		return
 	}
 

cmd/metacache-entries.go

@@ -27,7 +27,6 @@ import (
 	"strings"
 
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 )
 
@@ -121,6 +120,16 @@ func (e *metaCacheEntry) matches(other *metaCacheEntry, strict bool) (prefer *me
 	for i, eVer := range eVers.versions {
 		oVer := oVers.versions[i]
 		if eVer.header != oVer.header {
+			if eVer.header.hasEC() != oVer.header.hasEC() {
+				// One version has EC and the other doesn't - may have been written later.
+				// Compare without considering EC.
+				a, b := eVer.header, oVer.header
+				a.EcN, a.EcM = 0, 0
+				b.EcN, b.EcM = 0, 0
+				if a == b {
+					continue
+				}
+			}
 			if !strict && eVer.header.matchesNotStrict(oVer.header) {
 				if prefer == nil {
 					if eVer.header.sortsBefore(oVer.header) {
@@ -377,7 +386,7 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa
 		xl, err := entry.xlmeta()
 		if err != nil {
 			if !errors.Is(err, errFileNotFound) {
-				logger.LogIf(GlobalContext, err)
+				internalLogIf(GlobalContext, err)
 			}
 			continue
 		}
@@ -437,7 +446,7 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa
 	var err error
 	selected.metadata, err = selected.cached.AppendTo(metaDataPoolGet())
 	if err != nil {
-		logger.LogIf(context.Background(), err)
+		bugLogIf(context.Background(), err)
 		return nil, false
 	}
 	return selected, true

cmd/metacache-marker.go

@@ -22,8 +22,6 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
-
-	"github.com/minio/minio/internal/logger"
 )
 
 // markerTagVersion is the marker version.
@@ -86,7 +84,7 @@ func (o listPathOptions) encodeMarker(marker string) string {
 		return fmt.Sprintf("%s[minio_cache:%s,return:]", marker, markerTagVersion)
 	}
 	if strings.ContainsAny(o.ID, "[:,") {
-		logger.LogIf(context.Background(), fmt.Errorf("encodeMarker: uuid %s contained invalid characters", o.ID))
+		internalLogIf(context.Background(), fmt.Errorf("encodeMarker: uuid %s contained invalid characters", o.ID))
 	}
 	return fmt.Sprintf("%s[minio_cache:%s,id:%s,p:%d,s:%d]", marker, markerTagVersion, o.ID, o.pool, o.set)
 }

cmd/metacache-server-pool.go

@@ -29,7 +29,6 @@ import (
 	"time"
 
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 )
 
 func renameAllBucketMetacache(epPath string) error {
@@ -136,7 +135,7 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 			}
 			if !errors.Is(err, context.DeadlineExceeded) {
 				// Report error once per bucket, but continue listing.
-				logger.LogOnceIf(ctx, err, "GetMetacacheListing:"+o.Bucket)
+				storageLogOnceIf(ctx, err, "GetMetacacheListing:"+o.Bucket)
 			}
 			o.Transient = true
 			o.Create = false
@@ -322,7 +321,7 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 			allAtEOF = false
 			continue
 		}
-		logger.LogIf(ctx, err)
+		storageLogIf(ctx, err)
 		return err
 	}
 	if allAtEOF {

cmd/metacache-set.go

@@ -38,7 +38,6 @@ import (
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/hash"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 )
 
@@ -285,7 +284,7 @@ func (o *listPathOptions) findFirstPart(fi FileInfo) (int, error) {
 		}
 		err := json.Unmarshal([]byte(v), &tmp)
 		if !ok {
-			logger.LogIf(context.Background(), err)
+			bugLogIf(context.Background(), err)
 			return -1, err
 		}
 		if tmp.First == "" && tmp.Last == "" && tmp.EOS {
@@ -538,7 +537,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				}
 				loadedPart = partN
 				bi, err := getMetacacheBlockInfo(fi, partN)
-				logger.LogIf(ctx, err)
+				internalLogIf(ctx, err)
 				if err == nil {
 					if bi.pastPrefix(o.Prefix) {
 						return entries, io.EOF
@@ -574,10 +573,11 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 					continue
 				case InsufficientReadQuorum:
 					retries++
+					loadedPart = -1
 					time.Sleep(retryDelay250)
 					continue
 				default:
-					logger.LogIf(ctx, err)
+					internalLogIf(ctx, err)
 					return entries, err
 				}
 			}
@@ -585,7 +585,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 			// We finished at the end of the block.
 			// And should not expect any more results.
 			bi, err := getMetacacheBlockInfo(fi, partN)
-			logger.LogIf(ctx, err)
+			internalLogIf(ctx, err)
 			if err != nil || bi.EOS {
 				// We are done and there are no more parts.
 				return entries, io.EOF
@@ -868,7 +868,7 @@ func (er *erasureObjects) saveMetaCacheStream(ctx context.Context, mc *metaCache
 		}
 		o.debugln(color.Green("saveMetaCacheStream:")+" saving block", b.n, "to", o.objectPath(b.n))
 		r, err := hash.NewReader(ctx, bytes.NewReader(b.data), int64(len(b.data)), "", "", int64(len(b.data)))
-		logger.LogIf(ctx, err)
+		bugLogIf(ctx, err)
 		custom := b.headerKV()
 		_, err = er.putMetacacheObject(ctx, o.objectPath(b.n), NewPutObjReader(r), ObjectOptions{
 			UserDefined: custom,
@@ -902,7 +902,7 @@ func (er *erasureObjects) saveMetaCacheStream(ctx context.Context, mc *metaCache
 				return err
 			case InsufficientReadQuorum:
 			default:
-				logger.LogIf(ctx, err)
+				internalLogIf(ctx, err)
 			}
 			if retries >= maxTries {
 				return err
@@ -1095,8 +1095,7 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 				switch err.Error() {
 				case errFileNotFound.Error(),
 					errVolumeNotFound.Error(),
-					errUnformattedDisk.Error(),
-					errDiskNotFound.Error():
+					errUnformattedDisk.Error():
 					atEOF++
 					fnf++
 					// This is a special case, to handle bucket does

cmd/metacache-stream.go

@@ -28,7 +28,6 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"github.com/klauspost/compress/s2"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	"github.com/tinylib/msgp/msgp"
 	"github.com/valyala/bytebufferpool"
 )
@@ -845,7 +844,7 @@ func (b metacacheBlock) headerKV() map[string]string {
 	json := jsoniter.ConfigCompatibleWithStandardLibrary
 	v, err := json.Marshal(b)
 	if err != nil {
-		logger.LogIf(context.Background(), err) // Unlikely
+		bugLogIf(context.Background(), err) // Unlikely
 		return nil
 	}
 	return map[string]string{fmt.Sprintf("%s-metacache-part-%d", ReservedMetadataPrefixLower, b.n): string(v)}

cmd/metacache-walk.go

@@ -25,7 +25,6 @@ import (
 
 	"github.com/minio/minio/internal/grid"
 	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 	"github.com/valyala/bytebufferpool"
 )
 
@@ -60,10 +59,22 @@ type WalkDirOptions struct {
 	DiskID string
 }
 
+// supported FS for Nlink optimization in readdir.
+const (
+	xfs  = "XFS"
+	ext4 = "EXT4"
+)
+
 // WalkDir will traverse a directory and return all entries found.
 // On success a sorted meta cache stream will be returned.
 // Metadata has data stripped, if any.
 func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) (err error) {
+	legacyFS := !(s.fsType == xfs || s.fsType == ext4)
+
+	s.RLock()
+	legacy := s.formatLegacy
+	s.RUnlock()
+
 	// Verify if volume is valid and it exists.
 	volumeDir, err := s.getVolDir(opts.Bucket)
 	if err != nil {
@@ -77,10 +88,6 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 		}
 	}
 
-	s.RLock()
-	legacy := s.formatLegacy
-	s.RUnlock()
-
 	// Use a small block size to start sending quickly
 	w := newMetacacheWriter(wr, 16<<10)
 	w.reuseBlocks = true // We are not sharing results, so reuse buffers.
@@ -171,7 +178,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 		if err != nil {
 			// Folder could have gone away in-between
 			if err != errVolumeNotFound && err != errFileNotFound {
-				logger.LogOnceIf(ctx, err, "metacache-walk-scan-dir")
+				internalLogOnceIf(ctx, err, "metacache-walk-scan-dir")
 			}
 			if opts.ReportNotFound && err == errFileNotFound && current == opts.BaseDir {
 				err = errFileNotFound
@@ -239,7 +246,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 					// while being concurrently listed at the same time in
 					// such scenarios the 'xl.meta' might get truncated
 					if !IsErrIgnored(err, io.EOF, io.ErrUnexpectedEOF) {
-						logger.LogOnceIf(ctx, err, "metacache-walk-read-metadata")
+						internalLogOnceIf(ctx, err, "metacache-walk-read-metadata")
 					}
 					continue
 				}
@@ -257,7 +264,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				diskHealthCheckOK(ctx, err)
 				if err != nil {
 					if !IsErrIgnored(err, io.EOF, io.ErrUnexpectedEOF) {
-						logger.LogIf(ctx, err)
+						internalLogIf(ctx, err)
 					}
 					continue
 				}
@@ -308,7 +315,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 					// Scan folder we found. Should be in correct sort order where we are.
 					err := scanDir(pop)
 					if err != nil && !IsErrIgnored(err, context.Canceled) {
-						logger.LogIf(ctx, err)
+						internalLogIf(ctx, err)
 					}
 				}
 				dirStack = dirStack[:len(dirStack)-1]
@@ -354,7 +361,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				// NOT an object, append to stack (with slash)
 				// If dirObject, but no metadata (which is unexpected) we skip it.
 				if !isDirObj {
-					if !isDirEmpty(pathJoinBuf(sb, volumeDir, meta.name)) {
+					if !isDirEmpty(pathJoinBuf(sb, volumeDir, meta.name), legacyFS) {
 						dirStack = append(dirStack, meta.name+slashSeparator)
 					}
 				}
@@ -379,7 +386,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			}
 			if opts.Recursive {
 				// Scan folder we found. Should be in correct sort order where we are.
-				logger.LogIf(ctx, scanDir(pop))
+				internalLogIf(ctx, scanDir(pop))
 			}
 			dirStack = dirStack[:len(dirStack)-1]
 		}
@@ -407,7 +414,7 @@ func (p *xlStorageDiskIDCheck) WalkDir(ctx context.Context, opts WalkDirOptions,
 // On success a meta cache stream will be returned, that should be closed when done.
 func (client *storageRESTClient) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) error {
 	// Ensure remote has the same disk ID.
-	opts.DiskID = client.diskID
+	opts.DiskID = *client.diskID.Load()
 	b, err := opts.MarshalMsg(grid.GetByteBuffer()[:0])
 	if err != nil {
 		return toStorageErr(err)

cmd/metacache.go

@@ -24,8 +24,6 @@ import (
 	"path"
 	"strings"
 	"time"
-
-	"github.com/minio/minio/internal/logger"
 )
 
 type scanStatus uint8
@@ -148,16 +146,17 @@ func (m *metacache) update(update metacache) {
 // delete all cache data on disks.
 func (m *metacache) delete(ctx context.Context) {
 	if m.bucket == "" || m.id == "" {
-		logger.LogIf(ctx, fmt.Errorf("metacache.delete: bucket (%s) or id (%s) empty", m.bucket, m.id))
+		bugLogIf(ctx, fmt.Errorf("metacache.delete: bucket (%s) or id (%s) empty", m.bucket, m.id))
+		return
 	}
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
-		logger.LogIf(ctx, errors.New("metacache.delete: no object layer"))
+		internalLogIf(ctx, errors.New("metacache.delete: no object layer"))
 		return
 	}
 	ez, ok := objAPI.(deleteAllStorager)
 	if !ok {
-		logger.LogIf(ctx, errors.New("metacache.delete: expected objAPI to be 'deleteAllStorager'"))
+		bugLogIf(ctx, errors.New("metacache.delete: expected objAPI to be 'deleteAllStorager'"))
 		return
 	}
 	ez.deleteAll(ctx, minioMetaBucket, metacachePrefixForID(m.bucket, m.id))

cmd/metrics-realtime.go

@@ -20,6 +20,8 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"net/http"
+	"strings"
 	"time"
 
 	"github.com/minio/madmin-go/v3"
@@ -41,6 +43,22 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 		return
 	}
 
+	byHostName := globalMinioAddr
+	if len(opts.hosts) > 0 {
+		server := getLocalServerProperty(globalEndpoints, &http.Request{
+			Host: globalLocalNodeName,
+		}, false)
+		if _, ok := opts.hosts[server.Endpoint]; ok {
+			byHostName = server.Endpoint
+		} else {
+			return
+		}
+	}
+
+	if strings.HasPrefix(byHostName, ":") && !strings.HasPrefix(globalLocalNodeName, ":") {
+		byHostName = globalLocalNodeName
+	}
+
 	if types.Contains(madmin.MetricsDisk) {
 		m.ByDisk = make(map[string]madmin.DiskMetric)
 		aggr := madmin.DiskMetric{
@@ -74,7 +92,7 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 		}
 		netStats, err := net.GetInterfaceNetStats(globalInternodeInterface)
 		if err != nil {
-			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v  (nicstats)", globalMinioAddr, err.Error()))
+			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v  (nicstats)", byHostName, err.Error()))
 		} else {
 			m.Aggregated.Net.NetStats = netStats
 		}
@@ -83,7 +101,7 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 		m.Aggregated.Mem = &madmin.MemMetrics{
 			CollectedAt: UTCNow(),
 		}
-		m.Aggregated.Mem.Info = madmin.GetMemInfo(GlobalContext, globalMinioAddr)
+		m.Aggregated.Mem.Info = madmin.GetMemInfo(GlobalContext, byHostName)
 	}
 	if types.Contains(madmin.MetricsCPU) {
 		m.Aggregated.CPU = &madmin.CPUMetrics{
@@ -91,25 +109,25 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 		}
 		cm, err := c.Times(false)
 		if err != nil {
-			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (cpuTimes)", globalMinioAddr, err.Error()))
+			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (cpuTimes)", byHostName, err.Error()))
 		} else {
 			// not collecting per-cpu stats, so there will be only one element
 			if len(cm) == 1 {
 				m.Aggregated.CPU.TimesStat = &cm[0]
 			} else {
-				m.Errors = append(m.Errors, fmt.Sprintf("%s: Expected one CPU stat, got %d", globalMinioAddr, len(cm)))
+				m.Errors = append(m.Errors, fmt.Sprintf("%s: Expected one CPU stat, got %d", byHostName, len(cm)))
 			}
 		}
 		cpuCount, err := c.Counts(true)
 		if err != nil {
-			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (cpuCount)", globalMinioAddr, err.Error()))
+			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (cpuCount)", byHostName, err.Error()))
 		} else {
 			m.Aggregated.CPU.CPUCount = cpuCount
 		}
 
 		loadStat, err := load.Avg()
 		if err != nil {
-			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (loadStat)", globalMinioAddr, err.Error()))
+			m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (loadStat)", byHostName, err.Error()))
 		} else {
 			m.Aggregated.CPU.LoadStat = loadStat
 		}
@@ -117,8 +135,8 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 	// Add types...
 
 	// ByHost is a shallow reference, so careful about sharing.
-	m.ByHost = map[string]madmin.Metrics{globalMinioAddr: m.Aggregated}
-	m.Hosts = append(m.Hosts, globalMinioAddr)
+	m.ByHost = map[string]madmin.Metrics{byHostName: m.Aggregated}
+	m.Hosts = append(m.Hosts, byHostName)
 
 	return m
 }